@withgateway/sdk 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +12 -4
- package/dist/agents/index.d.ts +12 -4
- package/dist/agents/index.js +30 -0
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/index.mjs +30 -0
- package/dist/agents/index.mjs.map +1 -1
- package/dist/cli.js +2291 -945
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2290 -944
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.js +325 -184
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +324 -183
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +3 -1
- package/dist/worlds/index.d.ts +3 -1
- package/dist/worlds/index.js +178 -67
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +177 -66
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +21 -2
- package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
- package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -0
- package/skills/connector-schema-authoring/connector-worlds.md +1 -0
- package/skills/connector-schema-authoring/world-schema.md +7 -0
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +4 -4
- package/skills/worlds-getting-started/SKILL.md +2 -2
- package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
package/dist/worlds/index.mjs
CHANGED
|
@@ -145,8 +145,8 @@ function invalidFields(error) {
|
|
|
145
145
|
return issues.filter(
|
|
146
146
|
(issue) => issue !== null && typeof issue === "object" && typeof issue.message === "string"
|
|
147
147
|
).map((issue) => {
|
|
148
|
-
const
|
|
149
|
-
return
|
|
148
|
+
const path14 = Array.isArray(issue.path) ? issue.path.map(String).join(".") : "";
|
|
149
|
+
return path14 ? `${path14}: ${issue.message}` : issue.message;
|
|
150
150
|
}).join("; ");
|
|
151
151
|
}
|
|
152
152
|
var ApiError, DETAIL_LIMIT, EDGE_ANSWERS, EDGE_FAILURE, EDGE_PAGE;
|
|
@@ -154,9 +154,9 @@ var init_apiError = __esm({
|
|
|
154
154
|
"src/worlds/apiError.ts"() {
|
|
155
155
|
"use strict";
|
|
156
156
|
ApiError = class extends Error {
|
|
157
|
-
constructor(status, method,
|
|
157
|
+
constructor(status, method, path14, text2) {
|
|
158
158
|
const parsed = parseErrorBody(text2, status);
|
|
159
|
-
super(`${method} ${
|
|
159
|
+
super(`${method} ${path14} -> ${status}: ${parsed.detail}`);
|
|
160
160
|
this.status = status;
|
|
161
161
|
this.text = text2;
|
|
162
162
|
this.name = "ApiError";
|
|
@@ -792,9 +792,9 @@ function surfaceKinds(s) {
|
|
|
792
792
|
if (s.browser) kinds.push("browser");
|
|
793
793
|
return kinds;
|
|
794
794
|
}
|
|
795
|
-
function uiUrl(ui,
|
|
795
|
+
function uiUrl(ui, path14) {
|
|
796
796
|
const base = ui.url.replace(/\/+$/, "");
|
|
797
|
-
const rel =
|
|
797
|
+
const rel = path14.startsWith("/") ? path14 : `/${path14}`;
|
|
798
798
|
if (!ui.token) return base + rel;
|
|
799
799
|
const [beforeHash, hash] = splitHash(rel);
|
|
800
800
|
const joiner = beforeHash.includes("?") ? "&" : "?";
|
|
@@ -2212,13 +2212,13 @@ var init_screen = __esm({
|
|
|
2212
2212
|
});
|
|
2213
2213
|
}
|
|
2214
2214
|
async drag(action) {
|
|
2215
|
-
const
|
|
2216
|
-
if (
|
|
2215
|
+
const path14 = action.args.path ?? [];
|
|
2216
|
+
if (path14.length < 2) throw new Error("a drag needs at least two points");
|
|
2217
2217
|
return this.held(action.args.modifiers ?? [], async () => {
|
|
2218
|
-
await this.moveTo(
|
|
2218
|
+
await this.moveTo(path14[0]);
|
|
2219
2219
|
await this.page.mouse.down({ button: "left" });
|
|
2220
2220
|
try {
|
|
2221
|
-
for (const point2 of
|
|
2221
|
+
for (const point2 of path14.slice(1)) await this.moveTo(point2, DRAG_STEPS);
|
|
2222
2222
|
} finally {
|
|
2223
2223
|
await this.page.mouse.up({ button: "left" });
|
|
2224
2224
|
}
|
|
@@ -4052,17 +4052,17 @@ var init_session2 = __esm({
|
|
|
4052
4052
|
}
|
|
4053
4053
|
/** The reporter `browser()` uses; exposed for a page you drive yourself. */
|
|
4054
4054
|
browserRecorder(agentId) {
|
|
4055
|
-
const
|
|
4055
|
+
const path14 = `/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/browser-events`;
|
|
4056
4056
|
const id = agentId ?? process.env.GATEWAY_AGENT_ID ?? `agent-${randomUUID().replace(/-/g, "").slice(0, 8)}`;
|
|
4057
4057
|
return new BrowserActionRecorder(
|
|
4058
4058
|
async (events) => {
|
|
4059
|
-
await apiRequest(this.options, "POST",
|
|
4059
|
+
await apiRequest(this.options, "POST", path14, { events });
|
|
4060
4060
|
},
|
|
4061
4061
|
id,
|
|
4062
4062
|
{
|
|
4063
4063
|
// Continue this agent's timeline where the platform has it, so a reopened id never re-numbers from zero.
|
|
4064
4064
|
resumeFrom: async () => {
|
|
4065
|
-
const summary = await apiRequest(this.options, "GET",
|
|
4065
|
+
const summary = await apiRequest(this.options, "GET", path14);
|
|
4066
4066
|
return summary.agents.find((a) => a.agentId === id)?.nextSeq ?? 0;
|
|
4067
4067
|
}
|
|
4068
4068
|
}
|
|
@@ -5015,8 +5015,8 @@ var init_taskRunDir = __esm({
|
|
|
5015
5015
|
"use strict";
|
|
5016
5016
|
init_browser();
|
|
5017
5017
|
TaskRunDir = class {
|
|
5018
|
-
constructor(
|
|
5019
|
-
this.path =
|
|
5018
|
+
constructor(path14) {
|
|
5019
|
+
this.path = path14;
|
|
5020
5020
|
}
|
|
5021
5021
|
path;
|
|
5022
5022
|
writeManifest(manifest) {
|
|
@@ -5415,10 +5415,10 @@ var init_users = __esm({
|
|
|
5415
5415
|
totalItems: Number(meta.totalItems ?? rows.length)
|
|
5416
5416
|
};
|
|
5417
5417
|
}
|
|
5418
|
-
async requestJson(method,
|
|
5418
|
+
async requestJson(method, path14, payload) {
|
|
5419
5419
|
let response;
|
|
5420
5420
|
try {
|
|
5421
|
-
response = await fetch(`${this.host}${
|
|
5421
|
+
response = await fetch(`${this.host}${path14}`, {
|
|
5422
5422
|
method,
|
|
5423
5423
|
headers: {
|
|
5424
5424
|
Authorization: this.authHeader(),
|
|
@@ -5429,17 +5429,17 @@ var init_users = __esm({
|
|
|
5429
5429
|
});
|
|
5430
5430
|
} catch (error) {
|
|
5431
5431
|
throw new UserProfileError(
|
|
5432
|
-
`${method} ${
|
|
5432
|
+
`${method} ${path14} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
5433
5433
|
{ cause: error }
|
|
5434
5434
|
);
|
|
5435
5435
|
}
|
|
5436
5436
|
const text2 = await response.text();
|
|
5437
5437
|
if (!response.ok) {
|
|
5438
5438
|
throw new UserProfileError(
|
|
5439
|
-
`${method} ${
|
|
5439
|
+
`${method} ${path14} failed (${response.status}): ${text2.slice(0, 500)}`
|
|
5440
5440
|
);
|
|
5441
5441
|
}
|
|
5442
|
-
this.log("%s %s -> %s", method,
|
|
5442
|
+
this.log("%s %s -> %s", method, path14, response.status);
|
|
5443
5443
|
return safeJson(text2);
|
|
5444
5444
|
}
|
|
5445
5445
|
authHeader() {
|
|
@@ -5884,10 +5884,10 @@ var init_scores = __esm({
|
|
|
5884
5884
|
metadata: signalMetadata
|
|
5885
5885
|
});
|
|
5886
5886
|
}
|
|
5887
|
-
async request(method,
|
|
5887
|
+
async request(method, path14, payload) {
|
|
5888
5888
|
let response;
|
|
5889
5889
|
try {
|
|
5890
|
-
response = await fetch(`${this.host}${
|
|
5890
|
+
response = await fetch(`${this.host}${path14}`, {
|
|
5891
5891
|
method,
|
|
5892
5892
|
headers: {
|
|
5893
5893
|
Authorization: this.authHeader(),
|
|
@@ -5898,17 +5898,17 @@ var init_scores = __esm({
|
|
|
5898
5898
|
});
|
|
5899
5899
|
} catch (error) {
|
|
5900
5900
|
throw new ScoreError(
|
|
5901
|
-
`${method} ${
|
|
5901
|
+
`${method} ${path14} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
5902
5902
|
{ cause: error }
|
|
5903
5903
|
);
|
|
5904
5904
|
}
|
|
5905
5905
|
const text2 = await response.text();
|
|
5906
5906
|
if (!response.ok) {
|
|
5907
5907
|
throw new ScoreError(
|
|
5908
|
-
`${method} ${
|
|
5908
|
+
`${method} ${path14} failed (${response.status}): ${text2.slice(0, 500)}`
|
|
5909
5909
|
);
|
|
5910
5910
|
}
|
|
5911
|
-
this.log("%s %s -> %s", method,
|
|
5911
|
+
this.log("%s %s -> %s", method, path14, response.status);
|
|
5912
5912
|
}
|
|
5913
5913
|
authHeader() {
|
|
5914
5914
|
return `Basic ${Buffer.from(`${this.publicKey}:${this.secretKey}`).toString(
|
|
@@ -5936,7 +5936,7 @@ var require_package = __commonJS({
|
|
|
5936
5936
|
"package.json"(exports, module) {
|
|
5937
5937
|
module.exports = {
|
|
5938
5938
|
name: "@withgateway/sdk",
|
|
5939
|
-
version: "0.9.
|
|
5939
|
+
version: "0.9.3",
|
|
5940
5940
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
5941
5941
|
main: "./dist/index.js",
|
|
5942
5942
|
module: "./dist/index.mjs",
|
|
@@ -8320,7 +8320,7 @@ function rebaseConflicts(error) {
|
|
|
8320
8320
|
}
|
|
8321
8321
|
const conflicts = body?.conflicts;
|
|
8322
8322
|
if (!Array.isArray(conflicts)) return null;
|
|
8323
|
-
return conflicts.filter(isConflictPath).map(({ path:
|
|
8323
|
+
return conflicts.filter(isConflictPath).map(({ path: path14, reason }) => ({ path: path14, reason }));
|
|
8324
8324
|
}
|
|
8325
8325
|
function isConflictPath(value) {
|
|
8326
8326
|
const entry = value;
|
|
@@ -8444,8 +8444,8 @@ var init_sandbox = __esm({
|
|
|
8444
8444
|
this.options = options;
|
|
8445
8445
|
}
|
|
8446
8446
|
options;
|
|
8447
|
-
request(method,
|
|
8448
|
-
return apiRequest(this.options, method,
|
|
8447
|
+
request(method, path14, body) {
|
|
8448
|
+
return apiRequest(this.options, method, path14, body, {
|
|
8449
8449
|
timeoutMs: SANDBOX_REQUEST_TIMEOUT_MS
|
|
8450
8450
|
});
|
|
8451
8451
|
}
|
|
@@ -8575,13 +8575,16 @@ var init_sandbox = __esm({
|
|
|
8575
8575
|
});
|
|
8576
8576
|
|
|
8577
8577
|
// src/worlds/describe.ts
|
|
8578
|
-
async function describeWorld(slugRef, options = {}) {
|
|
8578
|
+
async function describeWorld(slugRef, options = {}, entity) {
|
|
8579
8579
|
const { slug, ref } = parseSlugRef(slugRef);
|
|
8580
|
-
const query =
|
|
8580
|
+
const query = new URLSearchParams();
|
|
8581
|
+
if (ref !== void 0) query.set("ref", ref);
|
|
8582
|
+
if (entity !== void 0) query.set("entity", entity);
|
|
8583
|
+
const search = query.toString() ? `?${query}` : "";
|
|
8581
8584
|
return apiRequest(
|
|
8582
8585
|
options,
|
|
8583
8586
|
"GET",
|
|
8584
|
-
`/api/public/worlds/${encodeURIComponent(slug)}${
|
|
8587
|
+
`/api/public/worlds/${encodeURIComponent(slug)}${search}`
|
|
8585
8588
|
);
|
|
8586
8589
|
}
|
|
8587
8590
|
var init_describe = __esm({
|
|
@@ -9503,8 +9506,8 @@ runpy.run_module('gatewaysdk.environment.schema',run_name='__main__')`;
|
|
|
9503
9506
|
});
|
|
9504
9507
|
|
|
9505
9508
|
// src/worlds/databaseEngines.ts
|
|
9506
|
-
function isWorldSnapshotOutput(
|
|
9507
|
-
return SNAPSHOT_OUTPUTS.has(
|
|
9509
|
+
function isWorldSnapshotOutput(path14) {
|
|
9510
|
+
return SNAPSHOT_OUTPUTS.has(path14.slice(path14.lastIndexOf("/") + 1));
|
|
9508
9511
|
}
|
|
9509
9512
|
var WORLD_DATABASE_ENGINES, DEFAULT_WORLD_DATABASE_ENGINE, WORLD_SNAPSHOT_FILE_PATH, WORLD_SNAPSHOT_SIDECAR_PATH, WORLD_SNAPSHOT_CONTENT_TYPE, SNAPSHOT_OUTPUTS;
|
|
9510
9513
|
var init_databaseEngines = __esm({
|
|
@@ -10112,9 +10115,83 @@ if (fn) {
|
|
|
10112
10115
|
}
|
|
10113
10116
|
});
|
|
10114
10117
|
|
|
10118
|
+
// src/cli/platform/worldData/manifest.ts
|
|
10119
|
+
import * as fs11 from "fs";
|
|
10120
|
+
import * as path12 from "path";
|
|
10121
|
+
function recordChunk(manifest, chunk) {
|
|
10122
|
+
manifest.chunks = [...manifest.chunks.filter((c) => c.ordinal !== chunk.ordinal), chunk].sort((a, b) => a.ordinal - b.ordinal);
|
|
10123
|
+
}
|
|
10124
|
+
function uploadedEntry(manifest, ordinal, digest) {
|
|
10125
|
+
return manifest.chunks.find((c) => c.ordinal === ordinal && c.digest === digest && c.uploaded) ?? null;
|
|
10126
|
+
}
|
|
10127
|
+
var STEM, MANIFEST_DIR, stamp, sameInputs, ManifestShelf, IMPORT_MANIFESTS;
|
|
10128
|
+
var init_manifest = __esm({
|
|
10129
|
+
"src/cli/platform/worldData/manifest.ts"() {
|
|
10130
|
+
"use strict";
|
|
10131
|
+
init_router();
|
|
10132
|
+
STEM = /^[A-Za-z0-9_-]+$/;
|
|
10133
|
+
MANIFEST_DIR = path12.join(".gateway", "imports");
|
|
10134
|
+
stamp = () => (/* @__PURE__ */ new Date()).toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
10135
|
+
sameInputs = (a, b) => a.length === b.length && a.every((x, i) => x === b[i]);
|
|
10136
|
+
ManifestShelf = class {
|
|
10137
|
+
constructor(dir, stem) {
|
|
10138
|
+
this.dir = dir;
|
|
10139
|
+
this.stem = stem;
|
|
10140
|
+
}
|
|
10141
|
+
dir;
|
|
10142
|
+
stem;
|
|
10143
|
+
dirIn(cwd = process.cwd()) {
|
|
10144
|
+
return path12.join(cwd, this.dir);
|
|
10145
|
+
}
|
|
10146
|
+
file(stem, cwd) {
|
|
10147
|
+
if (!STEM.test(stem)) throw new CliError(`Not a manifest name: ${stem}`);
|
|
10148
|
+
return path12.join(this.dirIn(cwd), `${stem}.json`);
|
|
10149
|
+
}
|
|
10150
|
+
// .gateway/ ignores itself so a manifest, which names hosts and inputs, never enters a repository.
|
|
10151
|
+
write(manifest, cwd = process.cwd()) {
|
|
10152
|
+
const file = this.file(this.stem(manifest), cwd);
|
|
10153
|
+
fs11.mkdirSync(path12.dirname(file), { recursive: true });
|
|
10154
|
+
const ignore = path12.join(cwd, ".gateway", ".gitignore");
|
|
10155
|
+
if (!fs11.existsSync(ignore)) fs11.writeFileSync(ignore, "*\n");
|
|
10156
|
+
manifest.updatedAt = stamp();
|
|
10157
|
+
const tmp = `${file}.tmp`;
|
|
10158
|
+
fs11.writeFileSync(tmp, JSON.stringify(manifest, null, 1) + "\n");
|
|
10159
|
+
fs11.renameSync(tmp, file);
|
|
10160
|
+
return file;
|
|
10161
|
+
}
|
|
10162
|
+
record(manifest, chunk, cwd = process.cwd()) {
|
|
10163
|
+
recordChunk(manifest, chunk);
|
|
10164
|
+
this.write(manifest, cwd);
|
|
10165
|
+
}
|
|
10166
|
+
read(stem, cwd = process.cwd()) {
|
|
10167
|
+
try {
|
|
10168
|
+
const parsed = JSON.parse(fs11.readFileSync(this.file(stem, cwd), "utf8"));
|
|
10169
|
+
return parsed && parsed.version === 1 && this.stem(parsed) === stem ? parsed : null;
|
|
10170
|
+
} catch {
|
|
10171
|
+
return null;
|
|
10172
|
+
}
|
|
10173
|
+
}
|
|
10174
|
+
list(cwd = process.cwd()) {
|
|
10175
|
+
const dir = this.dirIn(cwd);
|
|
10176
|
+
if (!fs11.existsSync(dir)) return [];
|
|
10177
|
+
const out = [];
|
|
10178
|
+
for (const name of fs11.readdirSync(dir)) {
|
|
10179
|
+
const manifest = name.endsWith(".json") ? this.read(name.slice(0, -5), cwd) : null;
|
|
10180
|
+
if (manifest) out.push(manifest);
|
|
10181
|
+
}
|
|
10182
|
+
return out.sort((a, b) => Number(a.updatedAt < b.updatedAt) - Number(a.updatedAt > b.updatedAt));
|
|
10183
|
+
}
|
|
10184
|
+
resumable(slug, inputs, cwd = process.cwd()) {
|
|
10185
|
+
return this.list(cwd).find((m) => !m.completed && m.slug === slug && sameInputs(m.inputs, inputs)) ?? null;
|
|
10186
|
+
}
|
|
10187
|
+
};
|
|
10188
|
+
IMPORT_MANIFESTS = new ManifestShelf(MANIFEST_DIR, (key) => key.batchId);
|
|
10189
|
+
}
|
|
10190
|
+
});
|
|
10191
|
+
|
|
10115
10192
|
// src/cli/platform/worldData/batches.ts
|
|
10116
10193
|
import { createHash as createHash8 } from "crypto";
|
|
10117
|
-
import * as
|
|
10194
|
+
import * as fs12 from "fs";
|
|
10118
10195
|
function isSettled(status) {
|
|
10119
10196
|
if (!TERMINAL_STATES.has(status.state)) return false;
|
|
10120
10197
|
if (status.state !== "applied") return true;
|
|
@@ -10176,12 +10253,14 @@ function phaseLine(status) {
|
|
|
10176
10253
|
const phase = status.progress?.phase;
|
|
10177
10254
|
return phase ? `${status.state} (${phase})` : status.state;
|
|
10178
10255
|
}
|
|
10179
|
-
var TERMINAL_STATES, sleepMs, WorldDataApi;
|
|
10256
|
+
var CHUNK_PARALLEL, TERMINAL_STATES, sleepMs, WorldDataApi, ChunkUploader;
|
|
10180
10257
|
var init_batches = __esm({
|
|
10181
10258
|
"src/cli/platform/worldData/batches.ts"() {
|
|
10182
10259
|
"use strict";
|
|
10183
10260
|
init_config();
|
|
10184
10261
|
init_router();
|
|
10262
|
+
init_manifest();
|
|
10263
|
+
CHUNK_PARALLEL = 4;
|
|
10185
10264
|
TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
10186
10265
|
"applied",
|
|
10187
10266
|
"refused",
|
|
@@ -10208,13 +10287,7 @@ var init_batches = __esm({
|
|
|
10208
10287
|
this.creds,
|
|
10209
10288
|
"POST",
|
|
10210
10289
|
`${this.base}/batches/${batchId}/chunks`,
|
|
10211
|
-
|
|
10212
|
-
ordinal: chunk.ordinal,
|
|
10213
|
-
digest: chunk.digest,
|
|
10214
|
-
bytes: chunk.bytes,
|
|
10215
|
-
rows: chunk.rows,
|
|
10216
|
-
...chunk.entity ? { entity: chunk.entity } : {}
|
|
10217
|
-
}
|
|
10290
|
+
chunk
|
|
10218
10291
|
);
|
|
10219
10292
|
}
|
|
10220
10293
|
async stageDrive(batchId, chunk) {
|
|
@@ -10270,6 +10343,23 @@ var init_batches = __esm({
|
|
|
10270
10343
|
message ? { message } : {}
|
|
10271
10344
|
);
|
|
10272
10345
|
}
|
|
10346
|
+
chunks(batchId) {
|
|
10347
|
+
return new ChunkUploader(
|
|
10348
|
+
{
|
|
10349
|
+
register: (chunk) => this.register(batchId, chunk),
|
|
10350
|
+
uploaded: (chunkId) => this.markUploaded(batchId, chunkId)
|
|
10351
|
+
},
|
|
10352
|
+
this.sleep
|
|
10353
|
+
);
|
|
10354
|
+
}
|
|
10355
|
+
};
|
|
10356
|
+
ChunkUploader = class {
|
|
10357
|
+
constructor(door, sleep4 = sleepMs) {
|
|
10358
|
+
this.door = door;
|
|
10359
|
+
this.sleep = sleep4;
|
|
10360
|
+
}
|
|
10361
|
+
door;
|
|
10362
|
+
sleep;
|
|
10273
10363
|
async put(uploadUrl, gz, signed) {
|
|
10274
10364
|
const headers = signed ?? uploadHeadersFor(gz);
|
|
10275
10365
|
const delays = [500, 1500, 4e3];
|
|
@@ -10296,17 +10386,15 @@ var init_batches = __esm({
|
|
|
10296
10386
|
await this.sleep(delays[attempt]);
|
|
10297
10387
|
}
|
|
10298
10388
|
}
|
|
10299
|
-
async upload(
|
|
10300
|
-
const registered = await this.register(
|
|
10301
|
-
|
|
10302
|
-
|
|
10303
|
-
|
|
10304
|
-
|
|
10305
|
-
|
|
10306
|
-
|
|
10307
|
-
|
|
10308
|
-
await this.markUploaded(batchId, registered.chunkId);
|
|
10309
|
-
}
|
|
10389
|
+
async upload(chunk) {
|
|
10390
|
+
const registered = await this.door.register({
|
|
10391
|
+
ordinal: chunk.ordinal,
|
|
10392
|
+
digest: chunk.digest,
|
|
10393
|
+
bytes: chunk.bytes,
|
|
10394
|
+
rows: chunk.rows,
|
|
10395
|
+
...chunk.entity ? { entity: chunk.entity } : {}
|
|
10396
|
+
});
|
|
10397
|
+
if (!registered.uploaded) await this.send(chunk, registered);
|
|
10310
10398
|
return {
|
|
10311
10399
|
ordinal: chunk.ordinal,
|
|
10312
10400
|
digest: chunk.digest,
|
|
@@ -10317,6 +10405,28 @@ var init_batches = __esm({
|
|
|
10317
10405
|
uploaded: true
|
|
10318
10406
|
};
|
|
10319
10407
|
}
|
|
10408
|
+
async send(chunk, registered) {
|
|
10409
|
+
if (!registered.uploadUrl)
|
|
10410
|
+
throw new CliError(
|
|
10411
|
+
`the server registered chunk ${chunk.ordinal} without an upload URL`,
|
|
10412
|
+
1
|
|
10413
|
+
);
|
|
10414
|
+
await this.put(registered.uploadUrl, chunk.gz, registered.uploadHeaders);
|
|
10415
|
+
await this.door.uploaded(registered.chunkId);
|
|
10416
|
+
}
|
|
10417
|
+
async uploadAll(chunks, kept, parallel, progress) {
|
|
10418
|
+
await forEachLimited(chunks, parallel, async (chunk) => {
|
|
10419
|
+
if (uploadedEntry(kept.manifest, chunk.ordinal, chunk.digest)) {
|
|
10420
|
+
progress(`chunk ${chunk.ordinal}: already uploaded (${chunk.rows} rows)`);
|
|
10421
|
+
return;
|
|
10422
|
+
}
|
|
10423
|
+
const entry = await this.upload(chunk);
|
|
10424
|
+
kept.shelf.record(kept.manifest, entry);
|
|
10425
|
+
progress(
|
|
10426
|
+
`chunk ${chunk.ordinal}: uploaded ${chunk.rows} rows, ${chunk.bytes} bytes (${entry.chunkId})`
|
|
10427
|
+
);
|
|
10428
|
+
});
|
|
10429
|
+
}
|
|
10320
10430
|
};
|
|
10321
10431
|
}
|
|
10322
10432
|
});
|
|
@@ -10452,11 +10562,12 @@ async function pushRows(creds, slug, rows, opts = {}) {
|
|
|
10452
10562
|
chunkRows: null,
|
|
10453
10563
|
entity: null
|
|
10454
10564
|
});
|
|
10565
|
+
const uploader = client.chunks(opened.batchId);
|
|
10455
10566
|
await forEachLimited(
|
|
10456
10567
|
chunks,
|
|
10457
|
-
Math.max(1, opts.parallel ??
|
|
10568
|
+
Math.max(1, opts.parallel ?? CHUNK_PARALLEL),
|
|
10458
10569
|
async (chunk) => {
|
|
10459
|
-
await
|
|
10570
|
+
await uploader.upload(chunk);
|
|
10460
10571
|
}
|
|
10461
10572
|
);
|
|
10462
10573
|
await client.complete(opened.batchId);
|
|
@@ -10474,9 +10585,9 @@ var init_push = __esm({
|
|
|
10474
10585
|
});
|
|
10475
10586
|
|
|
10476
10587
|
// src/worlds/ingest.ts
|
|
10477
|
-
import * as
|
|
10588
|
+
import * as fs13 from "fs";
|
|
10478
10589
|
import * as os3 from "os";
|
|
10479
|
-
import * as
|
|
10590
|
+
import * as path13 from "path";
|
|
10480
10591
|
function parseToolCallRecord(raw) {
|
|
10481
10592
|
const none = { record: null, skip: null };
|
|
10482
10593
|
if (!isObject2(raw)) return { ...none, refusal: "record is not an object" };
|
|
@@ -10497,7 +10608,7 @@ function parseToolCallRecord(raw) {
|
|
|
10497
10608
|
}
|
|
10498
10609
|
function readToolCallRecords(paths) {
|
|
10499
10610
|
const records = [];
|
|
10500
|
-
for (const file of paths) records.push(...toolCallRecordsIn(
|
|
10611
|
+
for (const file of paths) records.push(...toolCallRecordsIn(fs13.readFileSync(file, "utf8"), file));
|
|
10501
10612
|
return records;
|
|
10502
10613
|
}
|
|
10503
10614
|
function toolCallRecordsIn(text2, where) {
|
|
@@ -10533,7 +10644,7 @@ async function emit(sink, record) {
|
|
|
10533
10644
|
return;
|
|
10534
10645
|
}
|
|
10535
10646
|
if (typeof sink === "string") {
|
|
10536
|
-
|
|
10647
|
+
fs13.appendFileSync(sink, JSON.stringify(record) + "\n");
|
|
10537
10648
|
return;
|
|
10538
10649
|
}
|
|
10539
10650
|
if (typeof sink === "function") {
|
|
@@ -10615,7 +10726,7 @@ async function ingest(world, records, options = {}) {
|
|
|
10615
10726
|
const redact = options.redact ?? "off";
|
|
10616
10727
|
const mode = options.mode ?? "append";
|
|
10617
10728
|
const dryRun = Boolean(options.dryRun);
|
|
10618
|
-
const local =
|
|
10729
|
+
const local = fs13.existsSync(world) && fs13.statSync(world).isDirectory();
|
|
10619
10730
|
const creds = local ? await runtimeEndpoint(options) : platformEndpoint(options);
|
|
10620
10731
|
const map = options.map === void 0 ? void 0 : options.map;
|
|
10621
10732
|
let rows = null;
|
|
@@ -10632,9 +10743,9 @@ async function ingest(world, records, options = {}) {
|
|
|
10632
10743
|
if (!creds) throw new Error(GATEWAY_KEYS_MISSING);
|
|
10633
10744
|
const { pull: pull2 } = await Promise.resolve().then(() => (init_hub(), hub_exports));
|
|
10634
10745
|
const slug = world.split("@", 1)[0];
|
|
10635
|
-
const scratch =
|
|
10746
|
+
const scratch = fs13.mkdtempSync(path13.join(os3.tmpdir(), "gateway-ingest-"));
|
|
10636
10747
|
try {
|
|
10637
|
-
const pulled = await pull2(world,
|
|
10748
|
+
const pulled = await pull2(world, path13.join(scratch, slug), options);
|
|
10638
10749
|
let report;
|
|
10639
10750
|
let projected;
|
|
10640
10751
|
if (rows === null) {
|
|
@@ -10655,7 +10766,7 @@ async function ingest(world, records, options = {}) {
|
|
|
10655
10766
|
if (status.entityCounts) report.entityCounts = status.entityCounts;
|
|
10656
10767
|
return report;
|
|
10657
10768
|
} finally {
|
|
10658
|
-
|
|
10769
|
+
fs13.rmSync(scratch, { recursive: true, force: true });
|
|
10659
10770
|
}
|
|
10660
10771
|
}
|
|
10661
10772
|
async function ingestToolCalls(world, records, options = {}) {
|
|
@@ -11120,8 +11231,8 @@ __export(worlds_exports, {
|
|
|
11120
11231
|
worldRequestsUnrecorded: () => worldRequestsUnrecorded,
|
|
11121
11232
|
writeLocalMeta: () => writeLocalMeta
|
|
11122
11233
|
});
|
|
11123
|
-
async function apiRequest(options, method,
|
|
11124
|
-
return new PlatformTransport(requirePlatformEndpoint(options)).json(method,
|
|
11234
|
+
async function apiRequest(options, method, path14, body, init2 = {}) {
|
|
11235
|
+
return new PlatformTransport(requirePlatformEndpoint(options)).json(method, path14, body, init2);
|
|
11125
11236
|
}
|
|
11126
11237
|
async function setEnvSecret(slug, key, value, options = {}) {
|
|
11127
11238
|
return apiRequest(
|