@withgateway/sdk 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +4304 -2381
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4266 -2343
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +56 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +52 -27
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +280 -8
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +272 -8
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
- package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
- package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
- package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1100 -438
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1079 -424
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +45 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +41 -16
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +98 -61
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +98 -61
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +179 -25
- package/dist/worlds/index.d.ts +179 -25
- package/dist/worlds/index.js +639 -284
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +631 -276
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +2 -1
- package/runtime/gatewaysdk/environment/_bundle.py +15 -25
- package/runtime/gatewaysdk/environment/_materialize.py +53 -9
- package/runtime/gatewaysdk/environment/_toml.py +13 -0
- package/runtime/gatewaysdk/environment/schema/api.py +40 -25
- package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
- package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +156 -546
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/platform.py +30 -26
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +67 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
- package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
- package/runtime/gatewaysdk/environment/schema/tools_world.py +71 -23
- package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +507 -0
- package/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/skills/connector-schema-authoring/connector-worlds.md +8 -1
- package/skills/connector-schema-authoring/world-schema.md +1 -1
- package/skills/hooks-getting-started/SKILL.md +67 -6
- package/skills/world-data-ingestion/SKILL.md +4 -1
- package/skills/worlds-getting-started/SKILL.md +11 -11
- package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
package/dist/worlds/index.js
CHANGED
|
@@ -41,9 +41,9 @@ function surfaceKinds(s) {
|
|
|
41
41
|
if (s.browser) kinds.push("browser");
|
|
42
42
|
return kinds;
|
|
43
43
|
}
|
|
44
|
-
function uiUrl(ui,
|
|
44
|
+
function uiUrl(ui, path10) {
|
|
45
45
|
const base = ui.url.replace(/\/+$/, "");
|
|
46
|
-
const rel =
|
|
46
|
+
const rel = path10.startsWith("/") ? path10 : `/${path10}`;
|
|
47
47
|
if (!ui.token) return base + rel;
|
|
48
48
|
const [beforeHash, hash] = splitHash(rel);
|
|
49
49
|
const joiner = beforeHash.includes("?") ? "&" : "?";
|
|
@@ -600,12 +600,13 @@ async function openSession(slugOrRef, taskOrOpts, positionalOpts = {}) {
|
|
|
600
600
|
);
|
|
601
601
|
const at = slugOrRef.indexOf("@");
|
|
602
602
|
const slug = at === -1 ? slugOrRef : slugOrRef.slice(0, at);
|
|
603
|
+
const ref = at === -1 ? "" : slugOrRef.slice(at + 1);
|
|
603
604
|
const state2 = await apiRequest(
|
|
604
605
|
options,
|
|
605
606
|
"POST",
|
|
606
607
|
"/api/public/world-sessions",
|
|
607
608
|
{
|
|
608
|
-
...opts.versionId ? { versionId: opts.versionId } : { slug },
|
|
609
|
+
...opts.versionId ? { versionId: opts.versionId } : { slug, ...ref && { ref } },
|
|
609
610
|
task,
|
|
610
611
|
...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm },
|
|
611
612
|
...opts.surfaces?.length ? { surfaces: opts.surfaces } : {}
|
|
@@ -619,8 +620,8 @@ async function runSessions(opts) {
|
|
|
619
620
|
let tasks = opts.tasks;
|
|
620
621
|
let versionId = opts.versionId;
|
|
621
622
|
if (!tasks || !versionId) {
|
|
622
|
-
const { openWorld: openWorld2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
|
|
623
|
-
const world = await openWorld2(opts.world, options);
|
|
623
|
+
const { openWorld: openWorld2, openWorldVersion: openWorldVersion2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
|
|
624
|
+
const world = versionId ? await openWorldVersion2(opts.world.split("@", 1)[0], versionId, options) : await openWorld2(opts.world, options);
|
|
624
625
|
tasks = tasks ?? world.tasks.map((t) => t.name);
|
|
625
626
|
versionId = versionId ?? world.version.versionId;
|
|
626
627
|
}
|
|
@@ -676,10 +677,12 @@ async function runSessions(opts) {
|
|
|
676
677
|
})
|
|
677
678
|
);
|
|
678
679
|
const mean = results.reduce((sum, r) => sum + (r.reward ?? 0), 0) / (results.length || 1);
|
|
680
|
+
const failed = results.filter((r) => r.error).length;
|
|
679
681
|
return {
|
|
680
682
|
results: results.sort((a, b) => a.task.localeCompare(b.task)),
|
|
681
683
|
mean,
|
|
682
|
-
|
|
684
|
+
failed,
|
|
685
|
+
passed: failed === 0 && (opts.failUnder === void 0 || mean >= opts.failUnder)
|
|
683
686
|
};
|
|
684
687
|
}
|
|
685
688
|
var import_node_crypto, isTaskIdRef, isTaskSpec, taskLabel, READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, JUDGED_GRADE_TIMEOUT_MS, JUDGED_GRADE_POLL_MS, EXPORT_PAGE, sleep, modelToolName, WorldSessionTimeout, WorldSessionCallError, WorldSession;
|
|
@@ -804,18 +807,18 @@ var init_session = __esm({
|
|
|
804
807
|
}
|
|
805
808
|
/** The reporter `browser()` uses; exposed for a page you drive yourself. */
|
|
806
809
|
browserRecorder(agentId) {
|
|
807
|
-
const
|
|
810
|
+
const path10 = `/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/browser-events`;
|
|
808
811
|
const id = agentId ?? process.env.GATEWAY_AGENT_ID ?? `agent-${(0, import_node_crypto.randomUUID)().replace(/-/g, "").slice(0, 8)}`;
|
|
809
812
|
return new BrowserActionRecorder(
|
|
810
813
|
async (events) => {
|
|
811
|
-
await apiRequest(this.options, "POST",
|
|
814
|
+
await apiRequest(this.options, "POST", path10, { events });
|
|
812
815
|
},
|
|
813
816
|
id,
|
|
814
817
|
{
|
|
815
818
|
// Continue this agent's timeline where the platform has it, so the
|
|
816
819
|
// same id reopened later never re-numbers from zero.
|
|
817
820
|
resumeFrom: async () => {
|
|
818
|
-
const summary = await apiRequest(this.options, "GET",
|
|
821
|
+
const summary = await apiRequest(this.options, "GET", path10);
|
|
819
822
|
return summary.agents.find((a) => a.agentId === id)?.nextSeq ?? 0;
|
|
820
823
|
}
|
|
821
824
|
}
|
|
@@ -1621,10 +1624,10 @@ var init_users = __esm({
|
|
|
1621
1624
|
totalItems: Number(meta.totalItems ?? rows.length)
|
|
1622
1625
|
};
|
|
1623
1626
|
}
|
|
1624
|
-
async requestJson(method,
|
|
1627
|
+
async requestJson(method, path10, payload) {
|
|
1625
1628
|
let response;
|
|
1626
1629
|
try {
|
|
1627
|
-
response = await fetch(`${this.host}${
|
|
1630
|
+
response = await fetch(`${this.host}${path10}`, {
|
|
1628
1631
|
method,
|
|
1629
1632
|
headers: {
|
|
1630
1633
|
Authorization: this.authHeader(),
|
|
@@ -1635,17 +1638,17 @@ var init_users = __esm({
|
|
|
1635
1638
|
});
|
|
1636
1639
|
} catch (error) {
|
|
1637
1640
|
throw new UserProfileError(
|
|
1638
|
-
`${method} ${
|
|
1641
|
+
`${method} ${path10} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1639
1642
|
{ cause: error }
|
|
1640
1643
|
);
|
|
1641
1644
|
}
|
|
1642
1645
|
const text2 = await response.text();
|
|
1643
1646
|
if (!response.ok) {
|
|
1644
1647
|
throw new UserProfileError(
|
|
1645
|
-
`${method} ${
|
|
1648
|
+
`${method} ${path10} failed (${response.status}): ${text2.slice(0, 500)}`
|
|
1646
1649
|
);
|
|
1647
1650
|
}
|
|
1648
|
-
this.log("%s %s -> %s", method,
|
|
1651
|
+
this.log("%s %s -> %s", method, path10, response.status);
|
|
1649
1652
|
return safeJson(text2);
|
|
1650
1653
|
}
|
|
1651
1654
|
authHeader() {
|
|
@@ -1909,10 +1912,10 @@ var init_scores = __esm({
|
|
|
1909
1912
|
metadata: signalMetadata
|
|
1910
1913
|
});
|
|
1911
1914
|
}
|
|
1912
|
-
async request(method,
|
|
1915
|
+
async request(method, path10, payload) {
|
|
1913
1916
|
let response;
|
|
1914
1917
|
try {
|
|
1915
|
-
response = await fetch(`${this.host}${
|
|
1918
|
+
response = await fetch(`${this.host}${path10}`, {
|
|
1916
1919
|
method,
|
|
1917
1920
|
headers: {
|
|
1918
1921
|
Authorization: this.authHeader(),
|
|
@@ -1923,17 +1926,17 @@ var init_scores = __esm({
|
|
|
1923
1926
|
});
|
|
1924
1927
|
} catch (error) {
|
|
1925
1928
|
throw new ScoreError(
|
|
1926
|
-
`${method} ${
|
|
1929
|
+
`${method} ${path10} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1927
1930
|
{ cause: error }
|
|
1928
1931
|
);
|
|
1929
1932
|
}
|
|
1930
1933
|
const text2 = await response.text();
|
|
1931
1934
|
if (!response.ok) {
|
|
1932
1935
|
throw new ScoreError(
|
|
1933
|
-
`${method} ${
|
|
1936
|
+
`${method} ${path10} failed (${response.status}): ${text2.slice(0, 500)}`
|
|
1934
1937
|
);
|
|
1935
1938
|
}
|
|
1936
|
-
this.log("%s %s -> %s", method,
|
|
1939
|
+
this.log("%s %s -> %s", method, path10, response.status);
|
|
1937
1940
|
}
|
|
1938
1941
|
authHeader() {
|
|
1939
1942
|
return `Basic ${Buffer.from(`${this.publicKey}:${this.secretKey}`).toString(
|
|
@@ -1961,7 +1964,7 @@ var require_package = __commonJS({
|
|
|
1961
1964
|
"package.json"(exports2, module2) {
|
|
1962
1965
|
module2.exports = {
|
|
1963
1966
|
name: "@withgateway/sdk",
|
|
1964
|
-
version: "0.2.
|
|
1967
|
+
version: "0.2.6",
|
|
1965
1968
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
1966
1969
|
main: "./dist/index.js",
|
|
1967
1970
|
module: "./dist/index.mjs",
|
|
@@ -2065,6 +2068,7 @@ var require_package = __commonJS({
|
|
|
2065
2068
|
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
2066
2069
|
test: "vitest",
|
|
2067
2070
|
"test:run": "vitest run",
|
|
2071
|
+
"test:integration": "vitest run --config vitest.integration.config.ts",
|
|
2068
2072
|
lint: "eslint src --ext .ts",
|
|
2069
2073
|
clean: "rm -rf dist",
|
|
2070
2074
|
example: "tsx examples/test-ab-experiment.ts",
|
|
@@ -2693,7 +2697,7 @@ async function fileTaskRun(live, runId, model, report, grade, agentError, traced
|
|
|
2693
2697
|
task: live.name ?? live.taskId ?? runId,
|
|
2694
2698
|
rolloutIndex: 0,
|
|
2695
2699
|
prompt: live.instruction,
|
|
2696
|
-
completion: report,
|
|
2700
|
+
...report === null ? {} : { completion: report },
|
|
2697
2701
|
rewards: grade.rewards,
|
|
2698
2702
|
info: {
|
|
2699
2703
|
sessionId: runId,
|
|
@@ -2731,17 +2735,20 @@ async function fileTaskRun(live, runId, model, report, grade, agentError, traced
|
|
|
2731
2735
|
return evaluationId;
|
|
2732
2736
|
}
|
|
2733
2737
|
function reportFromResult(result) {
|
|
2738
|
+
if (result === void 0 || result === null)
|
|
2739
|
+
return { report: null, extra: {} };
|
|
2734
2740
|
if (typeof result === "string") return { report: result, extra: {} };
|
|
2735
|
-
if (
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2741
|
+
if (typeof result !== "object" || Array.isArray(result))
|
|
2742
|
+
throw new Error(
|
|
2743
|
+
`agent returned ${describeValue(result)}; ${RESULT_CONTRACT}`
|
|
2744
|
+
);
|
|
2745
|
+
const { report, ...extra } = result;
|
|
2746
|
+
if (report === void 0 || report === null) return { report: null, extra };
|
|
2747
|
+
if (typeof report !== "string")
|
|
2748
|
+
throw new Error(
|
|
2749
|
+
`agent returned a report of type ${describeValue(report)}; ${RESULT_CONTRACT}`
|
|
2750
|
+
);
|
|
2751
|
+
return { report, extra };
|
|
2745
2752
|
}
|
|
2746
2753
|
async function runTask(ref, agent, opts) {
|
|
2747
2754
|
const runId = `wtr-${(0, import_node_crypto3.randomBytes)(6).toString("hex")}`;
|
|
@@ -2845,7 +2852,7 @@ async function runTask(ref, agent, opts) {
|
|
|
2845
2852
|
}
|
|
2846
2853
|
}
|
|
2847
2854
|
}
|
|
2848
|
-
var import_node_crypto3, fs, path, TaskRun;
|
|
2855
|
+
var import_node_crypto3, fs, path, TaskRun, describeValue, RESULT_CONTRACT;
|
|
2849
2856
|
var init_taskRun = __esm({
|
|
2850
2857
|
"src/worlds/taskRun.ts"() {
|
|
2851
2858
|
"use strict";
|
|
@@ -2919,6 +2926,154 @@ var init_taskRun = __esm({
|
|
|
2919
2926
|
Object.assign(this.notes, notes);
|
|
2920
2927
|
}
|
|
2921
2928
|
};
|
|
2929
|
+
describeValue = (value) => Array.isArray(value) ? "array" : value === null ? "null" : typeof value;
|
|
2930
|
+
RESULT_CONTRACT = "return a string report, an object {report?: string, ...} or nothing";
|
|
2931
|
+
}
|
|
2932
|
+
});
|
|
2933
|
+
|
|
2934
|
+
// src/worlds/worldDir.ts
|
|
2935
|
+
var import_node_crypto4, fs2, path2, WORLD_MANIFEST, API_BUNDLE_MANIFEST, BUNDLED_WORLD_PREFIX, CONNECTOR_SPEC, PYPROJECT, WORLD_SCHEMA_MARKERS, WORLD_ROOT_MARKERS, BROAD_ROOT_MARKERS, SKIPPED_DIRS, V1_DEP, V1_IMPORT, V1_TASKSET, HARBOR_TASK, skippedDir, byCodePoint, WorldDir;
|
|
2936
|
+
var init_worldDir = __esm({
|
|
2937
|
+
"src/worlds/worldDir.ts"() {
|
|
2938
|
+
"use strict";
|
|
2939
|
+
import_node_crypto4 = require("crypto");
|
|
2940
|
+
fs2 = __toESM(require("fs"));
|
|
2941
|
+
path2 = __toESM(require("path"));
|
|
2942
|
+
WORLD_MANIFEST = "gateway-env.toml";
|
|
2943
|
+
API_BUNDLE_MANIFEST = "gateway-api-world.toml";
|
|
2944
|
+
BUNDLED_WORLD_PREFIX = "server/world/";
|
|
2945
|
+
CONNECTOR_SPEC = "connector.toml";
|
|
2946
|
+
PYPROJECT = "pyproject.toml";
|
|
2947
|
+
WORLD_SCHEMA_MARKERS = ["schema/world.json", "schema/runtime.json", "schema/lock.json"];
|
|
2948
|
+
WORLD_ROOT_MARKERS = [CONNECTOR_SPEC, ...WORLD_SCHEMA_MARKERS];
|
|
2949
|
+
BROAD_ROOT_MARKERS = [WORLD_MANIFEST, API_BUNDLE_MANIFEST];
|
|
2950
|
+
SKIPPED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
2951
|
+
V1_DEP = /["']verifiers\s*(?:\[[^\]]*\])?\s*(?:>=|~=|==|>)\s*(?:0\.(?:[2-9]\d*|[1-9]\d+)|[1-9]\d*)/;
|
|
2952
|
+
V1_IMPORT = /^\s*(?:from\s+verifiers\.v1\s+import\b|from\s+verifiers\s+import\s+v1\b|import\s+verifiers\.v1\b)/m;
|
|
2953
|
+
V1_TASKSET = /class\s+\w+\s*\(\s*(?:(?:vf|v1|verifiers(?:\.v1)?)\.)?Taskset\s*[,)]/;
|
|
2954
|
+
HARBOR_TASK = /^(?:tasks\/)?([^/]+)\/task\.toml$/;
|
|
2955
|
+
skippedDir = (name) => name.startsWith(".") || SKIPPED_DIRS.has(name) || name.endsWith(".egg-info");
|
|
2956
|
+
byCodePoint = (a, b) => Buffer.compare(Buffer.from(a, "utf-8"), Buffer.from(b, "utf-8"));
|
|
2957
|
+
WorldDir = class _WorldDir {
|
|
2958
|
+
constructor(dir, worldRoot) {
|
|
2959
|
+
this.dir = dir;
|
|
2960
|
+
this.worldRoot = worldRoot;
|
|
2961
|
+
}
|
|
2962
|
+
pathsMemo;
|
|
2963
|
+
static open(dir) {
|
|
2964
|
+
const present = (rel) => fs2.existsSync(path2.join(dir, rel));
|
|
2965
|
+
const bundled = present(API_BUNDLE_MANIFEST) && WORLD_ROOT_MARKERS.some((m) => present(BUNDLED_WORLD_PREFIX + m));
|
|
2966
|
+
return new _WorldDir(dir, bundled ? BUNDLED_WORLD_PREFIX : "");
|
|
2967
|
+
}
|
|
2968
|
+
// ── Files ──────────────────────────────────────────────────────────────
|
|
2969
|
+
has(rel) {
|
|
2970
|
+
return fs2.existsSync(path2.join(this.dir, rel));
|
|
2971
|
+
}
|
|
2972
|
+
text(rel) {
|
|
2973
|
+
try {
|
|
2974
|
+
return fs2.readFileSync(path2.join(this.dir, rel), "utf-8");
|
|
2975
|
+
} catch {
|
|
2976
|
+
return null;
|
|
2977
|
+
}
|
|
2978
|
+
}
|
|
2979
|
+
/** A path under the authored world's root. */
|
|
2980
|
+
worldPath(rel) {
|
|
2981
|
+
return this.worldRoot + rel;
|
|
2982
|
+
}
|
|
2983
|
+
/** Every regular file the archive rules keep (`_archive.py`: no hidden,
|
|
2984
|
+
* dist, build, outputs, pycache or egg-info directories), as sorted
|
|
2985
|
+
* relpaths. Root files enter only when a Gateway manifest widens the root
|
|
2986
|
+
* or they are README.md, pyproject.toml or Python. */
|
|
2987
|
+
get archivePaths() {
|
|
2988
|
+
if (this.pathsMemo) return this.pathsMemo;
|
|
2989
|
+
const broad = BROAD_ROOT_MARKERS.some((m) => this.has(m));
|
|
2990
|
+
const out = [];
|
|
2991
|
+
const walk = (rel) => {
|
|
2992
|
+
for (const entry of fs2.readdirSync(rel ? path2.join(this.dir, rel) : this.dir, { withFileTypes: true })) {
|
|
2993
|
+
const child = rel ? `${rel}/${entry.name}` : entry.name;
|
|
2994
|
+
if (entry.isDirectory()) {
|
|
2995
|
+
if (!skippedDir(entry.name)) walk(child);
|
|
2996
|
+
} else if (entry.isFile() && !entry.name.startsWith(".")) {
|
|
2997
|
+
const rootKept = broad || ["README.md", PYPROJECT].includes(entry.name) || entry.name.endsWith(".py");
|
|
2998
|
+
if (rel || rootKept) out.push(child);
|
|
2999
|
+
}
|
|
3000
|
+
}
|
|
3001
|
+
};
|
|
3002
|
+
walk("");
|
|
3003
|
+
return this.pathsMemo = out.sort(byCodePoint);
|
|
3004
|
+
}
|
|
3005
|
+
/** A root `.gitignore` is present: the Python SDK honours it when
|
|
3006
|
+
* collecting, this package does not reimplement gitignore. */
|
|
3007
|
+
get hasRootGitignore() {
|
|
3008
|
+
return this.has(".gitignore");
|
|
3009
|
+
}
|
|
3010
|
+
/** SHA-256 over `utf8("file:" + relpath)` + bytes per archive path in
|
|
3011
|
+
* sorted order: the version's identity, byte-compatible with
|
|
3012
|
+
* `_archive.compute_content_hash` and Prime Intellect. */
|
|
3013
|
+
get contentHash() {
|
|
3014
|
+
const hasher = (0, import_node_crypto4.createHash)("sha256");
|
|
3015
|
+
for (const rel of this.archivePaths) {
|
|
3016
|
+
hasher.update(Buffer.from(`file:${rel}`, "utf-8"));
|
|
3017
|
+
hasher.update(fs2.readFileSync(path2.join(this.dir, rel)));
|
|
3018
|
+
}
|
|
3019
|
+
return hasher.digest("hex");
|
|
3020
|
+
}
|
|
3021
|
+
// ── Markers ────────────────────────────────────────────────────────────
|
|
3022
|
+
get isApiBundle() {
|
|
3023
|
+
return this.has(API_BUNDLE_MANIFEST);
|
|
3024
|
+
}
|
|
3025
|
+
get hasManifest() {
|
|
3026
|
+
return this.has(this.worldPath(WORLD_MANIFEST));
|
|
3027
|
+
}
|
|
3028
|
+
get hasSchemaMarker() {
|
|
3029
|
+
return WORLD_SCHEMA_MARKERS.some((m) => this.has(this.worldPath(m)));
|
|
3030
|
+
}
|
|
3031
|
+
/** Harbor task dirs, relative to the tree: `tasks/<name>` when a `tasks/`
|
|
3032
|
+
* dir holds them, else the root `<name>` dirs. */
|
|
3033
|
+
taskDirs() {
|
|
3034
|
+
const names = /* @__PURE__ */ new Set();
|
|
3035
|
+
for (const rel of this.archivePaths) {
|
|
3036
|
+
const m = HARBOR_TASK.exec(rel);
|
|
3037
|
+
if (m) names.add(rel.startsWith("tasks/") ? `tasks/${m[1]}` : m[1]);
|
|
3038
|
+
}
|
|
3039
|
+
const under = [...names].filter((n) => n.startsWith("tasks/"));
|
|
3040
|
+
return (under.length ? under : [...names]).sort();
|
|
3041
|
+
}
|
|
3042
|
+
/** The Python package of a verifiers tree (`<pkg>/__init__.py` or
|
|
3043
|
+
* `src/<pkg>/__init__.py`), or null. */
|
|
3044
|
+
pythonPackage() {
|
|
3045
|
+
if (!this.has(PYPROJECT)) return null;
|
|
3046
|
+
for (const base of ["", "src/"]) {
|
|
3047
|
+
const init2 = this.archivePaths.find((rel) => {
|
|
3048
|
+
const m = /^([^/]+)\/__init__\.py$/.exec(rel.startsWith(base) ? rel.slice(base.length) : "");
|
|
3049
|
+
return m !== null && m[1] !== "tests";
|
|
3050
|
+
});
|
|
3051
|
+
if (init2) return { name: init2.slice(base.length).split("/")[0], init: init2 };
|
|
3052
|
+
}
|
|
3053
|
+
return null;
|
|
3054
|
+
}
|
|
3055
|
+
/** The package targets the `verifiers.v1` API: read from the root
|
|
3056
|
+
* pyproject and every archived Python module. */
|
|
3057
|
+
get isVerifiersV1() {
|
|
3058
|
+
for (const rel of this.archivePaths) {
|
|
3059
|
+
if (rel !== PYPROJECT && !rel.endsWith(".py")) continue;
|
|
3060
|
+
const text2 = this.text(rel);
|
|
3061
|
+
if (text2 === null) continue;
|
|
3062
|
+
if (rel === PYPROJECT ? V1_DEP.test(text2) : V1_IMPORT.test(text2) || text2.includes("__all__") && V1_TASKSET.test(text2))
|
|
3063
|
+
return true;
|
|
3064
|
+
}
|
|
3065
|
+
return false;
|
|
3066
|
+
}
|
|
3067
|
+
// ── The answer ─────────────────────────────────────────────────────────
|
|
3068
|
+
/** The runner's kind (`detect_env_kind`): api / mock-tools win over
|
|
3069
|
+
* everything, a harbor task tree next, then the verifiers API version. */
|
|
3070
|
+
get kind() {
|
|
3071
|
+
if (this.isApiBundle) return "api";
|
|
3072
|
+
if (this.has(WORLD_MANIFEST)) return "mock-tools";
|
|
3073
|
+
if (this.taskDirs().length) return "harbor";
|
|
3074
|
+
return this.isVerifiersV1 ? "verifiers-v1" : "verifiers";
|
|
3075
|
+
}
|
|
3076
|
+
};
|
|
2922
3077
|
}
|
|
2923
3078
|
});
|
|
2924
3079
|
|
|
@@ -2926,75 +3081,50 @@ var init_taskRun = __esm({
|
|
|
2926
3081
|
var hub_exports = {};
|
|
2927
3082
|
__export(hub_exports, {
|
|
2928
3083
|
DEFAULT_BRANCH: () => DEFAULT_BRANCH,
|
|
3084
|
+
PushSlugConflict: () => PushSlugConflict,
|
|
3085
|
+
archiveMode: () => archiveMode,
|
|
3086
|
+
buildFileManifest: () => buildFileManifest,
|
|
2929
3087
|
buildSourceTarball: () => buildSourceTarball,
|
|
2930
3088
|
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
2931
3089
|
computeContentHash: () => computeContentHash,
|
|
2932
3090
|
createRef: () => createRef,
|
|
2933
|
-
|
|
3091
|
+
extractBundle: () => extractBundle,
|
|
2934
3092
|
localTreeState: () => localTreeState,
|
|
2935
3093
|
propose: () => propose,
|
|
2936
3094
|
pull: () => pull,
|
|
2937
3095
|
push: () => push,
|
|
2938
3096
|
pushSlug: () => pushSlug,
|
|
2939
3097
|
readLocalMeta: () => readLocalMeta,
|
|
3098
|
+
resolvePushSlug: () => resolvePushSlug,
|
|
2940
3099
|
snapshotTreeHashes: () => snapshotTreeHashes,
|
|
2941
3100
|
writeLocalMeta: () => writeLocalMeta
|
|
2942
3101
|
});
|
|
2943
|
-
function isExcludedDirComponent(component) {
|
|
2944
|
-
return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
|
|
2945
|
-
}
|
|
2946
|
-
function isIncludedInContentHash(relPath, includeAllRootFiles) {
|
|
2947
|
-
const parts = relPath.split("/");
|
|
2948
|
-
const name = parts[parts.length - 1];
|
|
2949
|
-
const dirParts = parts.slice(0, -1);
|
|
2950
|
-
if (name.startsWith(".")) return false;
|
|
2951
|
-
if (parts.includes("__pycache__")) return false;
|
|
2952
|
-
if (dirParts.length === 0) {
|
|
2953
|
-
return includeAllRootFiles || name === "README.md" || name === "pyproject.toml" || BROAD_ROOT_MARKERS.includes(name) || name.endsWith(".py");
|
|
2954
|
-
}
|
|
2955
|
-
return !dirParts.some(isExcludedDirComponent);
|
|
2956
|
-
}
|
|
2957
3102
|
function collectArchiveRelPaths(dir) {
|
|
2958
|
-
|
|
3103
|
+
const world = WorldDir.open(dir);
|
|
3104
|
+
if (world.hasRootGitignore) {
|
|
2959
3105
|
throw new Error(
|
|
2960
3106
|
"This bundle has a root .gitignore, which the Python CLI honors via gitignore_parser. Reimplementing that imperfectly here would silently produce a different content hash, so the TS CLI refuses. Remove the .gitignore (or bake its effect into the tree) and retry."
|
|
2961
3107
|
);
|
|
2962
3108
|
}
|
|
2963
|
-
|
|
2964
|
-
const includeAllRootFiles = BROAD_ROOT_MARKERS.some(
|
|
2965
|
-
(marker) => fs2.existsSync(path2.join(dir, marker))
|
|
2966
|
-
);
|
|
2967
|
-
const walk = (rel) => {
|
|
2968
|
-
const abs = rel ? path2.join(dir, rel) : dir;
|
|
2969
|
-
for (const entry of fs2.readdirSync(abs, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
2970
|
-
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
2971
|
-
if (entry.isDirectory()) {
|
|
2972
|
-
if (!isExcludedDirComponent(entry.name)) walk(childRel);
|
|
2973
|
-
} else if (entry.isFile()) {
|
|
2974
|
-
if (isIncludedInContentHash(childRel, includeAllRootFiles)) {
|
|
2975
|
-
out.push(childRel);
|
|
2976
|
-
}
|
|
2977
|
-
}
|
|
2978
|
-
}
|
|
2979
|
-
};
|
|
2980
|
-
walk("");
|
|
2981
|
-
return out.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
3109
|
+
return world.archivePaths;
|
|
2982
3110
|
}
|
|
2983
3111
|
function computeContentHash(dir) {
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
hasher.update(Buffer.from(`file:${relPath}`, "utf-8"));
|
|
2987
|
-
hasher.update(fs2.readFileSync(path2.join(dir, relPath)));
|
|
2988
|
-
}
|
|
2989
|
-
return hasher.digest("hex");
|
|
3112
|
+
collectArchiveRelPaths(dir);
|
|
3113
|
+
return WorldDir.open(dir).contentHash;
|
|
2990
3114
|
}
|
|
2991
3115
|
function octal(value, length) {
|
|
2992
|
-
return Buffer.from(
|
|
3116
|
+
return Buffer.from(
|
|
3117
|
+
value.toString(8).padStart(length - 1, "0") + "\0",
|
|
3118
|
+
"ascii"
|
|
3119
|
+
);
|
|
3120
|
+
}
|
|
3121
|
+
function archiveMode(mode) {
|
|
3122
|
+
return (mode & 73) !== 0 ? EXEC_MODE : FILE_MODE;
|
|
2993
3123
|
}
|
|
2994
|
-
function tarHeader(name, size) {
|
|
3124
|
+
function tarHeader(name, size, mode = FILE_MODE) {
|
|
2995
3125
|
const header = Buffer.alloc(512);
|
|
2996
3126
|
header.write(name, 0, 100, "utf-8");
|
|
2997
|
-
octal(
|
|
3127
|
+
octal(mode, 8).copy(header, 100);
|
|
2998
3128
|
octal(0, 8).copy(header, 108);
|
|
2999
3129
|
octal(0, 8).copy(header, 116);
|
|
3000
3130
|
octal(size, 12).copy(header, 124);
|
|
@@ -3008,7 +3138,7 @@ function tarHeader(name, size) {
|
|
|
3008
3138
|
header.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
|
|
3009
3139
|
return header;
|
|
3010
3140
|
}
|
|
3011
|
-
function tarMember(name, payload) {
|
|
3141
|
+
function tarMember(name, payload, mode) {
|
|
3012
3142
|
const parts = [];
|
|
3013
3143
|
const nameBytes = Buffer.byteLength(name, "utf-8");
|
|
3014
3144
|
if (nameBytes > 100) {
|
|
@@ -3022,9 +3152,9 @@ function tarMember(name, payload) {
|
|
|
3022
3152
|
parts.push(lHeader, body);
|
|
3023
3153
|
const pad2 = body.length % 512 === 0 ? 0 : 512 - body.length % 512;
|
|
3024
3154
|
if (pad2) parts.push(Buffer.alloc(pad2));
|
|
3025
|
-
parts.push(tarHeader(name.slice(0, 100), payload.length));
|
|
3155
|
+
parts.push(tarHeader(name.slice(0, 100), payload.length, mode));
|
|
3026
3156
|
} else {
|
|
3027
|
-
parts.push(tarHeader(name, payload.length));
|
|
3157
|
+
parts.push(tarHeader(name, payload.length, mode));
|
|
3028
3158
|
}
|
|
3029
3159
|
parts.push(payload);
|
|
3030
3160
|
const pad = payload.length % 512 === 0 ? 0 : 512 - payload.length % 512;
|
|
@@ -3034,14 +3164,21 @@ function tarMember(name, payload) {
|
|
|
3034
3164
|
function buildSourceTarball(dir) {
|
|
3035
3165
|
const parts = [];
|
|
3036
3166
|
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
3037
|
-
|
|
3167
|
+
const abs = path3.join(dir, relPath);
|
|
3168
|
+
parts.push(
|
|
3169
|
+
...tarMember(
|
|
3170
|
+
relPath,
|
|
3171
|
+
fs3.readFileSync(abs),
|
|
3172
|
+
archiveMode(fs3.statSync(abs).mode)
|
|
3173
|
+
)
|
|
3174
|
+
);
|
|
3038
3175
|
}
|
|
3039
3176
|
parts.push(Buffer.alloc(1024));
|
|
3040
3177
|
return (0, import_node_zlib.gzipSync)(Buffer.concat(parts), { level: 9 });
|
|
3041
3178
|
}
|
|
3042
3179
|
function readLocalMeta(dir) {
|
|
3043
3180
|
try {
|
|
3044
|
-
const raw =
|
|
3181
|
+
const raw = fs3.readFileSync(path3.join(dir, META_RELPATH), "utf-8");
|
|
3045
3182
|
const parsed = JSON.parse(raw);
|
|
3046
3183
|
return parsed && typeof parsed === "object" ? parsed : null;
|
|
3047
3184
|
} catch {
|
|
@@ -3049,19 +3186,31 @@ function readLocalMeta(dir) {
|
|
|
3049
3186
|
}
|
|
3050
3187
|
}
|
|
3051
3188
|
function pushSlug(dir, explicit) {
|
|
3052
|
-
|
|
3189
|
+
return resolvePushSlug(dir, { explicit }).slug;
|
|
3190
|
+
}
|
|
3191
|
+
function resolvePushSlug(dir, opts = {}) {
|
|
3053
3192
|
const meta = readLocalMeta(dir);
|
|
3054
|
-
|
|
3055
|
-
|
|
3193
|
+
const tracked = typeof meta?.slug === "string" && meta.slug ? meta.slug : null;
|
|
3194
|
+
const declared = Object.entries({ ...opts.declared ?? {}, "pyproject.toml": readPyproject(dir).name }).filter((entry) => typeof entry[1] === "string" && entry[1] !== "").map(([source, slug]) => ({ source, slug }));
|
|
3195
|
+
const base = { tracked, declared, conflict: null };
|
|
3196
|
+
if (opts.explicit) return { ...base, slug: opts.explicit, source: "--slug" };
|
|
3197
|
+
if (tracked) {
|
|
3198
|
+
const minted = (d) => tracked.startsWith(d.slug) && MINTED_SUFFIX.test(tracked.slice(d.slug.length));
|
|
3199
|
+
const conflict = declared.find((d) => d.slug !== tracked && !minted(d)) ?? null;
|
|
3200
|
+
return { ...base, slug: tracked, source: ".gateway metadata", conflict };
|
|
3201
|
+
}
|
|
3202
|
+
const first = declared[0];
|
|
3203
|
+
if (first) return { ...base, slug: first.slug, source: first.source };
|
|
3204
|
+
return { ...base, slug: path3.basename(path3.resolve(dir)), source: "directory name" };
|
|
3056
3205
|
}
|
|
3057
3206
|
function writeLocalMeta(dir, meta) {
|
|
3058
|
-
|
|
3059
|
-
|
|
3207
|
+
fs3.mkdirSync(path3.join(dir, ".gateway"), { recursive: true });
|
|
3208
|
+
fs3.writeFileSync(path3.join(dir, META_RELPATH), JSON.stringify(meta, null, 2));
|
|
3060
3209
|
}
|
|
3061
3210
|
function snapshotTreeHashes(dir) {
|
|
3062
3211
|
const out = {};
|
|
3063
3212
|
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
3064
|
-
out[relPath] = (0,
|
|
3213
|
+
out[relPath] = (0, import_node_crypto5.createHash)("sha256").update(fs3.readFileSync(path3.join(dir, relPath))).digest("hex");
|
|
3065
3214
|
}
|
|
3066
3215
|
return out;
|
|
3067
3216
|
}
|
|
@@ -3079,7 +3228,7 @@ function localTreeState(dir) {
|
|
|
3079
3228
|
}
|
|
3080
3229
|
function readPyproject(dir) {
|
|
3081
3230
|
try {
|
|
3082
|
-
const text2 =
|
|
3231
|
+
const text2 = fs3.readFileSync(path3.join(dir, "pyproject.toml"), "utf-8");
|
|
3083
3232
|
const section = /\[project\]([\s\S]*?)(\n\[|$)/.exec(text2)?.[1] ?? "";
|
|
3084
3233
|
const name = /\bname\s*=\s*"([^"]+)"/.exec(section)?.[1];
|
|
3085
3234
|
const version = /\bversion\s*=\s*"([^"]+)"/.exec(section)?.[1];
|
|
@@ -3089,39 +3238,27 @@ function readPyproject(dir) {
|
|
|
3089
3238
|
}
|
|
3090
3239
|
}
|
|
3091
3240
|
function readWorldContractVersion(dir) {
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
}
|
|
3241
|
+
const world = WorldDir.open(dir);
|
|
3242
|
+
try {
|
|
3243
|
+
const parsed = JSON.parse(
|
|
3244
|
+
world.text(world.worldPath("schema/world.json")) ?? ""
|
|
3245
|
+
);
|
|
3246
|
+
const pkg = parsed["x-gateway"]?.package;
|
|
3247
|
+
if (pkg?.version) return { name: pkg.id, version: pkg.version };
|
|
3248
|
+
} catch {
|
|
3099
3249
|
}
|
|
3100
3250
|
return {};
|
|
3101
3251
|
}
|
|
3102
3252
|
function buildFileManifest(dir) {
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
} else if (entry.isFile()) {
|
|
3113
|
-
const payload = fs2.readFileSync(path2.join(dir, childRel));
|
|
3114
|
-
entries.push({
|
|
3115
|
-
path: childRel,
|
|
3116
|
-
isDirectory: false,
|
|
3117
|
-
size: payload.length,
|
|
3118
|
-
sha256: (0, import_node_crypto4.createHash)("sha256").update(payload).digest("hex")
|
|
3119
|
-
});
|
|
3120
|
-
}
|
|
3121
|
-
}
|
|
3122
|
-
};
|
|
3123
|
-
walk("");
|
|
3124
|
-
return entries;
|
|
3253
|
+
return collectArchiveRelPaths(dir).map((relPath) => {
|
|
3254
|
+
const payload = fs3.readFileSync(path3.join(dir, relPath));
|
|
3255
|
+
return {
|
|
3256
|
+
path: relPath,
|
|
3257
|
+
isDirectory: false,
|
|
3258
|
+
size: payload.length,
|
|
3259
|
+
sha256: (0, import_node_crypto5.createHash)("sha256").update(payload).digest("hex")
|
|
3260
|
+
};
|
|
3261
|
+
});
|
|
3125
3262
|
}
|
|
3126
3263
|
async function putObject(url, data, shaB64) {
|
|
3127
3264
|
const resp = await fetch(url, {
|
|
@@ -3133,31 +3270,39 @@ async function putObject(url, data, shaB64) {
|
|
|
3133
3270
|
body: new Uint8Array(data)
|
|
3134
3271
|
});
|
|
3135
3272
|
if (!resp.ok) {
|
|
3136
|
-
throw new Error(
|
|
3273
|
+
throw new Error(
|
|
3274
|
+
`object PUT -> ${resp.status}: ${(await resp.text()).slice(0, 300)}`
|
|
3275
|
+
);
|
|
3137
3276
|
}
|
|
3138
3277
|
}
|
|
3139
3278
|
async function push(dir, opts = {}) {
|
|
3140
3279
|
const options = opts.options ?? {};
|
|
3141
|
-
if (!
|
|
3280
|
+
if (!fs3.statSync(dir).isDirectory())
|
|
3281
|
+
throw new Error(`Not a directory: ${dir}`);
|
|
3142
3282
|
const localMeta = readLocalMeta(dir) ?? {};
|
|
3143
3283
|
const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
|
|
3144
3284
|
const baseVersionId = opts.baseVersionId ?? localMeta.versionId ?? null;
|
|
3285
|
+
const fileManifest = buildFileManifest(dir);
|
|
3286
|
+
if (fileManifest.length === 0)
|
|
3287
|
+
throw new Error(
|
|
3288
|
+
`Nothing to push: ${dir} has no files the archive rules keep (a world root marker, README.md, pyproject.toml, *.py, or any file below a subdirectory).`
|
|
3289
|
+
);
|
|
3145
3290
|
const contentHash = computeContentHash(dir);
|
|
3146
3291
|
const pyproject = readPyproject(dir);
|
|
3147
3292
|
const contractVersion = pyproject.version ? void 0 : readWorldContractVersion(dir).version;
|
|
3148
3293
|
if (contractVersion) pyproject.version = contractVersion;
|
|
3149
|
-
const slug = opts.slug ?? localMeta.slug ?? pyproject.name ??
|
|
3294
|
+
const slug = opts.slug ?? localMeta.slug ?? pyproject.name ?? path3.basename(path3.resolve(dir));
|
|
3295
|
+
const metadata = {
|
|
3296
|
+
...opts.name !== void 0 ? { name: opts.name } : {},
|
|
3297
|
+
...opts.description !== void 0 ? { description: opts.description } : {},
|
|
3298
|
+
...opts.tags !== void 0 ? { tags: opts.tags } : {},
|
|
3299
|
+
...opts.visibility ? { visibility: opts.visibility } : {}
|
|
3300
|
+
};
|
|
3150
3301
|
const resolve6 = await apiRequest(
|
|
3151
3302
|
options,
|
|
3152
3303
|
"POST",
|
|
3153
3304
|
"/api/public/benchmark-containers/resolve",
|
|
3154
|
-
{
|
|
3155
|
-
slug,
|
|
3156
|
-
name: opts.name ?? pyproject.name ?? slug,
|
|
3157
|
-
...opts.description ? { description: opts.description } : {},
|
|
3158
|
-
...opts.tags?.length ? { tags: opts.tags } : {},
|
|
3159
|
-
...opts.visibility ? { visibility: opts.visibility } : {}
|
|
3160
|
-
}
|
|
3305
|
+
{ slug, ...metadata }
|
|
3161
3306
|
);
|
|
3162
3307
|
const tarball = buildSourceTarball(dir);
|
|
3163
3308
|
const bundleSha256 = sha256b64(tarball);
|
|
@@ -3168,7 +3313,7 @@ async function push(dir, opts = {}) {
|
|
|
3168
3313
|
bundleSizeBytes: tarball.length,
|
|
3169
3314
|
...pyproject.version ? { semanticVersion: pyproject.version } : {},
|
|
3170
3315
|
pyprojectMeta: pyproject,
|
|
3171
|
-
fileManifest
|
|
3316
|
+
fileManifest,
|
|
3172
3317
|
branch,
|
|
3173
3318
|
baseVersionId,
|
|
3174
3319
|
...opts.force ? { force: true } : {},
|
|
@@ -3176,12 +3321,18 @@ async function push(dir, opts = {}) {
|
|
|
3176
3321
|
...opts.author ? { author: opts.author } : {},
|
|
3177
3322
|
...opts.imageRef ? { imageRef: opts.imageRef } : {}
|
|
3178
3323
|
});
|
|
3179
|
-
if (version.uploadUrl)
|
|
3324
|
+
if (version.uploadUrl)
|
|
3325
|
+
await putObject(version.uploadUrl, tarball, bundleSha256);
|
|
3180
3326
|
const final = await apiRequest(
|
|
3181
3327
|
options,
|
|
3182
3328
|
"POST",
|
|
3183
3329
|
`/api/public/benchmark-containers/versions/${version.versionId}/finalize`,
|
|
3184
|
-
{
|
|
3330
|
+
{
|
|
3331
|
+
branch,
|
|
3332
|
+
baseVersionId,
|
|
3333
|
+
...opts.force ? { force: true } : {},
|
|
3334
|
+
...metadata
|
|
3335
|
+
}
|
|
3185
3336
|
);
|
|
3186
3337
|
writeLocalMeta(dir, {
|
|
3187
3338
|
...localMeta,
|
|
@@ -3214,7 +3365,14 @@ function extractBundle(bundle, dir) {
|
|
|
3214
3365
|
offset += 512;
|
|
3215
3366
|
if (header.every((b) => b === 0)) break;
|
|
3216
3367
|
const rawName = header.subarray(0, 100).toString("utf-8").replace(/\0.*$/, "");
|
|
3217
|
-
const size = parseInt(
|
|
3368
|
+
const size = parseInt(
|
|
3369
|
+
header.subarray(124, 136).toString("ascii").replace(/\0.*$/, "").trim(),
|
|
3370
|
+
8
|
|
3371
|
+
) || 0;
|
|
3372
|
+
const mode = parseInt(
|
|
3373
|
+
header.subarray(100, 108).toString("ascii").replace(/\0.*$/, "").trim(),
|
|
3374
|
+
8
|
|
3375
|
+
) || FILE_MODE;
|
|
3218
3376
|
const typeflag = header.subarray(156, 157).toString("ascii");
|
|
3219
3377
|
const body = tar.subarray(offset, offset + size);
|
|
3220
3378
|
offset += size + (size % 512 === 0 ? 0 : 512 - size % 512);
|
|
@@ -3226,9 +3384,10 @@ function extractBundle(bundle, dir) {
|
|
|
3226
3384
|
longName = null;
|
|
3227
3385
|
if (typeflag !== "0" && typeflag !== "") continue;
|
|
3228
3386
|
if (name.startsWith("/") || name.split("/").includes("..")) continue;
|
|
3229
|
-
const target =
|
|
3230
|
-
|
|
3231
|
-
|
|
3387
|
+
const target = path3.join(dir, name);
|
|
3388
|
+
fs3.mkdirSync(path3.dirname(target), { recursive: true });
|
|
3389
|
+
fs3.writeFileSync(target, body, { mode: archiveMode(mode) });
|
|
3390
|
+
fs3.chmodSync(target, archiveMode(mode));
|
|
3232
3391
|
written++;
|
|
3233
3392
|
}
|
|
3234
3393
|
return written;
|
|
@@ -3244,13 +3403,13 @@ async function pull(slugRef, target, options = {}) {
|
|
|
3244
3403
|
if (!details.downloadUrl) {
|
|
3245
3404
|
throw new Error(`${slug}@${ref} resolved but has no uploaded bundle`);
|
|
3246
3405
|
}
|
|
3247
|
-
let dir = target ??
|
|
3406
|
+
let dir = target ?? path3.join(process.cwd(), slug);
|
|
3248
3407
|
if (!target) {
|
|
3249
3408
|
let i = 1;
|
|
3250
3409
|
const base = dir;
|
|
3251
|
-
while (
|
|
3410
|
+
while (fs3.existsSync(dir)) dir = `${base}-${i++}`;
|
|
3252
3411
|
}
|
|
3253
|
-
|
|
3412
|
+
fs3.mkdirSync(dir, { recursive: true });
|
|
3254
3413
|
const resp = await fetch(details.downloadUrl);
|
|
3255
3414
|
if (!resp.ok) throw new Error(`bundle download -> ${resp.status}`);
|
|
3256
3415
|
const bundle = Buffer.from(await resp.arrayBuffer());
|
|
@@ -3275,17 +3434,21 @@ async function pull(slugRef, target, options = {}) {
|
|
|
3275
3434
|
async function propose(dir, opts) {
|
|
3276
3435
|
const meta = readLocalMeta(dir);
|
|
3277
3436
|
if (!meta?.slug || !meta.baseFiles) {
|
|
3278
|
-
throw new Error(
|
|
3437
|
+
throw new Error(
|
|
3438
|
+
"This dir has no pull/push base \u2014 run pull first, then edit, then propose."
|
|
3439
|
+
);
|
|
3279
3440
|
}
|
|
3280
3441
|
const state2 = localTreeState(dir);
|
|
3281
|
-
if (state2.clean)
|
|
3442
|
+
if (state2.clean)
|
|
3443
|
+
throw new Error("Nothing to propose \u2014 the tree matches its base.");
|
|
3282
3444
|
const changes = state2.dirty.map((relPath) => {
|
|
3283
|
-
const abs =
|
|
3284
|
-
if (!
|
|
3445
|
+
const abs = path3.join(dir, relPath);
|
|
3446
|
+
if (!fs3.existsSync(abs))
|
|
3447
|
+
return { path: relPath, action: "delete" };
|
|
3285
3448
|
return {
|
|
3286
3449
|
path: relPath,
|
|
3287
3450
|
action: "write",
|
|
3288
|
-
newContent:
|
|
3451
|
+
newContent: fs3.readFileSync(abs, "utf-8")
|
|
3289
3452
|
};
|
|
3290
3453
|
});
|
|
3291
3454
|
const res = await apiRequest(
|
|
@@ -3311,30 +3474,33 @@ async function createRef(slug, name, kind, versionId, options = {}) {
|
|
|
3311
3474
|
...versionId ? { versionId } : {}
|
|
3312
3475
|
});
|
|
3313
3476
|
}
|
|
3314
|
-
var
|
|
3477
|
+
var import_node_crypto5, import_node_zlib, fs3, path3, DEFAULT_BRANCH, META_RELPATH, FILE_MODE, EXEC_MODE, MINTED_SUFFIX, PushSlugConflict, sha256b64;
|
|
3315
3478
|
var init_hub = __esm({
|
|
3316
3479
|
"src/worlds/hub.ts"() {
|
|
3317
3480
|
"use strict";
|
|
3318
|
-
|
|
3481
|
+
import_node_crypto5 = require("crypto");
|
|
3319
3482
|
import_node_zlib = require("zlib");
|
|
3320
|
-
|
|
3321
|
-
|
|
3483
|
+
fs3 = __toESM(require("fs"));
|
|
3484
|
+
path3 = __toESM(require("path"));
|
|
3322
3485
|
init_worlds();
|
|
3486
|
+
init_worldDir();
|
|
3323
3487
|
DEFAULT_BRANCH = "main";
|
|
3324
|
-
META_RELPATH =
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3488
|
+
META_RELPATH = path3.join(".gateway", ".env-metadata.json");
|
|
3489
|
+
FILE_MODE = 420;
|
|
3490
|
+
EXEC_MODE = 493;
|
|
3491
|
+
MINTED_SUFFIX = /^-\d+$/;
|
|
3492
|
+
PushSlugConflict = class extends Error {
|
|
3493
|
+
constructor(dir, tracked, declared) {
|
|
3494
|
+
super(
|
|
3495
|
+
`${dir} tracks slug \`${tracked}\` (${META_RELPATH}, from an earlier push) but its ${declared.source} says \`${declared.slug}\`. Nothing was pushed. Pass --slug ${declared.slug} to push there (and retrack the dir), --slug ${tracked} to keep pushing to \`${tracked}\`, or delete ${META_RELPATH} to forget the earlier push.`
|
|
3496
|
+
);
|
|
3497
|
+
this.dir = dir;
|
|
3498
|
+
this.tracked = tracked;
|
|
3499
|
+
this.declared = declared;
|
|
3500
|
+
this.name = "PushSlugConflict";
|
|
3501
|
+
}
|
|
3502
|
+
};
|
|
3503
|
+
sha256b64 = (data) => (0, import_node_crypto5.createHash)("sha256").update(data).digest("base64");
|
|
3338
3504
|
}
|
|
3339
3505
|
});
|
|
3340
3506
|
|
|
@@ -3850,15 +4016,15 @@ function applyTransforms(value, chain) {
|
|
|
3850
4016
|
for (const step of steps) out = step(out);
|
|
3851
4017
|
return out;
|
|
3852
4018
|
}
|
|
3853
|
-
var
|
|
4019
|
+
var import_node_crypto6, WHITESPACE, WHITESPACE_EDGES, WHITESPACE_RUN, NOT_DIGIT, STRIP_PREFIX, trim, emailLocalPart, emailDomain, stripPrefix, hex, SIMPLE, TRANSFORMS, text;
|
|
3854
4020
|
var init_transforms = __esm({
|
|
3855
4021
|
"src/worlds/transforms.ts"() {
|
|
3856
4022
|
"use strict";
|
|
3857
|
-
|
|
4023
|
+
import_node_crypto6 = require("crypto");
|
|
3858
4024
|
init_json();
|
|
3859
4025
|
WHITESPACE = " \n\r\f\v";
|
|
3860
|
-
WHITESPACE_EDGES =
|
|
3861
|
-
WHITESPACE_RUN =
|
|
4026
|
+
WHITESPACE_EDGES = new RegExp(`^[${WHITESPACE}]+|[${WHITESPACE}]+$`, "g");
|
|
4027
|
+
WHITESPACE_RUN = new RegExp(`[${WHITESPACE}]+`, "g");
|
|
3862
4028
|
NOT_DIGIT = /[^0-9]/g;
|
|
3863
4029
|
STRIP_PREFIX = "strip_prefix";
|
|
3864
4030
|
trim = (value) => value.replace(WHITESPACE_EDGES, "");
|
|
@@ -3871,7 +4037,7 @@ var init_transforms = __esm({
|
|
|
3871
4037
|
return at === -1 ? "" : value.slice(at + 1).toLowerCase();
|
|
3872
4038
|
};
|
|
3873
4039
|
stripPrefix = (prefix) => (value) => value.startsWith(prefix) ? value.slice(prefix.length) : value;
|
|
3874
|
-
hex = (algorithm, value) => (0,
|
|
4040
|
+
hex = (algorithm, value) => (0, import_node_crypto6.createHash)(algorithm).update(value, "utf8").digest("hex");
|
|
3875
4041
|
SIMPLE = {
|
|
3876
4042
|
lower: (value) => value.toLowerCase(),
|
|
3877
4043
|
upper: (value) => value.toUpperCase(),
|
|
@@ -3903,7 +4069,7 @@ function pyText(value) {
|
|
|
3903
4069
|
return pyFloatRepr(value);
|
|
3904
4070
|
}
|
|
3905
4071
|
function hashForm(slug, plaintext) {
|
|
3906
|
-
return "sha256:" + (0,
|
|
4072
|
+
return "sha256:" + (0, import_node_crypto7.createHash)("sha256").update(`${slug}:${pyText(plaintext)}`, "utf8").digest("hex");
|
|
3907
4073
|
}
|
|
3908
4074
|
function preserveForm(slug, plaintext) {
|
|
3909
4075
|
const kind = typeof plaintext === "string" ? "str" : isInt(plaintext) ? "int" : "float";
|
|
@@ -3912,12 +4078,12 @@ function preserveForm(slug, plaintext) {
|
|
|
3912
4078
|
}
|
|
3913
4079
|
function shapeText(slug, text2, kind) {
|
|
3914
4080
|
const seed = Buffer.from(`${slug}:${text2}`, "utf8");
|
|
3915
|
-
let digest = (0,
|
|
4081
|
+
let digest = (0, import_node_crypto7.createHash)("sha256").update(seed).digest();
|
|
3916
4082
|
let cursor = 0;
|
|
3917
4083
|
const out = [];
|
|
3918
4084
|
for (const char of text2) {
|
|
3919
4085
|
if (cursor === digest.length) {
|
|
3920
|
-
digest = (0,
|
|
4086
|
+
digest = (0, import_node_crypto7.createHash)("sha256").update(Buffer.concat([digest, seed])).digest();
|
|
3921
4087
|
cursor = 0;
|
|
3922
4088
|
}
|
|
3923
4089
|
const byte = digest[cursor];
|
|
@@ -3954,11 +4120,11 @@ function storedForms(world, field, plaintext, transform) {
|
|
|
3954
4120
|
if (policy !== null) return [roundForm(plaintext, world.redact.round[field])];
|
|
3955
4121
|
return [plaintext];
|
|
3956
4122
|
}
|
|
3957
|
-
var
|
|
4123
|
+
var import_node_crypto7, isInt;
|
|
3958
4124
|
var init_redaction = __esm({
|
|
3959
4125
|
"src/worlds/redaction.ts"() {
|
|
3960
4126
|
"use strict";
|
|
3961
|
-
|
|
4127
|
+
import_node_crypto7 = require("crypto");
|
|
3962
4128
|
init_json();
|
|
3963
4129
|
init_transforms();
|
|
3964
4130
|
isInt = (value) => typeof value === "number" && Number.isInteger(value);
|
|
@@ -3966,8 +4132,9 @@ var init_redaction = __esm({
|
|
|
3966
4132
|
});
|
|
3967
4133
|
|
|
3968
4134
|
// src/worlds/query.ts
|
|
3969
|
-
function queryRowsSearch(entity, opts = {}) {
|
|
4135
|
+
function queryRowsSearch(entity, opts = {}, ref) {
|
|
3970
4136
|
const search = new URLSearchParams();
|
|
4137
|
+
if (ref) search.set("ref", ref);
|
|
3971
4138
|
search.set("entity", entity);
|
|
3972
4139
|
if (opts.where && Object.keys(opts.where).length) search.set("where", JSON.stringify(opts.where));
|
|
3973
4140
|
if (opts.limit !== void 0) search.set("limit", String(opts.limit));
|
|
@@ -3977,17 +4144,15 @@ function queryRowsSearch(entity, opts = {}) {
|
|
|
3977
4144
|
if (opts.compare) search.set("compare", "true");
|
|
3978
4145
|
return `?${search.toString()}`;
|
|
3979
4146
|
}
|
|
3980
|
-
async function queryRows(
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
`queryRows reads a world's head version: pass the slug without @ref (got ${slug})`
|
|
3984
|
-
);
|
|
4147
|
+
async function queryRows(slugRef, entity, opts = {}, options = {}) {
|
|
4148
|
+
const [slug, ref] = slugRef.split("@", 2);
|
|
4149
|
+
if (!slug || ref === "") throw new Error(`queryRows needs a slug, optionally slug@ref (got ${slugRef})`);
|
|
3985
4150
|
if (!entity) throw new Error("queryRows needs an entity");
|
|
3986
4151
|
if (opts.compare && !opts.across) throw new Error("queryRows: compare needs across");
|
|
3987
4152
|
return apiRequest(
|
|
3988
4153
|
options,
|
|
3989
4154
|
"GET",
|
|
3990
|
-
`/api/public/worlds/${encodeURIComponent(slug)}/data${queryRowsSearch(entity, opts)}`
|
|
4155
|
+
`/api/public/worlds/${encodeURIComponent(slug)}/data${queryRowsSearch(entity, opts, ref)}`
|
|
3991
4156
|
);
|
|
3992
4157
|
}
|
|
3993
4158
|
var init_query = __esm({
|
|
@@ -4043,11 +4208,20 @@ var init_router = __esm({
|
|
|
4043
4208
|
});
|
|
4044
4209
|
|
|
4045
4210
|
// src/cli/config.ts
|
|
4211
|
+
function transportDetail(error, timeoutMs) {
|
|
4212
|
+
if (error instanceof Error && error.name === "AbortError") return `timed out after ${Math.round(timeoutMs / 1e3)}s`;
|
|
4213
|
+
const cause = error instanceof Error && error.cause instanceof Error ? error.cause : void 0;
|
|
4214
|
+
const code = cause && "code" in cause && typeof cause.code === "string" ? `${cause.code}: ` : "";
|
|
4215
|
+
const message = cause?.message ?? (error instanceof Error ? error.message : String(error));
|
|
4216
|
+
return `${code}${message}`;
|
|
4217
|
+
}
|
|
4046
4218
|
async function api(creds, method, route, body, init2 = {}) {
|
|
4047
4219
|
const controller = new AbortController();
|
|
4048
|
-
const
|
|
4220
|
+
const timeoutMs = init2.timeoutMs ?? 12e4;
|
|
4221
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
4222
|
+
let res;
|
|
4049
4223
|
try {
|
|
4050
|
-
|
|
4224
|
+
res = await fetch(`${creds.host}${route}`, {
|
|
4051
4225
|
method,
|
|
4052
4226
|
headers: {
|
|
4053
4227
|
"content-type": "application/json",
|
|
@@ -4057,6 +4231,11 @@ async function api(creds, method, route, body, init2 = {}) {
|
|
|
4057
4231
|
...body === void 0 ? {} : { body: JSON.stringify(body) },
|
|
4058
4232
|
signal: controller.signal
|
|
4059
4233
|
});
|
|
4234
|
+
} catch (error) {
|
|
4235
|
+
clearTimeout(timer);
|
|
4236
|
+
throw new UnreachableError(creds.host, method, route, transportDetail(error, timeoutMs));
|
|
4237
|
+
}
|
|
4238
|
+
try {
|
|
4060
4239
|
const text2 = await res.text();
|
|
4061
4240
|
if (!res.ok) throw new ApiError(res.status, method, route, text2);
|
|
4062
4241
|
return text2 ? JSON.parse(text2) : {};
|
|
@@ -4064,13 +4243,13 @@ async function api(creds, method, route, body, init2 = {}) {
|
|
|
4064
4243
|
clearTimeout(timer);
|
|
4065
4244
|
}
|
|
4066
4245
|
}
|
|
4067
|
-
var
|
|
4246
|
+
var fs4, os, path4, ApiError, UnreachableError;
|
|
4068
4247
|
var init_config = __esm({
|
|
4069
4248
|
"src/cli/config.ts"() {
|
|
4070
4249
|
"use strict";
|
|
4071
|
-
|
|
4250
|
+
fs4 = __toESM(require("fs"));
|
|
4072
4251
|
os = __toESM(require("os"));
|
|
4073
|
-
|
|
4252
|
+
path4 = __toESM(require("path"));
|
|
4074
4253
|
init_router();
|
|
4075
4254
|
ApiError = class extends CliError {
|
|
4076
4255
|
constructor(status, method, route, text2) {
|
|
@@ -4081,6 +4260,15 @@ var init_config = __esm({
|
|
|
4081
4260
|
this.text = text2;
|
|
4082
4261
|
}
|
|
4083
4262
|
};
|
|
4263
|
+
UnreachableError = class extends CliError {
|
|
4264
|
+
constructor(host, method, route, detail) {
|
|
4265
|
+
super(`${method} ${route}: no answer from ${host} (${detail}). Check the host and the network, then retry.`, 1);
|
|
4266
|
+
this.host = host;
|
|
4267
|
+
this.method = method;
|
|
4268
|
+
this.route = route;
|
|
4269
|
+
this.detail = detail;
|
|
4270
|
+
}
|
|
4271
|
+
};
|
|
4084
4272
|
}
|
|
4085
4273
|
});
|
|
4086
4274
|
|
|
@@ -4099,16 +4287,16 @@ function runtimeRoot() {
|
|
|
4099
4287
|
const candidates = [];
|
|
4100
4288
|
if (process.env.GATEWAY_RUNTIME_PATH)
|
|
4101
4289
|
candidates.push(process.env.GATEWAY_RUNTIME_PATH);
|
|
4102
|
-
const entry = process.argv[1] &&
|
|
4290
|
+
const entry = process.argv[1] && fs5.existsSync(process.argv[1]) ? path5.dirname(fs5.realpathSync(process.argv[1])) : null;
|
|
4103
4291
|
if (entry) {
|
|
4104
|
-
candidates.push(
|
|
4105
|
-
candidates.push(
|
|
4106
|
-
candidates.push(
|
|
4292
|
+
candidates.push(path5.resolve(entry, "..", "runtime"));
|
|
4293
|
+
candidates.push(path5.resolve(entry, "..", "..", "runtime"));
|
|
4294
|
+
candidates.push(path5.resolve(entry, "..", "..", "sdk", "src"));
|
|
4107
4295
|
}
|
|
4108
|
-
candidates.push(
|
|
4296
|
+
candidates.push(path5.resolve(process.cwd(), "runtime"));
|
|
4109
4297
|
for (const root of candidates)
|
|
4110
|
-
if (
|
|
4111
|
-
|
|
4298
|
+
if (fs5.existsSync(
|
|
4299
|
+
path5.join(root, "gatewaysdk", "environment", "schema", "platform.py")
|
|
4112
4300
|
))
|
|
4113
4301
|
return root;
|
|
4114
4302
|
return null;
|
|
@@ -4249,14 +4437,14 @@ async function localBridge(request, local, opts = {}) {
|
|
|
4249
4437
|
child.stdin.end(input);
|
|
4250
4438
|
});
|
|
4251
4439
|
}
|
|
4252
|
-
var import_node_child_process,
|
|
4440
|
+
var import_node_child_process, fs5, net, path5, MIN_PYTHON, DIAGNOSTIC_TAIL, BRIDGE_BOOTSTRAP, pythonCache, pythonProbes, PYTHON_CANDIDATES, UV_FIND;
|
|
4253
4441
|
var init_local = __esm({
|
|
4254
4442
|
"src/cli/runtime/local.ts"() {
|
|
4255
4443
|
"use strict";
|
|
4256
4444
|
import_node_child_process = require("child_process");
|
|
4257
|
-
|
|
4445
|
+
fs5 = __toESM(require("fs"));
|
|
4258
4446
|
net = __toESM(require("net"));
|
|
4259
|
-
|
|
4447
|
+
path5 = __toESM(require("path"));
|
|
4260
4448
|
init_router();
|
|
4261
4449
|
init_config();
|
|
4262
4450
|
MIN_PYTHON = [3, 11];
|
|
@@ -4284,22 +4472,22 @@ function isBuildOutput(relPath) {
|
|
|
4284
4472
|
return BUILD_OUTPUT.test(relPath);
|
|
4285
4473
|
}
|
|
4286
4474
|
function readWorldFiles(dir, opts = {}) {
|
|
4287
|
-
if (!
|
|
4475
|
+
if (!fs6.existsSync(dir) || !fs6.statSync(dir).isDirectory())
|
|
4288
4476
|
throw new CliError(`Not a directory: ${dir}`);
|
|
4289
4477
|
const out = [];
|
|
4290
4478
|
const walk = (rel) => {
|
|
4291
|
-
const abs = rel ?
|
|
4292
|
-
const entries =
|
|
4479
|
+
const abs = rel ? path6.join(dir, rel) : dir;
|
|
4480
|
+
const entries = fs6.readdirSync(abs, { withFileTypes: true });
|
|
4293
4481
|
entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
4294
4482
|
for (const entry of entries) {
|
|
4295
4483
|
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
4296
4484
|
if (entry.isDirectory()) {
|
|
4297
|
-
if (!
|
|
4485
|
+
if (!SKIP_DIRS.has(entry.name)) walk(childRel);
|
|
4298
4486
|
} else if (entry.isFile()) {
|
|
4299
4487
|
if (!opts.buildOutputs && isBuildOutput(childRel)) continue;
|
|
4300
4488
|
out.push({
|
|
4301
4489
|
path: childRel,
|
|
4302
|
-
content:
|
|
4490
|
+
content: fs6.readFileSync(path6.join(dir, childRel)).toString("base64")
|
|
4303
4491
|
});
|
|
4304
4492
|
}
|
|
4305
4493
|
}
|
|
@@ -4314,19 +4502,19 @@ function assertSafeRelPath(rel) {
|
|
|
4314
4502
|
}
|
|
4315
4503
|
function writeWorldFiles(dir, files, opts = {}) {
|
|
4316
4504
|
for (const file of files) assertSafeRelPath(file.path);
|
|
4317
|
-
if (opts.clean &&
|
|
4505
|
+
if (opts.clean && fs6.existsSync(dir)) {
|
|
4318
4506
|
const keep = new Set(files.map((f) => f.path));
|
|
4319
4507
|
for (const existing of readWorldFiles(dir)) {
|
|
4320
|
-
if (!keep.has(existing.path))
|
|
4508
|
+
if (!keep.has(existing.path)) fs6.rmSync(path6.join(dir, existing.path));
|
|
4321
4509
|
}
|
|
4322
4510
|
pruneEmptyDirs(dir);
|
|
4323
4511
|
}
|
|
4324
|
-
|
|
4512
|
+
fs6.mkdirSync(dir, { recursive: true });
|
|
4325
4513
|
const written = [];
|
|
4326
4514
|
for (const file of files) {
|
|
4327
|
-
const target =
|
|
4328
|
-
|
|
4329
|
-
|
|
4515
|
+
const target = path6.join(dir, file.path);
|
|
4516
|
+
fs6.mkdirSync(path6.dirname(target), { recursive: true });
|
|
4517
|
+
fs6.writeFileSync(target, Buffer.from(file.content, "base64"));
|
|
4330
4518
|
written.push(file.path);
|
|
4331
4519
|
}
|
|
4332
4520
|
return written;
|
|
@@ -4334,14 +4522,14 @@ function writeWorldFiles(dir, files, opts = {}) {
|
|
|
4334
4522
|
function pruneEmptyDirs(root) {
|
|
4335
4523
|
const walk = (abs) => {
|
|
4336
4524
|
let empty = true;
|
|
4337
|
-
for (const entry of
|
|
4338
|
-
const child =
|
|
4525
|
+
for (const entry of fs6.readdirSync(abs, { withFileTypes: true })) {
|
|
4526
|
+
const child = path6.join(abs, entry.name);
|
|
4339
4527
|
if (entry.isDirectory()) {
|
|
4340
|
-
if (
|
|
4528
|
+
if (SKIP_DIRS.has(entry.name)) {
|
|
4341
4529
|
empty = false;
|
|
4342
4530
|
continue;
|
|
4343
4531
|
}
|
|
4344
|
-
if (walk(child))
|
|
4532
|
+
if (walk(child)) fs6.rmdirSync(child);
|
|
4345
4533
|
else empty = false;
|
|
4346
4534
|
} else empty = false;
|
|
4347
4535
|
}
|
|
@@ -4405,15 +4593,15 @@ async function runtime(creds, request, opts = {}) {
|
|
|
4405
4593
|
clearTimeout(timer);
|
|
4406
4594
|
}
|
|
4407
4595
|
}
|
|
4408
|
-
var
|
|
4596
|
+
var fs6, path6, SKIP_DIRS, BUILD_OUTPUT, BridgeRejected;
|
|
4409
4597
|
var init_client = __esm({
|
|
4410
4598
|
"src/cli/runtime/client.ts"() {
|
|
4411
4599
|
"use strict";
|
|
4412
|
-
|
|
4413
|
-
|
|
4600
|
+
fs6 = __toESM(require("fs"));
|
|
4601
|
+
path6 = __toESM(require("path"));
|
|
4414
4602
|
init_router();
|
|
4415
4603
|
init_local();
|
|
4416
|
-
|
|
4604
|
+
SKIP_DIRS = /* @__PURE__ */ new Set(["__pycache__", ".git", "node_modules", ".gateway"]);
|
|
4417
4605
|
BUILD_OUTPUT = /(^|\/)state\.(sqlite|json)$/;
|
|
4418
4606
|
BridgeRejected = class extends CliError {
|
|
4419
4607
|
constructor(detail) {
|
|
@@ -4460,7 +4648,7 @@ function dropBuildOutputs(dir) {
|
|
|
4460
4648
|
const dropped = [];
|
|
4461
4649
|
for (const file of readWorldFiles(dir, { buildOutputs: true })) {
|
|
4462
4650
|
if (!BUILD_OUTPUT2.test(file.path)) continue;
|
|
4463
|
-
|
|
4651
|
+
fs7.rmSync(path7.join(dir, file.path), { force: true });
|
|
4464
4652
|
dropped.push(file.path);
|
|
4465
4653
|
}
|
|
4466
4654
|
return dropped;
|
|
@@ -4471,6 +4659,7 @@ async function checkLocalWorldData(dir, rows, creds, opts = {}) {
|
|
|
4471
4659
|
files: readWorldFiles(dir),
|
|
4472
4660
|
rows,
|
|
4473
4661
|
mode: "append",
|
|
4662
|
+
dry_run: true,
|
|
4474
4663
|
...opts.redact && opts.redact !== "off" ? { redact: opts.redact } : {}
|
|
4475
4664
|
});
|
|
4476
4665
|
return {
|
|
@@ -4526,7 +4715,7 @@ async function queryLocalWorldDataAcross(dir, entity, where, opts) {
|
|
|
4526
4715
|
const payload = await localBridge(
|
|
4527
4716
|
{
|
|
4528
4717
|
command: "query",
|
|
4529
|
-
world:
|
|
4718
|
+
world: path7.resolve(dir),
|
|
4530
4719
|
entity,
|
|
4531
4720
|
across: true,
|
|
4532
4721
|
...opts.compare ? { compare: true } : {},
|
|
@@ -4573,12 +4762,12 @@ async function ingestLocalToolCalls(dir, records, opts, creds) {
|
|
|
4573
4762
|
...res.batch !== void 0 ? { batch: res.batch } : {}
|
|
4574
4763
|
};
|
|
4575
4764
|
}
|
|
4576
|
-
var
|
|
4765
|
+
var fs7, path7, BUILD_OUTPUT2;
|
|
4577
4766
|
var init_data = __esm({
|
|
4578
4767
|
"src/cli/runtime/data.ts"() {
|
|
4579
4768
|
"use strict";
|
|
4580
|
-
|
|
4581
|
-
|
|
4769
|
+
fs7 = __toESM(require("fs"));
|
|
4770
|
+
path7 = __toESM(require("path"));
|
|
4582
4771
|
init_router();
|
|
4583
4772
|
init_client();
|
|
4584
4773
|
init_local();
|
|
@@ -4621,27 +4810,27 @@ function typescriptPrefix() {
|
|
|
4621
4810
|
}
|
|
4622
4811
|
}
|
|
4623
4812
|
function command(script, wrapperDir) {
|
|
4624
|
-
const suffix =
|
|
4813
|
+
const suffix = path8.extname(script).toLowerCase();
|
|
4625
4814
|
if (PYTHON_SUFFIXES.includes(suffix)) return { file: process.env.GATEWAY_PYTHON || "python3", args: ["-c", PY_WRAPPER, script] };
|
|
4626
4815
|
if (NODE_SUFFIXES.includes(suffix)) {
|
|
4627
|
-
const wrapper =
|
|
4628
|
-
|
|
4816
|
+
const wrapper = path8.join(wrapperDir, "transform-wrapper.mjs");
|
|
4817
|
+
fs8.writeFileSync(wrapper, NODE_WRAPPER);
|
|
4629
4818
|
const flags = suffix === ".ts" || suffix === ".mts" ? typescriptPrefix() : [];
|
|
4630
4819
|
return { file: process.env.GATEWAY_NODE || process.execPath, args: [...flags, wrapper, script] };
|
|
4631
4820
|
}
|
|
4632
|
-
throw new CliError(`${
|
|
4821
|
+
throw new CliError(`${path8.basename(script)}: a transform script is .py, .ts, .mjs, .js or .cjs`, 1);
|
|
4633
4822
|
}
|
|
4634
4823
|
async function runTransform(script, records, opts = {}) {
|
|
4635
|
-
if (!
|
|
4636
|
-
const resolved =
|
|
4637
|
-
const name =
|
|
4824
|
+
if (!fs8.existsSync(script) || !fs8.statSync(script).isFile()) throw new CliError(`no transform script at ${script}`, 1);
|
|
4825
|
+
const resolved = path8.resolve(script);
|
|
4826
|
+
const name = path8.basename(script);
|
|
4638
4827
|
const timeoutMs = opts.timeoutMs ?? DEFAULT_TRANSFORM_TIMEOUT_MS;
|
|
4639
|
-
const scratch =
|
|
4828
|
+
const scratch = fs8.mkdtempSync(path8.join(os2.tmpdir(), "gateway-transform-"));
|
|
4640
4829
|
try {
|
|
4641
4830
|
const { file, args } = command(resolved, scratch);
|
|
4642
4831
|
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !SCRUBBED.test(key)));
|
|
4643
4832
|
const done = await new Promise((resolve6, reject) => {
|
|
4644
|
-
const child = (0, import_node_child_process2.spawn)(file, args, { cwd:
|
|
4833
|
+
const child = (0, import_node_child_process2.spawn)(file, args, { cwd: path8.dirname(resolved), env, stdio: ["pipe", "pipe", "pipe"] });
|
|
4645
4834
|
let stdout = "";
|
|
4646
4835
|
let stderr = "";
|
|
4647
4836
|
let timedOut = false;
|
|
@@ -4672,21 +4861,21 @@ async function runTransform(script, records, opts = {}) {
|
|
|
4672
4861
|
}
|
|
4673
4862
|
return { rows: checkRows(value, name), stderr: done.stderr };
|
|
4674
4863
|
} finally {
|
|
4675
|
-
|
|
4864
|
+
fs8.rmSync(scratch, { recursive: true, force: true });
|
|
4676
4865
|
}
|
|
4677
4866
|
}
|
|
4678
4867
|
async function transformRecords(transform, records, opts = {}) {
|
|
4679
4868
|
if (typeof transform === "function") return { rows: checkRows(await transform(records), "transform(records)"), stderr: "" };
|
|
4680
4869
|
return runTransform(transform, records, opts);
|
|
4681
4870
|
}
|
|
4682
|
-
var import_node_child_process2,
|
|
4871
|
+
var import_node_child_process2, fs8, os2, path8, import_node_module, import_meta, DEFAULT_TRANSFORM_TIMEOUT_MS, NODE_SUFFIXES, PYTHON_SUFFIXES, SCRUBBED, PY_WRAPPER, NODE_WRAPPER, isObject, tail;
|
|
4683
4872
|
var init_transform = __esm({
|
|
4684
4873
|
"src/cli/platform/worldData/transform.ts"() {
|
|
4685
4874
|
"use strict";
|
|
4686
4875
|
import_node_child_process2 = require("child_process");
|
|
4687
|
-
|
|
4876
|
+
fs8 = __toESM(require("fs"));
|
|
4688
4877
|
os2 = __toESM(require("os"));
|
|
4689
|
-
|
|
4878
|
+
path8 = __toESM(require("path"));
|
|
4690
4879
|
import_node_module = require("module");
|
|
4691
4880
|
init_router();
|
|
4692
4881
|
import_meta = {};
|
|
@@ -4770,11 +4959,16 @@ if (fn) {
|
|
|
4770
4959
|
});
|
|
4771
4960
|
|
|
4772
4961
|
// src/cli/platform/worldData/batches.ts
|
|
4962
|
+
function isSettled(status) {
|
|
4963
|
+
if (!TERMINAL_STATES.has(status.state)) return false;
|
|
4964
|
+
if (status.state !== "applied") return true;
|
|
4965
|
+
return status.dryRun === true || status.publish !== "now" || !status.snapshotId;
|
|
4966
|
+
}
|
|
4773
4967
|
function uploadHeadersFor(gz) {
|
|
4774
4968
|
return {
|
|
4775
4969
|
"content-type": "application/gzip",
|
|
4776
4970
|
"content-length": String(gz.length),
|
|
4777
|
-
"x-amz-checksum-sha256": (0,
|
|
4971
|
+
"x-amz-checksum-sha256": (0, import_node_crypto8.createHash)("sha256").update(gz).digest("base64")
|
|
4778
4972
|
};
|
|
4779
4973
|
}
|
|
4780
4974
|
async function forEachLimited(items, limit, fn) {
|
|
@@ -4811,7 +5005,7 @@ async function waitForBatch(client, batchId, opts = {}) {
|
|
|
4811
5005
|
opts.onPhase?.(phase);
|
|
4812
5006
|
last = phase;
|
|
4813
5007
|
}
|
|
4814
|
-
if (
|
|
5008
|
+
if (isSettled(status)) return status;
|
|
4815
5009
|
await sleep2(delay);
|
|
4816
5010
|
delay = Math.min(1e4, Math.round(delay * 1.5));
|
|
4817
5011
|
}
|
|
@@ -4820,15 +5014,20 @@ function phaseLine(status) {
|
|
|
4820
5014
|
const phase = status.progress?.phase;
|
|
4821
5015
|
return phase ? `${status.state} (${phase})` : status.state;
|
|
4822
5016
|
}
|
|
4823
|
-
var
|
|
5017
|
+
var import_node_crypto8, fs9, TERMINAL_STATES, sleepMs, WorldDataApi;
|
|
4824
5018
|
var init_batches = __esm({
|
|
4825
5019
|
"src/cli/platform/worldData/batches.ts"() {
|
|
4826
5020
|
"use strict";
|
|
4827
|
-
|
|
4828
|
-
|
|
5021
|
+
import_node_crypto8 = require("crypto");
|
|
5022
|
+
fs9 = __toESM(require("fs"));
|
|
4829
5023
|
init_config();
|
|
4830
5024
|
init_router();
|
|
4831
|
-
TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
5025
|
+
TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
5026
|
+
"applied",
|
|
5027
|
+
"refused",
|
|
5028
|
+
"published",
|
|
5029
|
+
"failed"
|
|
5030
|
+
]);
|
|
4832
5031
|
sleepMs = (ms) => new Promise((resolve6) => setTimeout(resolve6, ms));
|
|
4833
5032
|
WorldDataApi = class {
|
|
4834
5033
|
constructor(creds, slug, sleep2 = sleepMs) {
|
|
@@ -4842,22 +5041,61 @@ var init_batches = __esm({
|
|
|
4842
5041
|
return api(this.creds, "POST", `${this.base}/batches`, body);
|
|
4843
5042
|
}
|
|
4844
5043
|
register(batchId, chunk) {
|
|
4845
|
-
return api(
|
|
5044
|
+
return api(
|
|
5045
|
+
this.creds,
|
|
5046
|
+
"POST",
|
|
5047
|
+
`${this.base}/batches/${batchId}/chunks`,
|
|
5048
|
+
{
|
|
5049
|
+
ordinal: chunk.ordinal,
|
|
5050
|
+
digest: chunk.digest,
|
|
5051
|
+
bytes: chunk.bytes,
|
|
5052
|
+
rows: chunk.rows,
|
|
5053
|
+
...chunk.entity ? { entity: chunk.entity } : {}
|
|
5054
|
+
}
|
|
5055
|
+
);
|
|
5056
|
+
}
|
|
5057
|
+
/**
|
|
5058
|
+
* Stage a project-drive rows file as one chunk, on the platform: the server
|
|
5059
|
+
* streams it out of drive storage into chunk storage and answers with the
|
|
5060
|
+
* digest and gzipped size it derived. No bytes pass through this process.
|
|
5061
|
+
*/
|
|
5062
|
+
async stageDrive(batchId, chunk) {
|
|
5063
|
+
const staged = await api(this.creds, "POST", `${this.base}/batches/${batchId}/chunks`, {
|
|
4846
5064
|
ordinal: chunk.ordinal,
|
|
4847
|
-
|
|
4848
|
-
bytes: chunk.bytes,
|
|
4849
|
-
rows: chunk.rows,
|
|
5065
|
+
drive: chunk.path,
|
|
4850
5066
|
...chunk.entity ? { entity: chunk.entity } : {}
|
|
4851
5067
|
});
|
|
5068
|
+
if (!staged.uploaded || !staged.digest || staged.bytes === void 0 || staged.rows === void 0)
|
|
5069
|
+
throw new CliError(
|
|
5070
|
+
`the server staged drive:${chunk.path} as chunk ${chunk.ordinal} without its digest, size and rows`,
|
|
5071
|
+
1
|
|
5072
|
+
);
|
|
5073
|
+
return {
|
|
5074
|
+
ordinal: chunk.ordinal,
|
|
5075
|
+
digest: staged.digest,
|
|
5076
|
+
bytes: staged.bytes,
|
|
5077
|
+
rows: staged.rows,
|
|
5078
|
+
entity: chunk.entity,
|
|
5079
|
+
chunkId: staged.chunkId,
|
|
5080
|
+
uploaded: true
|
|
5081
|
+
};
|
|
4852
5082
|
}
|
|
4853
5083
|
markUploaded(batchId, chunkId) {
|
|
4854
|
-
return api(
|
|
5084
|
+
return api(
|
|
5085
|
+
this.creds,
|
|
5086
|
+
"POST",
|
|
5087
|
+
`${this.base}/batches/${batchId}/chunks/${chunkId}/uploaded`
|
|
5088
|
+
);
|
|
4855
5089
|
}
|
|
4856
5090
|
complete(batchId) {
|
|
4857
5091
|
return api(this.creds, "POST", `${this.base}/batches/${batchId}/complete`);
|
|
4858
5092
|
}
|
|
4859
5093
|
get(batchId) {
|
|
4860
|
-
return api(
|
|
5094
|
+
return api(
|
|
5095
|
+
this.creds,
|
|
5096
|
+
"GET",
|
|
5097
|
+
`${this.base}/batches/${batchId}`
|
|
5098
|
+
);
|
|
4861
5099
|
}
|
|
4862
5100
|
list(limit, cursor) {
|
|
4863
5101
|
const params = new URLSearchParams();
|
|
@@ -4867,7 +5105,12 @@ var init_batches = __esm({
|
|
|
4867
5105
|
return api(this.creds, "GET", `${this.base}/batches${q ? `?${q}` : ""}`);
|
|
4868
5106
|
}
|
|
4869
5107
|
publish(message) {
|
|
4870
|
-
return api(
|
|
5108
|
+
return api(
|
|
5109
|
+
this.creds,
|
|
5110
|
+
"POST",
|
|
5111
|
+
`${this.base}/publish`,
|
|
5112
|
+
message ? { message } : {}
|
|
5113
|
+
);
|
|
4871
5114
|
}
|
|
4872
5115
|
/**
|
|
4873
5116
|
* PUT the gzipped chunk to its presigned URL with exactly the headers the
|
|
@@ -4879,11 +5122,23 @@ var init_batches = __esm({
|
|
|
4879
5122
|
const delays = [500, 1500, 4e3];
|
|
4880
5123
|
for (let attempt = 0; ; attempt++) {
|
|
4881
5124
|
try {
|
|
4882
|
-
const res = await fetch(uploadUrl, {
|
|
5125
|
+
const res = await fetch(uploadUrl, {
|
|
5126
|
+
method: "PUT",
|
|
5127
|
+
body: new Uint8Array(gz),
|
|
5128
|
+
headers
|
|
5129
|
+
});
|
|
4883
5130
|
if (res.ok) return;
|
|
4884
|
-
if (res.status < 500 || attempt >= delays.length)
|
|
5131
|
+
if (res.status < 500 || attempt >= delays.length)
|
|
5132
|
+
throw new CliError(
|
|
5133
|
+
`chunk upload -> ${res.status}: ${(await res.text()).slice(0, 300)}`,
|
|
5134
|
+
1
|
|
5135
|
+
);
|
|
4885
5136
|
} catch (error) {
|
|
4886
|
-
if (error instanceof CliError || attempt >= delays.length)
|
|
5137
|
+
if (error instanceof CliError || attempt >= delays.length)
|
|
5138
|
+
throw error instanceof CliError ? error : new CliError(
|
|
5139
|
+
`chunk upload failed: ${error.message}`,
|
|
5140
|
+
1
|
|
5141
|
+
);
|
|
4887
5142
|
}
|
|
4888
5143
|
await this.sleep(delays[attempt]);
|
|
4889
5144
|
}
|
|
@@ -4892,11 +5147,23 @@ var init_batches = __esm({
|
|
|
4892
5147
|
async upload(batchId, chunk) {
|
|
4893
5148
|
const registered = await this.register(batchId, chunk);
|
|
4894
5149
|
if (!registered.uploaded) {
|
|
4895
|
-
if (!registered.uploadUrl)
|
|
5150
|
+
if (!registered.uploadUrl)
|
|
5151
|
+
throw new CliError(
|
|
5152
|
+
`the server registered chunk ${chunk.ordinal} without an upload URL`,
|
|
5153
|
+
1
|
|
5154
|
+
);
|
|
4896
5155
|
await this.put(registered.uploadUrl, chunk.gz, registered.uploadHeaders);
|
|
4897
5156
|
await this.markUploaded(batchId, registered.chunkId);
|
|
4898
5157
|
}
|
|
4899
|
-
return {
|
|
5158
|
+
return {
|
|
5159
|
+
ordinal: chunk.ordinal,
|
|
5160
|
+
digest: chunk.digest,
|
|
5161
|
+
bytes: chunk.bytes,
|
|
5162
|
+
rows: chunk.rows,
|
|
5163
|
+
entity: chunk.entity,
|
|
5164
|
+
chunkId: registered.chunkId,
|
|
5165
|
+
uploaded: true
|
|
5166
|
+
};
|
|
4900
5167
|
}
|
|
4901
5168
|
};
|
|
4902
5169
|
}
|
|
@@ -4910,14 +5177,20 @@ function chunkLine(row, declared) {
|
|
|
4910
5177
|
tagged.set("row", row.row);
|
|
4911
5178
|
return dumps(tagged) + "\n";
|
|
4912
5179
|
}
|
|
4913
|
-
async function* buildChunks(rows, plan) {
|
|
5180
|
+
async function* buildChunks(rows, plan, ordinalStart = 0) {
|
|
4914
5181
|
let lines = [];
|
|
4915
5182
|
let rawBytes = 0;
|
|
4916
|
-
let ordinal =
|
|
5183
|
+
let ordinal = ordinalStart;
|
|
4917
5184
|
let seen = 0;
|
|
4918
5185
|
const flush2 = () => {
|
|
4919
5186
|
const raw = Buffer.concat(lines);
|
|
4920
|
-
const built = finishChunk(
|
|
5187
|
+
const built = finishChunk(
|
|
5188
|
+
ordinal++,
|
|
5189
|
+
raw,
|
|
5190
|
+
lines.length,
|
|
5191
|
+
plan.entity,
|
|
5192
|
+
seen - lines.length
|
|
5193
|
+
);
|
|
4921
5194
|
lines = [];
|
|
4922
5195
|
rawBytes = 0;
|
|
4923
5196
|
return built;
|
|
@@ -4936,12 +5209,12 @@ function finishChunk(ordinal, raw, rows, entity, firstRow = 0) {
|
|
|
4936
5209
|
const gz = gzipDeterministic(raw, { name: chunkName(ordinal) });
|
|
4937
5210
|
return {
|
|
4938
5211
|
ordinal,
|
|
4939
|
-
digest: (0,
|
|
5212
|
+
digest: (0, import_node_crypto9.createHash)("sha256").update(gz).digest("hex"),
|
|
4940
5213
|
bytes: gz.length,
|
|
4941
5214
|
rows,
|
|
4942
5215
|
entity,
|
|
4943
5216
|
rawBytes: raw.length,
|
|
4944
|
-
contentSha256: (0,
|
|
5217
|
+
contentSha256: (0, import_node_crypto9.createHash)("sha256").update(raw).digest("hex"),
|
|
4945
5218
|
firstRow,
|
|
4946
5219
|
gz
|
|
4947
5220
|
};
|
|
@@ -4949,7 +5222,18 @@ function finishChunk(ordinal, raw, rows, entity, firstRow = 0) {
|
|
|
4949
5222
|
function gzipDeterministic(raw, opts = {}) {
|
|
4950
5223
|
const level = opts.level ?? GZIP_LEVEL;
|
|
4951
5224
|
const name = opts.name ? Buffer.concat([Buffer.from(opts.name, "latin1"), Buffer.from([0])]) : Buffer.alloc(0);
|
|
4952
|
-
const header = Buffer.from([
|
|
5225
|
+
const header = Buffer.from([
|
|
5226
|
+
31,
|
|
5227
|
+
139,
|
|
5228
|
+
8,
|
|
5229
|
+
opts.name ? 8 : 0,
|
|
5230
|
+
0,
|
|
5231
|
+
0,
|
|
5232
|
+
0,
|
|
5233
|
+
0,
|
|
5234
|
+
level === 9 ? 2 : level === 1 ? 4 : 0,
|
|
5235
|
+
255
|
|
5236
|
+
]);
|
|
4953
5237
|
const body = (0, import_node_zlib2.deflateRawSync)(raw, { level });
|
|
4954
5238
|
const trailer = Buffer.alloc(8);
|
|
4955
5239
|
trailer.writeUInt32LE(crc32(raw), 0);
|
|
@@ -4966,14 +5250,15 @@ function crc32(data) {
|
|
|
4966
5250
|
}
|
|
4967
5251
|
}
|
|
4968
5252
|
let crc = 4294967295;
|
|
4969
|
-
for (let i = 0; i < data.length; i++)
|
|
5253
|
+
for (let i = 0; i < data.length; i++)
|
|
5254
|
+
crc = table[(crc ^ data[i]) & 255] ^ crc >>> 8;
|
|
4970
5255
|
return (crc ^ 4294967295) >>> 0;
|
|
4971
5256
|
}
|
|
4972
|
-
var
|
|
5257
|
+
var import_node_crypto9, import_node_zlib2, DEFAULT_CHUNK_BYTES, GZIP_LEVEL, chunkName, table;
|
|
4973
5258
|
var init_chunker = __esm({
|
|
4974
5259
|
"src/cli/platform/worldData/chunker.ts"() {
|
|
4975
5260
|
"use strict";
|
|
4976
|
-
|
|
5261
|
+
import_node_crypto9 = require("crypto");
|
|
4977
5262
|
import_node_zlib2 = require("zlib");
|
|
4978
5263
|
init_json();
|
|
4979
5264
|
DEFAULT_CHUNK_BYTES = 64 * 1024 * 1024;
|
|
@@ -5046,7 +5331,7 @@ function parseToolCallRecord(raw) {
|
|
|
5046
5331
|
}
|
|
5047
5332
|
function readToolCallRecords(paths) {
|
|
5048
5333
|
const records = [];
|
|
5049
|
-
for (const file of paths) records.push(...toolCallRecordsIn(
|
|
5334
|
+
for (const file of paths) records.push(...toolCallRecordsIn(fs10.readFileSync(file, "utf8"), file));
|
|
5050
5335
|
return records;
|
|
5051
5336
|
}
|
|
5052
5337
|
function toolCallRecordsIn(text2, where) {
|
|
@@ -5082,7 +5367,7 @@ async function emit(sink, record) {
|
|
|
5082
5367
|
return;
|
|
5083
5368
|
}
|
|
5084
5369
|
if (typeof sink === "string") {
|
|
5085
|
-
|
|
5370
|
+
fs10.appendFileSync(sink, JSON.stringify(record) + "\n");
|
|
5086
5371
|
return;
|
|
5087
5372
|
}
|
|
5088
5373
|
if (typeof sink === "function") {
|
|
@@ -5113,14 +5398,6 @@ function captureToolCalls(tools, sink, opts = {}) {
|
|
|
5113
5398
|
}
|
|
5114
5399
|
return wrapped;
|
|
5115
5400
|
}
|
|
5116
|
-
function jsonish(value) {
|
|
5117
|
-
if (typeof value !== "string") return value;
|
|
5118
|
-
try {
|
|
5119
|
-
return JSON.parse(value);
|
|
5120
|
-
} catch {
|
|
5121
|
-
return value;
|
|
5122
|
-
}
|
|
5123
|
-
}
|
|
5124
5401
|
function toolCallsFromObservations(observations, opts = {}) {
|
|
5125
5402
|
const known = opts.tools ? new Set(opts.tools) : null;
|
|
5126
5403
|
const sinceMs = opts.since ? Date.parse(opts.since) : NaN;
|
|
@@ -5147,10 +5424,9 @@ function toolCallsFromObservations(observations, opts = {}) {
|
|
|
5147
5424
|
out.unknownTools[tool] = (out.unknownTools[tool] ?? 0) + 1;
|
|
5148
5425
|
continue;
|
|
5149
5426
|
}
|
|
5150
|
-
const
|
|
5151
|
-
const record = { tool, args: isObject2(input) ? input : {} };
|
|
5427
|
+
const record = { tool, args: isObject2(observation.input) ? observation.input : {} };
|
|
5152
5428
|
if (observation.level === "ERROR") record.error = observation.statusMessage || "ERROR";
|
|
5153
|
-
else record.result =
|
|
5429
|
+
else record.result = observation.output;
|
|
5154
5430
|
if (at !== void 0) record.at = at;
|
|
5155
5431
|
if (opts.session) record.session = opts.session;
|
|
5156
5432
|
record.source = `trace:${observation.traceId ?? "?"}/${observation.id ?? "?"}`;
|
|
@@ -5171,7 +5447,7 @@ async function ingest(world, records, options = {}) {
|
|
|
5171
5447
|
const mode = options.mode ?? "append";
|
|
5172
5448
|
const dryRun = Boolean(options.dryRun);
|
|
5173
5449
|
const creds = credentialsOf(options);
|
|
5174
|
-
const local =
|
|
5450
|
+
const local = fs10.existsSync(world) && fs10.statSync(world).isDirectory();
|
|
5175
5451
|
const map = options.map === void 0 ? void 0 : options.map;
|
|
5176
5452
|
let rows = null;
|
|
5177
5453
|
let stderr = "";
|
|
@@ -5187,9 +5463,9 @@ async function ingest(world, records, options = {}) {
|
|
|
5187
5463
|
if (!creds) throw new Error("Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey).");
|
|
5188
5464
|
const { pull: pull2 } = await Promise.resolve().then(() => (init_hub(), hub_exports));
|
|
5189
5465
|
const slug = world.split("@", 1)[0];
|
|
5190
|
-
const scratch =
|
|
5466
|
+
const scratch = fs10.mkdtempSync(path9.join(os3.tmpdir(), "gateway-ingest-"));
|
|
5191
5467
|
try {
|
|
5192
|
-
const pulled = await pull2(world,
|
|
5468
|
+
const pulled = await pull2(world, path9.join(scratch, slug), options);
|
|
5193
5469
|
let report;
|
|
5194
5470
|
let projected;
|
|
5195
5471
|
if (rows === null) {
|
|
@@ -5210,7 +5486,7 @@ async function ingest(world, records, options = {}) {
|
|
|
5210
5486
|
if (status.entityCounts) report.entityCounts = status.entityCounts;
|
|
5211
5487
|
return report;
|
|
5212
5488
|
} finally {
|
|
5213
|
-
|
|
5489
|
+
fs10.rmSync(scratch, { recursive: true, force: true });
|
|
5214
5490
|
}
|
|
5215
5491
|
}
|
|
5216
5492
|
async function ingestToolCalls(world, records, options = {}) {
|
|
@@ -5263,19 +5539,71 @@ async function sessionToolObservations(sessionId, options = {}) {
|
|
|
5263
5539
|
for (const observation of trace3.observations ?? []) out.push({ traceId: trace3.id ?? null, ...observation });
|
|
5264
5540
|
return out;
|
|
5265
5541
|
}
|
|
5266
|
-
var
|
|
5542
|
+
var fs10, os3, path9, isObject2, isoOf;
|
|
5267
5543
|
var init_ingest = __esm({
|
|
5268
5544
|
"src/worlds/ingest.ts"() {
|
|
5269
5545
|
"use strict";
|
|
5270
|
-
|
|
5546
|
+
fs10 = __toESM(require("fs"));
|
|
5271
5547
|
os3 = __toESM(require("os"));
|
|
5272
|
-
|
|
5548
|
+
path9 = __toESM(require("path"));
|
|
5273
5549
|
init_worlds();
|
|
5274
5550
|
isObject2 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5275
5551
|
isoOf = (value) => value instanceof Date ? value.toISOString() : typeof value === "string" ? value : void 0;
|
|
5276
5552
|
}
|
|
5277
5553
|
});
|
|
5278
5554
|
|
|
5555
|
+
// src/worlds/sandbox.ts
|
|
5556
|
+
function parseSlugRef(slugRef) {
|
|
5557
|
+
const at = slugRef.indexOf("@");
|
|
5558
|
+
if (at === -1) return { slug: slugRef };
|
|
5559
|
+
return { slug: slugRef.slice(0, at), ref: slugRef.slice(at + 1) };
|
|
5560
|
+
}
|
|
5561
|
+
var sid, WorldSandboxClient;
|
|
5562
|
+
var init_sandbox = __esm({
|
|
5563
|
+
"src/worlds/sandbox.ts"() {
|
|
5564
|
+
"use strict";
|
|
5565
|
+
init_worlds();
|
|
5566
|
+
sid = encodeURIComponent;
|
|
5567
|
+
WorldSandboxClient = class {
|
|
5568
|
+
constructor(options = {}) {
|
|
5569
|
+
this.options = options;
|
|
5570
|
+
}
|
|
5571
|
+
open(target) {
|
|
5572
|
+
return apiRequest(this.options, "POST", "/api/public/benchmark-containers/sandboxes", target);
|
|
5573
|
+
}
|
|
5574
|
+
status(sandboxId) {
|
|
5575
|
+
return apiRequest(this.options, "GET", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}`);
|
|
5576
|
+
}
|
|
5577
|
+
async read(sandboxId, filePath) {
|
|
5578
|
+
const res = await apiRequest(
|
|
5579
|
+
this.options,
|
|
5580
|
+
"GET",
|
|
5581
|
+
`/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/file?path=${encodeURIComponent(filePath)}`
|
|
5582
|
+
);
|
|
5583
|
+
return { path: res.path, size: res.size, bytes: Buffer.from(res.contentBase64, "base64") };
|
|
5584
|
+
}
|
|
5585
|
+
write(sandboxId, filePath, bytes) {
|
|
5586
|
+
return apiRequest(this.options, "PUT", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/file`, {
|
|
5587
|
+
path: filePath,
|
|
5588
|
+
contentBase64: bytes.toString("base64")
|
|
5589
|
+
});
|
|
5590
|
+
}
|
|
5591
|
+
exec(sandboxId, input) {
|
|
5592
|
+
return apiRequest(this.options, "POST", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/exec`, input);
|
|
5593
|
+
}
|
|
5594
|
+
check(sandboxId) {
|
|
5595
|
+
return apiRequest(this.options, "POST", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/check`, {});
|
|
5596
|
+
}
|
|
5597
|
+
commit(sandboxId, input) {
|
|
5598
|
+
return apiRequest(this.options, "POST", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/commit`, input);
|
|
5599
|
+
}
|
|
5600
|
+
close(sandboxId) {
|
|
5601
|
+
return apiRequest(this.options, "DELETE", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}`);
|
|
5602
|
+
}
|
|
5603
|
+
};
|
|
5604
|
+
}
|
|
5605
|
+
});
|
|
5606
|
+
|
|
5279
5607
|
// src/worlds/index.ts
|
|
5280
5608
|
var worlds_exports = {};
|
|
5281
5609
|
__export(worlds_exports, {
|
|
@@ -5288,6 +5616,7 @@ __export(worlds_exports, {
|
|
|
5288
5616
|
EVENT_FLUSH_INTERVAL_MS: () => EVENT_FLUSH_INTERVAL_MS,
|
|
5289
5617
|
EVENT_MAX_RETRIES: () => EVENT_MAX_RETRIES,
|
|
5290
5618
|
GatewayEvalsClient: () => GatewayEvalsClient,
|
|
5619
|
+
PushSlugConflict: () => PushSlugConflict,
|
|
5291
5620
|
STRIP_PREFIX: () => STRIP_PREFIX,
|
|
5292
5621
|
SURFACE_KINDS: () => SURFACE_KINDS,
|
|
5293
5622
|
TRANSFORMS: () => TRANSFORMS,
|
|
@@ -5295,6 +5624,7 @@ __export(worlds_exports, {
|
|
|
5295
5624
|
TaskSessions: () => TaskSessions,
|
|
5296
5625
|
WHITESPACE: () => WHITESPACE,
|
|
5297
5626
|
WorldBrowser: () => WorldBrowser,
|
|
5627
|
+
WorldSandboxClient: () => WorldSandboxClient,
|
|
5298
5628
|
WorldSession: () => WorldSession,
|
|
5299
5629
|
WorldSessionCallError: () => WorldSessionCallError,
|
|
5300
5630
|
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
@@ -5304,9 +5634,11 @@ __export(worlds_exports, {
|
|
|
5304
5634
|
aggregateTaskGrade: () => aggregateTaskGrade,
|
|
5305
5635
|
apiRequest: () => apiRequest,
|
|
5306
5636
|
applyTransforms: () => applyTransforms,
|
|
5637
|
+
archiveMode: () => archiveMode,
|
|
5307
5638
|
attachManifest: () => attachManifest,
|
|
5308
5639
|
attachSession: () => attachSession,
|
|
5309
5640
|
attachTask: () => attachTask,
|
|
5641
|
+
buildFileManifest: () => buildFileManifest,
|
|
5310
5642
|
buildSourceTarball: () => buildSourceTarball,
|
|
5311
5643
|
captureToolCalls: () => captureToolCalls,
|
|
5312
5644
|
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
@@ -5318,12 +5650,12 @@ __export(worlds_exports, {
|
|
|
5318
5650
|
deleteTask: () => deleteTask,
|
|
5319
5651
|
describeWorld: () => describeWorld,
|
|
5320
5652
|
detectProvenance: () => detectProvenance,
|
|
5653
|
+
extractBundle: () => extractBundle,
|
|
5321
5654
|
getRun: () => getRun,
|
|
5322
5655
|
getTask: () => getTask,
|
|
5323
5656
|
hashForm: () => hashForm,
|
|
5324
5657
|
ingest: () => ingest,
|
|
5325
5658
|
ingestToolCalls: () => ingestToolCalls,
|
|
5326
|
-
isIncludedInContentHash: () => isIncludedInContentHash,
|
|
5327
5659
|
isTaskIdRef: () => isTaskIdRef,
|
|
5328
5660
|
isTaskManifest: () => isTaskManifest,
|
|
5329
5661
|
isTaskSpec: () => isTaskSpec,
|
|
@@ -5339,6 +5671,8 @@ __export(worlds_exports, {
|
|
|
5339
5671
|
openSession: () => openSession,
|
|
5340
5672
|
openTask: () => openTask,
|
|
5341
5673
|
openWorld: () => openWorld,
|
|
5674
|
+
openWorldVersion: () => openWorldVersion,
|
|
5675
|
+
parseSlugRef: () => parseSlugRef,
|
|
5342
5676
|
parseToolCallRecord: () => parseToolCallRecord,
|
|
5343
5677
|
parseTransform: () => parseTransform,
|
|
5344
5678
|
parseWorldTaskRefs: () => parseWorldTaskRefs,
|
|
@@ -5355,6 +5689,7 @@ __export(worlds_exports, {
|
|
|
5355
5689
|
redactActionArgs: () => redactActionArgs,
|
|
5356
5690
|
redactionOf: () => redactionOf,
|
|
5357
5691
|
resolveBrowserMode: () => resolveBrowserMode,
|
|
5692
|
+
resolvePushSlug: () => resolvePushSlug,
|
|
5358
5693
|
roundForm: () => roundForm,
|
|
5359
5694
|
runSessions: () => runSessions,
|
|
5360
5695
|
runTask: () => runTask,
|
|
@@ -5390,9 +5725,9 @@ function resolveEnv(options) {
|
|
|
5390
5725
|
}
|
|
5391
5726
|
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
5392
5727
|
}
|
|
5393
|
-
async function apiRequest(options, method,
|
|
5728
|
+
async function apiRequest(options, method, path10, body) {
|
|
5394
5729
|
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
5395
|
-
const res = await fetch(`${host}${
|
|
5730
|
+
const res = await fetch(`${host}${path10}`, {
|
|
5396
5731
|
method,
|
|
5397
5732
|
headers: {
|
|
5398
5733
|
"content-type": "application/json",
|
|
@@ -5401,7 +5736,7 @@ async function apiRequest(options, method, path9, body) {
|
|
|
5401
5736
|
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
5402
5737
|
});
|
|
5403
5738
|
const text2 = await res.text();
|
|
5404
|
-
if (!res.ok) throw new ApiError2(res.status, method,
|
|
5739
|
+
if (!res.ok) throw new ApiError2(res.status, method, path10, text2.slice(0, 400));
|
|
5405
5740
|
return JSON.parse(text2);
|
|
5406
5741
|
}
|
|
5407
5742
|
async function setEnvSecret(slug, key, value, options = {}) {
|
|
@@ -5530,9 +5865,21 @@ async function openWorld(slugRef, options = {}) {
|
|
|
5530
5865
|
const at = slugRef.indexOf("@");
|
|
5531
5866
|
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
5532
5867
|
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
5868
|
+
return resolveWorld(slugRef, slug, ref, ref ? { ref } : {}, options);
|
|
5869
|
+
}
|
|
5870
|
+
async function openWorldVersion(slug, versionId, options = {}) {
|
|
5871
|
+
return resolveWorld(
|
|
5872
|
+
`${slug}@${versionId}`,
|
|
5873
|
+
slug,
|
|
5874
|
+
null,
|
|
5875
|
+
{ versionId },
|
|
5876
|
+
options
|
|
5877
|
+
);
|
|
5878
|
+
}
|
|
5879
|
+
async function resolveWorld(slugRef, slug, ref, pin, options) {
|
|
5533
5880
|
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
5534
5881
|
slug,
|
|
5535
|
-
...
|
|
5882
|
+
...pin
|
|
5536
5883
|
});
|
|
5537
5884
|
const world = new WorldSet(
|
|
5538
5885
|
slugRef,
|
|
@@ -5570,9 +5917,10 @@ var init_worlds = __esm({
|
|
|
5570
5917
|
init_query();
|
|
5571
5918
|
init_migrate();
|
|
5572
5919
|
init_ingest();
|
|
5920
|
+
init_sandbox();
|
|
5573
5921
|
ApiError2 = class extends Error {
|
|
5574
|
-
constructor(status, method,
|
|
5575
|
-
super(`${method} ${
|
|
5922
|
+
constructor(status, method, path10, body) {
|
|
5923
|
+
super(`${method} ${path10} -> ${status}: ${body}`);
|
|
5576
5924
|
this.status = status;
|
|
5577
5925
|
this.name = "ApiError";
|
|
5578
5926
|
}
|
|
@@ -5653,6 +6001,7 @@ init_worlds();
|
|
|
5653
6001
|
EVENT_FLUSH_INTERVAL_MS,
|
|
5654
6002
|
EVENT_MAX_RETRIES,
|
|
5655
6003
|
GatewayEvalsClient,
|
|
6004
|
+
PushSlugConflict,
|
|
5656
6005
|
STRIP_PREFIX,
|
|
5657
6006
|
SURFACE_KINDS,
|
|
5658
6007
|
TRANSFORMS,
|
|
@@ -5660,6 +6009,7 @@ init_worlds();
|
|
|
5660
6009
|
TaskSessions,
|
|
5661
6010
|
WHITESPACE,
|
|
5662
6011
|
WorldBrowser,
|
|
6012
|
+
WorldSandboxClient,
|
|
5663
6013
|
WorldSession,
|
|
5664
6014
|
WorldSessionCallError,
|
|
5665
6015
|
WorldSessionTimeout,
|
|
@@ -5669,9 +6019,11 @@ init_worlds();
|
|
|
5669
6019
|
aggregateTaskGrade,
|
|
5670
6020
|
apiRequest,
|
|
5671
6021
|
applyTransforms,
|
|
6022
|
+
archiveMode,
|
|
5672
6023
|
attachManifest,
|
|
5673
6024
|
attachSession,
|
|
5674
6025
|
attachTask,
|
|
6026
|
+
buildFileManifest,
|
|
5675
6027
|
buildSourceTarball,
|
|
5676
6028
|
captureToolCalls,
|
|
5677
6029
|
collectArchiveRelPaths,
|
|
@@ -5683,12 +6035,12 @@ init_worlds();
|
|
|
5683
6035
|
deleteTask,
|
|
5684
6036
|
describeWorld,
|
|
5685
6037
|
detectProvenance,
|
|
6038
|
+
extractBundle,
|
|
5686
6039
|
getRun,
|
|
5687
6040
|
getTask,
|
|
5688
6041
|
hashForm,
|
|
5689
6042
|
ingest,
|
|
5690
6043
|
ingestToolCalls,
|
|
5691
|
-
isIncludedInContentHash,
|
|
5692
6044
|
isTaskIdRef,
|
|
5693
6045
|
isTaskManifest,
|
|
5694
6046
|
isTaskSpec,
|
|
@@ -5704,6 +6056,8 @@ init_worlds();
|
|
|
5704
6056
|
openSession,
|
|
5705
6057
|
openTask,
|
|
5706
6058
|
openWorld,
|
|
6059
|
+
openWorldVersion,
|
|
6060
|
+
parseSlugRef,
|
|
5707
6061
|
parseToolCallRecord,
|
|
5708
6062
|
parseTransform,
|
|
5709
6063
|
parseWorldTaskRefs,
|
|
@@ -5720,6 +6074,7 @@ init_worlds();
|
|
|
5720
6074
|
redactActionArgs,
|
|
5721
6075
|
redactionOf,
|
|
5722
6076
|
resolveBrowserMode,
|
|
6077
|
+
resolvePushSlug,
|
|
5723
6078
|
roundForm,
|
|
5724
6079
|
runSessions,
|
|
5725
6080
|
runTask,
|