@withgateway/sdk 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +4304 -2381
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4266 -2343
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +56 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +52 -27
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +280 -8
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +272 -8
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
- package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
- package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
- package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1100 -438
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1079 -424
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +45 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +41 -16
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +98 -61
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +98 -61
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +179 -25
- package/dist/worlds/index.d.ts +179 -25
- package/dist/worlds/index.js +639 -284
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +631 -276
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +2 -1
- package/runtime/gatewaysdk/environment/_bundle.py +15 -25
- package/runtime/gatewaysdk/environment/_materialize.py +53 -9
- package/runtime/gatewaysdk/environment/_toml.py +13 -0
- package/runtime/gatewaysdk/environment/schema/api.py +40 -25
- package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
- package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +156 -546
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/platform.py +30 -26
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +67 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
- package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
- package/runtime/gatewaysdk/environment/schema/tools_world.py +71 -23
- package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +507 -0
- package/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/skills/connector-schema-authoring/connector-worlds.md +8 -1
- package/skills/connector-schema-authoring/world-schema.md +1 -1
- package/skills/hooks-getting-started/SKILL.md +67 -6
- package/skills/world-data-ingestion/SKILL.md +4 -1
- package/skills/worlds-getting-started/SKILL.md +11 -11
- package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
package/dist/worlds/index.mjs
CHANGED
|
@@ -27,9 +27,9 @@ function surfaceKinds(s) {
|
|
|
27
27
|
if (s.browser) kinds.push("browser");
|
|
28
28
|
return kinds;
|
|
29
29
|
}
|
|
30
|
-
function uiUrl(ui,
|
|
30
|
+
function uiUrl(ui, path10) {
|
|
31
31
|
const base = ui.url.replace(/\/+$/, "");
|
|
32
|
-
const rel =
|
|
32
|
+
const rel = path10.startsWith("/") ? path10 : `/${path10}`;
|
|
33
33
|
if (!ui.token) return base + rel;
|
|
34
34
|
const [beforeHash, hash] = splitHash(rel);
|
|
35
35
|
const joiner = beforeHash.includes("?") ? "&" : "?";
|
|
@@ -587,12 +587,13 @@ async function openSession(slugOrRef, taskOrOpts, positionalOpts = {}) {
|
|
|
587
587
|
);
|
|
588
588
|
const at = slugOrRef.indexOf("@");
|
|
589
589
|
const slug = at === -1 ? slugOrRef : slugOrRef.slice(0, at);
|
|
590
|
+
const ref = at === -1 ? "" : slugOrRef.slice(at + 1);
|
|
590
591
|
const state2 = await apiRequest(
|
|
591
592
|
options,
|
|
592
593
|
"POST",
|
|
593
594
|
"/api/public/world-sessions",
|
|
594
595
|
{
|
|
595
|
-
...opts.versionId ? { versionId: opts.versionId } : { slug },
|
|
596
|
+
...opts.versionId ? { versionId: opts.versionId } : { slug, ...ref && { ref } },
|
|
596
597
|
task,
|
|
597
598
|
...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm },
|
|
598
599
|
...opts.surfaces?.length ? { surfaces: opts.surfaces } : {}
|
|
@@ -606,8 +607,8 @@ async function runSessions(opts) {
|
|
|
606
607
|
let tasks = opts.tasks;
|
|
607
608
|
let versionId = opts.versionId;
|
|
608
609
|
if (!tasks || !versionId) {
|
|
609
|
-
const { openWorld: openWorld2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
|
|
610
|
-
const world = await openWorld2(opts.world, options);
|
|
610
|
+
const { openWorld: openWorld2, openWorldVersion: openWorldVersion2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
|
|
611
|
+
const world = versionId ? await openWorldVersion2(opts.world.split("@", 1)[0], versionId, options) : await openWorld2(opts.world, options);
|
|
611
612
|
tasks = tasks ?? world.tasks.map((t) => t.name);
|
|
612
613
|
versionId = versionId ?? world.version.versionId;
|
|
613
614
|
}
|
|
@@ -663,10 +664,12 @@ async function runSessions(opts) {
|
|
|
663
664
|
})
|
|
664
665
|
);
|
|
665
666
|
const mean = results.reduce((sum, r) => sum + (r.reward ?? 0), 0) / (results.length || 1);
|
|
667
|
+
const failed = results.filter((r) => r.error).length;
|
|
666
668
|
return {
|
|
667
669
|
results: results.sort((a, b) => a.task.localeCompare(b.task)),
|
|
668
670
|
mean,
|
|
669
|
-
|
|
671
|
+
failed,
|
|
672
|
+
passed: failed === 0 && (opts.failUnder === void 0 || mean >= opts.failUnder)
|
|
670
673
|
};
|
|
671
674
|
}
|
|
672
675
|
var isTaskIdRef, isTaskSpec, taskLabel, READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, JUDGED_GRADE_TIMEOUT_MS, JUDGED_GRADE_POLL_MS, EXPORT_PAGE, sleep, modelToolName, WorldSessionTimeout, WorldSessionCallError, WorldSession;
|
|
@@ -790,18 +793,18 @@ var init_session = __esm({
|
|
|
790
793
|
}
|
|
791
794
|
/** The reporter `browser()` uses; exposed for a page you drive yourself. */
|
|
792
795
|
browserRecorder(agentId) {
|
|
793
|
-
const
|
|
796
|
+
const path10 = `/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/browser-events`;
|
|
794
797
|
const id = agentId ?? process.env.GATEWAY_AGENT_ID ?? `agent-${randomUUID().replace(/-/g, "").slice(0, 8)}`;
|
|
795
798
|
return new BrowserActionRecorder(
|
|
796
799
|
async (events) => {
|
|
797
|
-
await apiRequest(this.options, "POST",
|
|
800
|
+
await apiRequest(this.options, "POST", path10, { events });
|
|
798
801
|
},
|
|
799
802
|
id,
|
|
800
803
|
{
|
|
801
804
|
// Continue this agent's timeline where the platform has it, so the
|
|
802
805
|
// same id reopened later never re-numbers from zero.
|
|
803
806
|
resumeFrom: async () => {
|
|
804
|
-
const summary = await apiRequest(this.options, "GET",
|
|
807
|
+
const summary = await apiRequest(this.options, "GET", path10);
|
|
805
808
|
return summary.agents.find((a) => a.agentId === id)?.nextSeq ?? 0;
|
|
806
809
|
}
|
|
807
810
|
}
|
|
@@ -1607,10 +1610,10 @@ var init_users = __esm({
|
|
|
1607
1610
|
totalItems: Number(meta.totalItems ?? rows.length)
|
|
1608
1611
|
};
|
|
1609
1612
|
}
|
|
1610
|
-
async requestJson(method,
|
|
1613
|
+
async requestJson(method, path10, payload) {
|
|
1611
1614
|
let response;
|
|
1612
1615
|
try {
|
|
1613
|
-
response = await fetch(`${this.host}${
|
|
1616
|
+
response = await fetch(`${this.host}${path10}`, {
|
|
1614
1617
|
method,
|
|
1615
1618
|
headers: {
|
|
1616
1619
|
Authorization: this.authHeader(),
|
|
@@ -1621,17 +1624,17 @@ var init_users = __esm({
|
|
|
1621
1624
|
});
|
|
1622
1625
|
} catch (error) {
|
|
1623
1626
|
throw new UserProfileError(
|
|
1624
|
-
`${method} ${
|
|
1627
|
+
`${method} ${path10} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1625
1628
|
{ cause: error }
|
|
1626
1629
|
);
|
|
1627
1630
|
}
|
|
1628
1631
|
const text2 = await response.text();
|
|
1629
1632
|
if (!response.ok) {
|
|
1630
1633
|
throw new UserProfileError(
|
|
1631
|
-
`${method} ${
|
|
1634
|
+
`${method} ${path10} failed (${response.status}): ${text2.slice(0, 500)}`
|
|
1632
1635
|
);
|
|
1633
1636
|
}
|
|
1634
|
-
this.log("%s %s -> %s", method,
|
|
1637
|
+
this.log("%s %s -> %s", method, path10, response.status);
|
|
1635
1638
|
return safeJson(text2);
|
|
1636
1639
|
}
|
|
1637
1640
|
authHeader() {
|
|
@@ -1895,10 +1898,10 @@ var init_scores = __esm({
|
|
|
1895
1898
|
metadata: signalMetadata
|
|
1896
1899
|
});
|
|
1897
1900
|
}
|
|
1898
|
-
async request(method,
|
|
1901
|
+
async request(method, path10, payload) {
|
|
1899
1902
|
let response;
|
|
1900
1903
|
try {
|
|
1901
|
-
response = await fetch(`${this.host}${
|
|
1904
|
+
response = await fetch(`${this.host}${path10}`, {
|
|
1902
1905
|
method,
|
|
1903
1906
|
headers: {
|
|
1904
1907
|
Authorization: this.authHeader(),
|
|
@@ -1909,17 +1912,17 @@ var init_scores = __esm({
|
|
|
1909
1912
|
});
|
|
1910
1913
|
} catch (error) {
|
|
1911
1914
|
throw new ScoreError(
|
|
1912
|
-
`${method} ${
|
|
1915
|
+
`${method} ${path10} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1913
1916
|
{ cause: error }
|
|
1914
1917
|
);
|
|
1915
1918
|
}
|
|
1916
1919
|
const text2 = await response.text();
|
|
1917
1920
|
if (!response.ok) {
|
|
1918
1921
|
throw new ScoreError(
|
|
1919
|
-
`${method} ${
|
|
1922
|
+
`${method} ${path10} failed (${response.status}): ${text2.slice(0, 500)}`
|
|
1920
1923
|
);
|
|
1921
1924
|
}
|
|
1922
|
-
this.log("%s %s -> %s", method,
|
|
1925
|
+
this.log("%s %s -> %s", method, path10, response.status);
|
|
1923
1926
|
}
|
|
1924
1927
|
authHeader() {
|
|
1925
1928
|
return `Basic ${Buffer.from(`${this.publicKey}:${this.secretKey}`).toString(
|
|
@@ -1947,7 +1950,7 @@ var require_package = __commonJS({
|
|
|
1947
1950
|
"package.json"(exports, module) {
|
|
1948
1951
|
module.exports = {
|
|
1949
1952
|
name: "@withgateway/sdk",
|
|
1950
|
-
version: "0.2.
|
|
1953
|
+
version: "0.2.6",
|
|
1951
1954
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
1952
1955
|
main: "./dist/index.js",
|
|
1953
1956
|
module: "./dist/index.mjs",
|
|
@@ -2051,6 +2054,7 @@ var require_package = __commonJS({
|
|
|
2051
2054
|
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
2052
2055
|
test: "vitest",
|
|
2053
2056
|
"test:run": "vitest run",
|
|
2057
|
+
"test:integration": "vitest run --config vitest.integration.config.ts",
|
|
2054
2058
|
lint: "eslint src --ext .ts",
|
|
2055
2059
|
clean: "rm -rf dist",
|
|
2056
2060
|
example: "tsx examples/test-ab-experiment.ts",
|
|
@@ -2682,7 +2686,7 @@ async function fileTaskRun(live, runId, model, report, grade, agentError, traced
|
|
|
2682
2686
|
task: live.name ?? live.taskId ?? runId,
|
|
2683
2687
|
rolloutIndex: 0,
|
|
2684
2688
|
prompt: live.instruction,
|
|
2685
|
-
completion: report,
|
|
2689
|
+
...report === null ? {} : { completion: report },
|
|
2686
2690
|
rewards: grade.rewards,
|
|
2687
2691
|
info: {
|
|
2688
2692
|
sessionId: runId,
|
|
@@ -2720,17 +2724,20 @@ async function fileTaskRun(live, runId, model, report, grade, agentError, traced
|
|
|
2720
2724
|
return evaluationId;
|
|
2721
2725
|
}
|
|
2722
2726
|
function reportFromResult(result) {
|
|
2727
|
+
if (result === void 0 || result === null)
|
|
2728
|
+
return { report: null, extra: {} };
|
|
2723
2729
|
if (typeof result === "string") return { report: result, extra: {} };
|
|
2724
|
-
if (
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2730
|
+
if (typeof result !== "object" || Array.isArray(result))
|
|
2731
|
+
throw new Error(
|
|
2732
|
+
`agent returned ${describeValue(result)}; ${RESULT_CONTRACT}`
|
|
2733
|
+
);
|
|
2734
|
+
const { report, ...extra } = result;
|
|
2735
|
+
if (report === void 0 || report === null) return { report: null, extra };
|
|
2736
|
+
if (typeof report !== "string")
|
|
2737
|
+
throw new Error(
|
|
2738
|
+
`agent returned a report of type ${describeValue(report)}; ${RESULT_CONTRACT}`
|
|
2739
|
+
);
|
|
2740
|
+
return { report, extra };
|
|
2734
2741
|
}
|
|
2735
2742
|
async function runTask(ref, agent, opts) {
|
|
2736
2743
|
const runId = `wtr-${randomBytes(6).toString("hex")}`;
|
|
@@ -2834,7 +2841,7 @@ async function runTask(ref, agent, opts) {
|
|
|
2834
2841
|
}
|
|
2835
2842
|
}
|
|
2836
2843
|
}
|
|
2837
|
-
var TaskRun;
|
|
2844
|
+
var TaskRun, describeValue, RESULT_CONTRACT;
|
|
2838
2845
|
var init_taskRun = __esm({
|
|
2839
2846
|
"src/worlds/taskRun.ts"() {
|
|
2840
2847
|
"use strict";
|
|
@@ -2905,6 +2912,154 @@ var init_taskRun = __esm({
|
|
|
2905
2912
|
Object.assign(this.notes, notes);
|
|
2906
2913
|
}
|
|
2907
2914
|
};
|
|
2915
|
+
describeValue = (value) => Array.isArray(value) ? "array" : value === null ? "null" : typeof value;
|
|
2916
|
+
RESULT_CONTRACT = "return a string report, an object {report?: string, ...} or nothing";
|
|
2917
|
+
}
|
|
2918
|
+
});
|
|
2919
|
+
|
|
2920
|
+
// src/worlds/worldDir.ts
|
|
2921
|
+
import { createHash as createHash2 } from "crypto";
|
|
2922
|
+
import * as fs2 from "fs";
|
|
2923
|
+
import * as path2 from "path";
|
|
2924
|
+
var WORLD_MANIFEST, API_BUNDLE_MANIFEST, BUNDLED_WORLD_PREFIX, CONNECTOR_SPEC, PYPROJECT, WORLD_SCHEMA_MARKERS, WORLD_ROOT_MARKERS, BROAD_ROOT_MARKERS, SKIPPED_DIRS, V1_DEP, V1_IMPORT, V1_TASKSET, HARBOR_TASK, skippedDir, byCodePoint, WorldDir;
|
|
2925
|
+
var init_worldDir = __esm({
|
|
2926
|
+
"src/worlds/worldDir.ts"() {
|
|
2927
|
+
"use strict";
|
|
2928
|
+
WORLD_MANIFEST = "gateway-env.toml";
|
|
2929
|
+
API_BUNDLE_MANIFEST = "gateway-api-world.toml";
|
|
2930
|
+
BUNDLED_WORLD_PREFIX = "server/world/";
|
|
2931
|
+
CONNECTOR_SPEC = "connector.toml";
|
|
2932
|
+
PYPROJECT = "pyproject.toml";
|
|
2933
|
+
WORLD_SCHEMA_MARKERS = ["schema/world.json", "schema/runtime.json", "schema/lock.json"];
|
|
2934
|
+
WORLD_ROOT_MARKERS = [CONNECTOR_SPEC, ...WORLD_SCHEMA_MARKERS];
|
|
2935
|
+
BROAD_ROOT_MARKERS = [WORLD_MANIFEST, API_BUNDLE_MANIFEST];
|
|
2936
|
+
SKIPPED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
2937
|
+
V1_DEP = /["']verifiers\s*(?:\[[^\]]*\])?\s*(?:>=|~=|==|>)\s*(?:0\.(?:[2-9]\d*|[1-9]\d+)|[1-9]\d*)/;
|
|
2938
|
+
V1_IMPORT = /^\s*(?:from\s+verifiers\.v1\s+import\b|from\s+verifiers\s+import\s+v1\b|import\s+verifiers\.v1\b)/m;
|
|
2939
|
+
V1_TASKSET = /class\s+\w+\s*\(\s*(?:(?:vf|v1|verifiers(?:\.v1)?)\.)?Taskset\s*[,)]/;
|
|
2940
|
+
HARBOR_TASK = /^(?:tasks\/)?([^/]+)\/task\.toml$/;
|
|
2941
|
+
skippedDir = (name) => name.startsWith(".") || SKIPPED_DIRS.has(name) || name.endsWith(".egg-info");
|
|
2942
|
+
byCodePoint = (a, b) => Buffer.compare(Buffer.from(a, "utf-8"), Buffer.from(b, "utf-8"));
|
|
2943
|
+
WorldDir = class _WorldDir {
|
|
2944
|
+
constructor(dir, worldRoot) {
|
|
2945
|
+
this.dir = dir;
|
|
2946
|
+
this.worldRoot = worldRoot;
|
|
2947
|
+
}
|
|
2948
|
+
pathsMemo;
|
|
2949
|
+
static open(dir) {
|
|
2950
|
+
const present = (rel) => fs2.existsSync(path2.join(dir, rel));
|
|
2951
|
+
const bundled = present(API_BUNDLE_MANIFEST) && WORLD_ROOT_MARKERS.some((m) => present(BUNDLED_WORLD_PREFIX + m));
|
|
2952
|
+
return new _WorldDir(dir, bundled ? BUNDLED_WORLD_PREFIX : "");
|
|
2953
|
+
}
|
|
2954
|
+
// ── Files ──────────────────────────────────────────────────────────────
|
|
2955
|
+
has(rel) {
|
|
2956
|
+
return fs2.existsSync(path2.join(this.dir, rel));
|
|
2957
|
+
}
|
|
2958
|
+
text(rel) {
|
|
2959
|
+
try {
|
|
2960
|
+
return fs2.readFileSync(path2.join(this.dir, rel), "utf-8");
|
|
2961
|
+
} catch {
|
|
2962
|
+
return null;
|
|
2963
|
+
}
|
|
2964
|
+
}
|
|
2965
|
+
/** A path under the authored world's root. */
|
|
2966
|
+
worldPath(rel) {
|
|
2967
|
+
return this.worldRoot + rel;
|
|
2968
|
+
}
|
|
2969
|
+
/** Every regular file the archive rules keep (`_archive.py`: no hidden,
|
|
2970
|
+
* dist, build, outputs, pycache or egg-info directories), as sorted
|
|
2971
|
+
* relpaths. Root files enter only when a Gateway manifest widens the root
|
|
2972
|
+
* or they are README.md, pyproject.toml or Python. */
|
|
2973
|
+
get archivePaths() {
|
|
2974
|
+
if (this.pathsMemo) return this.pathsMemo;
|
|
2975
|
+
const broad = BROAD_ROOT_MARKERS.some((m) => this.has(m));
|
|
2976
|
+
const out = [];
|
|
2977
|
+
const walk = (rel) => {
|
|
2978
|
+
for (const entry of fs2.readdirSync(rel ? path2.join(this.dir, rel) : this.dir, { withFileTypes: true })) {
|
|
2979
|
+
const child = rel ? `${rel}/${entry.name}` : entry.name;
|
|
2980
|
+
if (entry.isDirectory()) {
|
|
2981
|
+
if (!skippedDir(entry.name)) walk(child);
|
|
2982
|
+
} else if (entry.isFile() && !entry.name.startsWith(".")) {
|
|
2983
|
+
const rootKept = broad || ["README.md", PYPROJECT].includes(entry.name) || entry.name.endsWith(".py");
|
|
2984
|
+
if (rel || rootKept) out.push(child);
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
};
|
|
2988
|
+
walk("");
|
|
2989
|
+
return this.pathsMemo = out.sort(byCodePoint);
|
|
2990
|
+
}
|
|
2991
|
+
/** A root `.gitignore` is present: the Python SDK honours it when
|
|
2992
|
+
* collecting, this package does not reimplement gitignore. */
|
|
2993
|
+
get hasRootGitignore() {
|
|
2994
|
+
return this.has(".gitignore");
|
|
2995
|
+
}
|
|
2996
|
+
/** SHA-256 over `utf8("file:" + relpath)` + bytes per archive path in
|
|
2997
|
+
* sorted order: the version's identity, byte-compatible with
|
|
2998
|
+
* `_archive.compute_content_hash` and Prime Intellect. */
|
|
2999
|
+
get contentHash() {
|
|
3000
|
+
const hasher = createHash2("sha256");
|
|
3001
|
+
for (const rel of this.archivePaths) {
|
|
3002
|
+
hasher.update(Buffer.from(`file:${rel}`, "utf-8"));
|
|
3003
|
+
hasher.update(fs2.readFileSync(path2.join(this.dir, rel)));
|
|
3004
|
+
}
|
|
3005
|
+
return hasher.digest("hex");
|
|
3006
|
+
}
|
|
3007
|
+
// ── Markers ────────────────────────────────────────────────────────────
|
|
3008
|
+
get isApiBundle() {
|
|
3009
|
+
return this.has(API_BUNDLE_MANIFEST);
|
|
3010
|
+
}
|
|
3011
|
+
get hasManifest() {
|
|
3012
|
+
return this.has(this.worldPath(WORLD_MANIFEST));
|
|
3013
|
+
}
|
|
3014
|
+
get hasSchemaMarker() {
|
|
3015
|
+
return WORLD_SCHEMA_MARKERS.some((m) => this.has(this.worldPath(m)));
|
|
3016
|
+
}
|
|
3017
|
+
/** Harbor task dirs, relative to the tree: `tasks/<name>` when a `tasks/`
|
|
3018
|
+
* dir holds them, else the root `<name>` dirs. */
|
|
3019
|
+
taskDirs() {
|
|
3020
|
+
const names = /* @__PURE__ */ new Set();
|
|
3021
|
+
for (const rel of this.archivePaths) {
|
|
3022
|
+
const m = HARBOR_TASK.exec(rel);
|
|
3023
|
+
if (m) names.add(rel.startsWith("tasks/") ? `tasks/${m[1]}` : m[1]);
|
|
3024
|
+
}
|
|
3025
|
+
const under = [...names].filter((n) => n.startsWith("tasks/"));
|
|
3026
|
+
return (under.length ? under : [...names]).sort();
|
|
3027
|
+
}
|
|
3028
|
+
/** The Python package of a verifiers tree (`<pkg>/__init__.py` or
|
|
3029
|
+
* `src/<pkg>/__init__.py`), or null. */
|
|
3030
|
+
pythonPackage() {
|
|
3031
|
+
if (!this.has(PYPROJECT)) return null;
|
|
3032
|
+
for (const base of ["", "src/"]) {
|
|
3033
|
+
const init2 = this.archivePaths.find((rel) => {
|
|
3034
|
+
const m = /^([^/]+)\/__init__\.py$/.exec(rel.startsWith(base) ? rel.slice(base.length) : "");
|
|
3035
|
+
return m !== null && m[1] !== "tests";
|
|
3036
|
+
});
|
|
3037
|
+
if (init2) return { name: init2.slice(base.length).split("/")[0], init: init2 };
|
|
3038
|
+
}
|
|
3039
|
+
return null;
|
|
3040
|
+
}
|
|
3041
|
+
/** The package targets the `verifiers.v1` API: read from the root
|
|
3042
|
+
* pyproject and every archived Python module. */
|
|
3043
|
+
get isVerifiersV1() {
|
|
3044
|
+
for (const rel of this.archivePaths) {
|
|
3045
|
+
if (rel !== PYPROJECT && !rel.endsWith(".py")) continue;
|
|
3046
|
+
const text2 = this.text(rel);
|
|
3047
|
+
if (text2 === null) continue;
|
|
3048
|
+
if (rel === PYPROJECT ? V1_DEP.test(text2) : V1_IMPORT.test(text2) || text2.includes("__all__") && V1_TASKSET.test(text2))
|
|
3049
|
+
return true;
|
|
3050
|
+
}
|
|
3051
|
+
return false;
|
|
3052
|
+
}
|
|
3053
|
+
// ── The answer ─────────────────────────────────────────────────────────
|
|
3054
|
+
/** The runner's kind (`detect_env_kind`): api / mock-tools win over
|
|
3055
|
+
* everything, a harbor task tree next, then the verifiers API version. */
|
|
3056
|
+
get kind() {
|
|
3057
|
+
if (this.isApiBundle) return "api";
|
|
3058
|
+
if (this.has(WORLD_MANIFEST)) return "mock-tools";
|
|
3059
|
+
if (this.taskDirs().length) return "harbor";
|
|
3060
|
+
return this.isVerifiersV1 ? "verifiers-v1" : "verifiers";
|
|
3061
|
+
}
|
|
3062
|
+
};
|
|
2908
3063
|
}
|
|
2909
3064
|
});
|
|
2910
3065
|
|
|
@@ -2912,79 +3067,54 @@ var init_taskRun = __esm({
|
|
|
2912
3067
|
var hub_exports = {};
|
|
2913
3068
|
__export(hub_exports, {
|
|
2914
3069
|
DEFAULT_BRANCH: () => DEFAULT_BRANCH,
|
|
3070
|
+
PushSlugConflict: () => PushSlugConflict,
|
|
3071
|
+
archiveMode: () => archiveMode,
|
|
3072
|
+
buildFileManifest: () => buildFileManifest,
|
|
2915
3073
|
buildSourceTarball: () => buildSourceTarball,
|
|
2916
3074
|
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
2917
3075
|
computeContentHash: () => computeContentHash,
|
|
2918
3076
|
createRef: () => createRef,
|
|
2919
|
-
|
|
3077
|
+
extractBundle: () => extractBundle,
|
|
2920
3078
|
localTreeState: () => localTreeState,
|
|
2921
3079
|
propose: () => propose,
|
|
2922
3080
|
pull: () => pull,
|
|
2923
3081
|
push: () => push,
|
|
2924
3082
|
pushSlug: () => pushSlug,
|
|
2925
3083
|
readLocalMeta: () => readLocalMeta,
|
|
3084
|
+
resolvePushSlug: () => resolvePushSlug,
|
|
2926
3085
|
snapshotTreeHashes: () => snapshotTreeHashes,
|
|
2927
3086
|
writeLocalMeta: () => writeLocalMeta
|
|
2928
3087
|
});
|
|
2929
|
-
import { createHash as
|
|
3088
|
+
import { createHash as createHash3 } from "crypto";
|
|
2930
3089
|
import { gzipSync, gunzipSync } from "zlib";
|
|
2931
|
-
import * as
|
|
2932
|
-
import * as
|
|
2933
|
-
function isExcludedDirComponent(component) {
|
|
2934
|
-
return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
|
|
2935
|
-
}
|
|
2936
|
-
function isIncludedInContentHash(relPath, includeAllRootFiles) {
|
|
2937
|
-
const parts = relPath.split("/");
|
|
2938
|
-
const name = parts[parts.length - 1];
|
|
2939
|
-
const dirParts = parts.slice(0, -1);
|
|
2940
|
-
if (name.startsWith(".")) return false;
|
|
2941
|
-
if (parts.includes("__pycache__")) return false;
|
|
2942
|
-
if (dirParts.length === 0) {
|
|
2943
|
-
return includeAllRootFiles || name === "README.md" || name === "pyproject.toml" || BROAD_ROOT_MARKERS.includes(name) || name.endsWith(".py");
|
|
2944
|
-
}
|
|
2945
|
-
return !dirParts.some(isExcludedDirComponent);
|
|
2946
|
-
}
|
|
3090
|
+
import * as fs3 from "fs";
|
|
3091
|
+
import * as path3 from "path";
|
|
2947
3092
|
function collectArchiveRelPaths(dir) {
|
|
2948
|
-
|
|
3093
|
+
const world = WorldDir.open(dir);
|
|
3094
|
+
if (world.hasRootGitignore) {
|
|
2949
3095
|
throw new Error(
|
|
2950
3096
|
"This bundle has a root .gitignore, which the Python CLI honors via gitignore_parser. Reimplementing that imperfectly here would silently produce a different content hash, so the TS CLI refuses. Remove the .gitignore (or bake its effect into the tree) and retry."
|
|
2951
3097
|
);
|
|
2952
3098
|
}
|
|
2953
|
-
|
|
2954
|
-
const includeAllRootFiles = BROAD_ROOT_MARKERS.some(
|
|
2955
|
-
(marker) => fs2.existsSync(path2.join(dir, marker))
|
|
2956
|
-
);
|
|
2957
|
-
const walk = (rel) => {
|
|
2958
|
-
const abs = rel ? path2.join(dir, rel) : dir;
|
|
2959
|
-
for (const entry of fs2.readdirSync(abs, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
2960
|
-
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
2961
|
-
if (entry.isDirectory()) {
|
|
2962
|
-
if (!isExcludedDirComponent(entry.name)) walk(childRel);
|
|
2963
|
-
} else if (entry.isFile()) {
|
|
2964
|
-
if (isIncludedInContentHash(childRel, includeAllRootFiles)) {
|
|
2965
|
-
out.push(childRel);
|
|
2966
|
-
}
|
|
2967
|
-
}
|
|
2968
|
-
}
|
|
2969
|
-
};
|
|
2970
|
-
walk("");
|
|
2971
|
-
return out.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
3099
|
+
return world.archivePaths;
|
|
2972
3100
|
}
|
|
2973
3101
|
function computeContentHash(dir) {
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
hasher.update(Buffer.from(`file:${relPath}`, "utf-8"));
|
|
2977
|
-
hasher.update(fs2.readFileSync(path2.join(dir, relPath)));
|
|
2978
|
-
}
|
|
2979
|
-
return hasher.digest("hex");
|
|
3102
|
+
collectArchiveRelPaths(dir);
|
|
3103
|
+
return WorldDir.open(dir).contentHash;
|
|
2980
3104
|
}
|
|
2981
3105
|
function octal(value, length) {
|
|
2982
|
-
return Buffer.from(
|
|
3106
|
+
return Buffer.from(
|
|
3107
|
+
value.toString(8).padStart(length - 1, "0") + "\0",
|
|
3108
|
+
"ascii"
|
|
3109
|
+
);
|
|
3110
|
+
}
|
|
3111
|
+
function archiveMode(mode) {
|
|
3112
|
+
return (mode & 73) !== 0 ? EXEC_MODE : FILE_MODE;
|
|
2983
3113
|
}
|
|
2984
|
-
function tarHeader(name, size) {
|
|
3114
|
+
function tarHeader(name, size, mode = FILE_MODE) {
|
|
2985
3115
|
const header = Buffer.alloc(512);
|
|
2986
3116
|
header.write(name, 0, 100, "utf-8");
|
|
2987
|
-
octal(
|
|
3117
|
+
octal(mode, 8).copy(header, 100);
|
|
2988
3118
|
octal(0, 8).copy(header, 108);
|
|
2989
3119
|
octal(0, 8).copy(header, 116);
|
|
2990
3120
|
octal(size, 12).copy(header, 124);
|
|
@@ -2998,7 +3128,7 @@ function tarHeader(name, size) {
|
|
|
2998
3128
|
header.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
|
|
2999
3129
|
return header;
|
|
3000
3130
|
}
|
|
3001
|
-
function tarMember(name, payload) {
|
|
3131
|
+
function tarMember(name, payload, mode) {
|
|
3002
3132
|
const parts = [];
|
|
3003
3133
|
const nameBytes = Buffer.byteLength(name, "utf-8");
|
|
3004
3134
|
if (nameBytes > 100) {
|
|
@@ -3012,9 +3142,9 @@ function tarMember(name, payload) {
|
|
|
3012
3142
|
parts.push(lHeader, body);
|
|
3013
3143
|
const pad2 = body.length % 512 === 0 ? 0 : 512 - body.length % 512;
|
|
3014
3144
|
if (pad2) parts.push(Buffer.alloc(pad2));
|
|
3015
|
-
parts.push(tarHeader(name.slice(0, 100), payload.length));
|
|
3145
|
+
parts.push(tarHeader(name.slice(0, 100), payload.length, mode));
|
|
3016
3146
|
} else {
|
|
3017
|
-
parts.push(tarHeader(name, payload.length));
|
|
3147
|
+
parts.push(tarHeader(name, payload.length, mode));
|
|
3018
3148
|
}
|
|
3019
3149
|
parts.push(payload);
|
|
3020
3150
|
const pad = payload.length % 512 === 0 ? 0 : 512 - payload.length % 512;
|
|
@@ -3024,14 +3154,21 @@ function tarMember(name, payload) {
|
|
|
3024
3154
|
function buildSourceTarball(dir) {
|
|
3025
3155
|
const parts = [];
|
|
3026
3156
|
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
3027
|
-
|
|
3157
|
+
const abs = path3.join(dir, relPath);
|
|
3158
|
+
parts.push(
|
|
3159
|
+
...tarMember(
|
|
3160
|
+
relPath,
|
|
3161
|
+
fs3.readFileSync(abs),
|
|
3162
|
+
archiveMode(fs3.statSync(abs).mode)
|
|
3163
|
+
)
|
|
3164
|
+
);
|
|
3028
3165
|
}
|
|
3029
3166
|
parts.push(Buffer.alloc(1024));
|
|
3030
3167
|
return gzipSync(Buffer.concat(parts), { level: 9 });
|
|
3031
3168
|
}
|
|
3032
3169
|
function readLocalMeta(dir) {
|
|
3033
3170
|
try {
|
|
3034
|
-
const raw =
|
|
3171
|
+
const raw = fs3.readFileSync(path3.join(dir, META_RELPATH), "utf-8");
|
|
3035
3172
|
const parsed = JSON.parse(raw);
|
|
3036
3173
|
return parsed && typeof parsed === "object" ? parsed : null;
|
|
3037
3174
|
} catch {
|
|
@@ -3039,19 +3176,31 @@ function readLocalMeta(dir) {
|
|
|
3039
3176
|
}
|
|
3040
3177
|
}
|
|
3041
3178
|
function pushSlug(dir, explicit) {
|
|
3042
|
-
|
|
3179
|
+
return resolvePushSlug(dir, { explicit }).slug;
|
|
3180
|
+
}
|
|
3181
|
+
function resolvePushSlug(dir, opts = {}) {
|
|
3043
3182
|
const meta = readLocalMeta(dir);
|
|
3044
|
-
|
|
3045
|
-
|
|
3183
|
+
const tracked = typeof meta?.slug === "string" && meta.slug ? meta.slug : null;
|
|
3184
|
+
const declared = Object.entries({ ...opts.declared ?? {}, "pyproject.toml": readPyproject(dir).name }).filter((entry) => typeof entry[1] === "string" && entry[1] !== "").map(([source, slug]) => ({ source, slug }));
|
|
3185
|
+
const base = { tracked, declared, conflict: null };
|
|
3186
|
+
if (opts.explicit) return { ...base, slug: opts.explicit, source: "--slug" };
|
|
3187
|
+
if (tracked) {
|
|
3188
|
+
const minted = (d) => tracked.startsWith(d.slug) && MINTED_SUFFIX.test(tracked.slice(d.slug.length));
|
|
3189
|
+
const conflict = declared.find((d) => d.slug !== tracked && !minted(d)) ?? null;
|
|
3190
|
+
return { ...base, slug: tracked, source: ".gateway metadata", conflict };
|
|
3191
|
+
}
|
|
3192
|
+
const first = declared[0];
|
|
3193
|
+
if (first) return { ...base, slug: first.slug, source: first.source };
|
|
3194
|
+
return { ...base, slug: path3.basename(path3.resolve(dir)), source: "directory name" };
|
|
3046
3195
|
}
|
|
3047
3196
|
function writeLocalMeta(dir, meta) {
|
|
3048
|
-
|
|
3049
|
-
|
|
3197
|
+
fs3.mkdirSync(path3.join(dir, ".gateway"), { recursive: true });
|
|
3198
|
+
fs3.writeFileSync(path3.join(dir, META_RELPATH), JSON.stringify(meta, null, 2));
|
|
3050
3199
|
}
|
|
3051
3200
|
function snapshotTreeHashes(dir) {
|
|
3052
3201
|
const out = {};
|
|
3053
3202
|
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
3054
|
-
out[relPath] =
|
|
3203
|
+
out[relPath] = createHash3("sha256").update(fs3.readFileSync(path3.join(dir, relPath))).digest("hex");
|
|
3055
3204
|
}
|
|
3056
3205
|
return out;
|
|
3057
3206
|
}
|
|
@@ -3069,7 +3218,7 @@ function localTreeState(dir) {
|
|
|
3069
3218
|
}
|
|
3070
3219
|
function readPyproject(dir) {
|
|
3071
3220
|
try {
|
|
3072
|
-
const text2 =
|
|
3221
|
+
const text2 = fs3.readFileSync(path3.join(dir, "pyproject.toml"), "utf-8");
|
|
3073
3222
|
const section = /\[project\]([\s\S]*?)(\n\[|$)/.exec(text2)?.[1] ?? "";
|
|
3074
3223
|
const name = /\bname\s*=\s*"([^"]+)"/.exec(section)?.[1];
|
|
3075
3224
|
const version = /\bversion\s*=\s*"([^"]+)"/.exec(section)?.[1];
|
|
@@ -3079,39 +3228,27 @@ function readPyproject(dir) {
|
|
|
3079
3228
|
}
|
|
3080
3229
|
}
|
|
3081
3230
|
function readWorldContractVersion(dir) {
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
}
|
|
3231
|
+
const world = WorldDir.open(dir);
|
|
3232
|
+
try {
|
|
3233
|
+
const parsed = JSON.parse(
|
|
3234
|
+
world.text(world.worldPath("schema/world.json")) ?? ""
|
|
3235
|
+
);
|
|
3236
|
+
const pkg = parsed["x-gateway"]?.package;
|
|
3237
|
+
if (pkg?.version) return { name: pkg.id, version: pkg.version };
|
|
3238
|
+
} catch {
|
|
3089
3239
|
}
|
|
3090
3240
|
return {};
|
|
3091
3241
|
}
|
|
3092
3242
|
function buildFileManifest(dir) {
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
} else if (entry.isFile()) {
|
|
3103
|
-
const payload = fs2.readFileSync(path2.join(dir, childRel));
|
|
3104
|
-
entries.push({
|
|
3105
|
-
path: childRel,
|
|
3106
|
-
isDirectory: false,
|
|
3107
|
-
size: payload.length,
|
|
3108
|
-
sha256: createHash2("sha256").update(payload).digest("hex")
|
|
3109
|
-
});
|
|
3110
|
-
}
|
|
3111
|
-
}
|
|
3112
|
-
};
|
|
3113
|
-
walk("");
|
|
3114
|
-
return entries;
|
|
3243
|
+
return collectArchiveRelPaths(dir).map((relPath) => {
|
|
3244
|
+
const payload = fs3.readFileSync(path3.join(dir, relPath));
|
|
3245
|
+
return {
|
|
3246
|
+
path: relPath,
|
|
3247
|
+
isDirectory: false,
|
|
3248
|
+
size: payload.length,
|
|
3249
|
+
sha256: createHash3("sha256").update(payload).digest("hex")
|
|
3250
|
+
};
|
|
3251
|
+
});
|
|
3115
3252
|
}
|
|
3116
3253
|
async function putObject(url, data, shaB64) {
|
|
3117
3254
|
const resp = await fetch(url, {
|
|
@@ -3123,31 +3260,39 @@ async function putObject(url, data, shaB64) {
|
|
|
3123
3260
|
body: new Uint8Array(data)
|
|
3124
3261
|
});
|
|
3125
3262
|
if (!resp.ok) {
|
|
3126
|
-
throw new Error(
|
|
3263
|
+
throw new Error(
|
|
3264
|
+
`object PUT -> ${resp.status}: ${(await resp.text()).slice(0, 300)}`
|
|
3265
|
+
);
|
|
3127
3266
|
}
|
|
3128
3267
|
}
|
|
3129
3268
|
async function push(dir, opts = {}) {
|
|
3130
3269
|
const options = opts.options ?? {};
|
|
3131
|
-
if (!
|
|
3270
|
+
if (!fs3.statSync(dir).isDirectory())
|
|
3271
|
+
throw new Error(`Not a directory: ${dir}`);
|
|
3132
3272
|
const localMeta = readLocalMeta(dir) ?? {};
|
|
3133
3273
|
const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
|
|
3134
3274
|
const baseVersionId = opts.baseVersionId ?? localMeta.versionId ?? null;
|
|
3275
|
+
const fileManifest = buildFileManifest(dir);
|
|
3276
|
+
if (fileManifest.length === 0)
|
|
3277
|
+
throw new Error(
|
|
3278
|
+
`Nothing to push: ${dir} has no files the archive rules keep (a world root marker, README.md, pyproject.toml, *.py, or any file below a subdirectory).`
|
|
3279
|
+
);
|
|
3135
3280
|
const contentHash = computeContentHash(dir);
|
|
3136
3281
|
const pyproject = readPyproject(dir);
|
|
3137
3282
|
const contractVersion = pyproject.version ? void 0 : readWorldContractVersion(dir).version;
|
|
3138
3283
|
if (contractVersion) pyproject.version = contractVersion;
|
|
3139
|
-
const slug = opts.slug ?? localMeta.slug ?? pyproject.name ??
|
|
3284
|
+
const slug = opts.slug ?? localMeta.slug ?? pyproject.name ?? path3.basename(path3.resolve(dir));
|
|
3285
|
+
const metadata = {
|
|
3286
|
+
...opts.name !== void 0 ? { name: opts.name } : {},
|
|
3287
|
+
...opts.description !== void 0 ? { description: opts.description } : {},
|
|
3288
|
+
...opts.tags !== void 0 ? { tags: opts.tags } : {},
|
|
3289
|
+
...opts.visibility ? { visibility: opts.visibility } : {}
|
|
3290
|
+
};
|
|
3140
3291
|
const resolve6 = await apiRequest(
|
|
3141
3292
|
options,
|
|
3142
3293
|
"POST",
|
|
3143
3294
|
"/api/public/benchmark-containers/resolve",
|
|
3144
|
-
{
|
|
3145
|
-
slug,
|
|
3146
|
-
name: opts.name ?? pyproject.name ?? slug,
|
|
3147
|
-
...opts.description ? { description: opts.description } : {},
|
|
3148
|
-
...opts.tags?.length ? { tags: opts.tags } : {},
|
|
3149
|
-
...opts.visibility ? { visibility: opts.visibility } : {}
|
|
3150
|
-
}
|
|
3295
|
+
{ slug, ...metadata }
|
|
3151
3296
|
);
|
|
3152
3297
|
const tarball = buildSourceTarball(dir);
|
|
3153
3298
|
const bundleSha256 = sha256b64(tarball);
|
|
@@ -3158,7 +3303,7 @@ async function push(dir, opts = {}) {
|
|
|
3158
3303
|
bundleSizeBytes: tarball.length,
|
|
3159
3304
|
...pyproject.version ? { semanticVersion: pyproject.version } : {},
|
|
3160
3305
|
pyprojectMeta: pyproject,
|
|
3161
|
-
fileManifest
|
|
3306
|
+
fileManifest,
|
|
3162
3307
|
branch,
|
|
3163
3308
|
baseVersionId,
|
|
3164
3309
|
...opts.force ? { force: true } : {},
|
|
@@ -3166,12 +3311,18 @@ async function push(dir, opts = {}) {
|
|
|
3166
3311
|
...opts.author ? { author: opts.author } : {},
|
|
3167
3312
|
...opts.imageRef ? { imageRef: opts.imageRef } : {}
|
|
3168
3313
|
});
|
|
3169
|
-
if (version.uploadUrl)
|
|
3314
|
+
if (version.uploadUrl)
|
|
3315
|
+
await putObject(version.uploadUrl, tarball, bundleSha256);
|
|
3170
3316
|
const final = await apiRequest(
|
|
3171
3317
|
options,
|
|
3172
3318
|
"POST",
|
|
3173
3319
|
`/api/public/benchmark-containers/versions/${version.versionId}/finalize`,
|
|
3174
|
-
{
|
|
3320
|
+
{
|
|
3321
|
+
branch,
|
|
3322
|
+
baseVersionId,
|
|
3323
|
+
...opts.force ? { force: true } : {},
|
|
3324
|
+
...metadata
|
|
3325
|
+
}
|
|
3175
3326
|
);
|
|
3176
3327
|
writeLocalMeta(dir, {
|
|
3177
3328
|
...localMeta,
|
|
@@ -3204,7 +3355,14 @@ function extractBundle(bundle, dir) {
|
|
|
3204
3355
|
offset += 512;
|
|
3205
3356
|
if (header.every((b) => b === 0)) break;
|
|
3206
3357
|
const rawName = header.subarray(0, 100).toString("utf-8").replace(/\0.*$/, "");
|
|
3207
|
-
const size = parseInt(
|
|
3358
|
+
const size = parseInt(
|
|
3359
|
+
header.subarray(124, 136).toString("ascii").replace(/\0.*$/, "").trim(),
|
|
3360
|
+
8
|
|
3361
|
+
) || 0;
|
|
3362
|
+
const mode = parseInt(
|
|
3363
|
+
header.subarray(100, 108).toString("ascii").replace(/\0.*$/, "").trim(),
|
|
3364
|
+
8
|
|
3365
|
+
) || FILE_MODE;
|
|
3208
3366
|
const typeflag = header.subarray(156, 157).toString("ascii");
|
|
3209
3367
|
const body = tar.subarray(offset, offset + size);
|
|
3210
3368
|
offset += size + (size % 512 === 0 ? 0 : 512 - size % 512);
|
|
@@ -3216,9 +3374,10 @@ function extractBundle(bundle, dir) {
|
|
|
3216
3374
|
longName = null;
|
|
3217
3375
|
if (typeflag !== "0" && typeflag !== "") continue;
|
|
3218
3376
|
if (name.startsWith("/") || name.split("/").includes("..")) continue;
|
|
3219
|
-
const target =
|
|
3220
|
-
|
|
3221
|
-
|
|
3377
|
+
const target = path3.join(dir, name);
|
|
3378
|
+
fs3.mkdirSync(path3.dirname(target), { recursive: true });
|
|
3379
|
+
fs3.writeFileSync(target, body, { mode: archiveMode(mode) });
|
|
3380
|
+
fs3.chmodSync(target, archiveMode(mode));
|
|
3222
3381
|
written++;
|
|
3223
3382
|
}
|
|
3224
3383
|
return written;
|
|
@@ -3234,13 +3393,13 @@ async function pull(slugRef, target, options = {}) {
|
|
|
3234
3393
|
if (!details.downloadUrl) {
|
|
3235
3394
|
throw new Error(`${slug}@${ref} resolved but has no uploaded bundle`);
|
|
3236
3395
|
}
|
|
3237
|
-
let dir = target ??
|
|
3396
|
+
let dir = target ?? path3.join(process.cwd(), slug);
|
|
3238
3397
|
if (!target) {
|
|
3239
3398
|
let i = 1;
|
|
3240
3399
|
const base = dir;
|
|
3241
|
-
while (
|
|
3400
|
+
while (fs3.existsSync(dir)) dir = `${base}-${i++}`;
|
|
3242
3401
|
}
|
|
3243
|
-
|
|
3402
|
+
fs3.mkdirSync(dir, { recursive: true });
|
|
3244
3403
|
const resp = await fetch(details.downloadUrl);
|
|
3245
3404
|
if (!resp.ok) throw new Error(`bundle download -> ${resp.status}`);
|
|
3246
3405
|
const bundle = Buffer.from(await resp.arrayBuffer());
|
|
@@ -3265,17 +3424,21 @@ async function pull(slugRef, target, options = {}) {
|
|
|
3265
3424
|
async function propose(dir, opts) {
|
|
3266
3425
|
const meta = readLocalMeta(dir);
|
|
3267
3426
|
if (!meta?.slug || !meta.baseFiles) {
|
|
3268
|
-
throw new Error(
|
|
3427
|
+
throw new Error(
|
|
3428
|
+
"This dir has no pull/push base \u2014 run pull first, then edit, then propose."
|
|
3429
|
+
);
|
|
3269
3430
|
}
|
|
3270
3431
|
const state2 = localTreeState(dir);
|
|
3271
|
-
if (state2.clean)
|
|
3432
|
+
if (state2.clean)
|
|
3433
|
+
throw new Error("Nothing to propose \u2014 the tree matches its base.");
|
|
3272
3434
|
const changes = state2.dirty.map((relPath) => {
|
|
3273
|
-
const abs =
|
|
3274
|
-
if (!
|
|
3435
|
+
const abs = path3.join(dir, relPath);
|
|
3436
|
+
if (!fs3.existsSync(abs))
|
|
3437
|
+
return { path: relPath, action: "delete" };
|
|
3275
3438
|
return {
|
|
3276
3439
|
path: relPath,
|
|
3277
3440
|
action: "write",
|
|
3278
|
-
newContent:
|
|
3441
|
+
newContent: fs3.readFileSync(abs, "utf-8")
|
|
3279
3442
|
};
|
|
3280
3443
|
});
|
|
3281
3444
|
const res = await apiRequest(
|
|
@@ -3301,26 +3464,29 @@ async function createRef(slug, name, kind, versionId, options = {}) {
|
|
|
3301
3464
|
...versionId ? { versionId } : {}
|
|
3302
3465
|
});
|
|
3303
3466
|
}
|
|
3304
|
-
var DEFAULT_BRANCH, META_RELPATH,
|
|
3467
|
+
var DEFAULT_BRANCH, META_RELPATH, FILE_MODE, EXEC_MODE, MINTED_SUFFIX, PushSlugConflict, sha256b64;
|
|
3305
3468
|
var init_hub = __esm({
|
|
3306
3469
|
"src/worlds/hub.ts"() {
|
|
3307
3470
|
"use strict";
|
|
3308
3471
|
init_worlds();
|
|
3472
|
+
init_worldDir();
|
|
3309
3473
|
DEFAULT_BRANCH = "main";
|
|
3310
|
-
META_RELPATH =
|
|
3311
|
-
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3474
|
+
META_RELPATH = path3.join(".gateway", ".env-metadata.json");
|
|
3475
|
+
FILE_MODE = 420;
|
|
3476
|
+
EXEC_MODE = 493;
|
|
3477
|
+
MINTED_SUFFIX = /^-\d+$/;
|
|
3478
|
+
PushSlugConflict = class extends Error {
|
|
3479
|
+
constructor(dir, tracked, declared) {
|
|
3480
|
+
super(
|
|
3481
|
+
`${dir} tracks slug \`${tracked}\` (${META_RELPATH}, from an earlier push) but its ${declared.source} says \`${declared.slug}\`. Nothing was pushed. Pass --slug ${declared.slug} to push there (and retrack the dir), --slug ${tracked} to keep pushing to \`${tracked}\`, or delete ${META_RELPATH} to forget the earlier push.`
|
|
3482
|
+
);
|
|
3483
|
+
this.dir = dir;
|
|
3484
|
+
this.tracked = tracked;
|
|
3485
|
+
this.declared = declared;
|
|
3486
|
+
this.name = "PushSlugConflict";
|
|
3487
|
+
}
|
|
3488
|
+
};
|
|
3489
|
+
sha256b64 = (data) => createHash3("sha256").update(data).digest("base64");
|
|
3324
3490
|
}
|
|
3325
3491
|
});
|
|
3326
3492
|
|
|
@@ -3819,7 +3985,7 @@ var init_json = __esm({
|
|
|
3819
3985
|
});
|
|
3820
3986
|
|
|
3821
3987
|
// src/worlds/transforms.ts
|
|
3822
|
-
import { createHash as
|
|
3988
|
+
import { createHash as createHash4 } from "crypto";
|
|
3823
3989
|
function parseTransform(name) {
|
|
3824
3990
|
const simple = Object.prototype.hasOwnProperty.call(SIMPLE, name) ? SIMPLE[name] : void 0;
|
|
3825
3991
|
if (simple) return simple;
|
|
@@ -3843,8 +4009,8 @@ var init_transforms = __esm({
|
|
|
3843
4009
|
"use strict";
|
|
3844
4010
|
init_json();
|
|
3845
4011
|
WHITESPACE = " \n\r\f\v";
|
|
3846
|
-
WHITESPACE_EDGES =
|
|
3847
|
-
WHITESPACE_RUN =
|
|
4012
|
+
WHITESPACE_EDGES = new RegExp(`^[${WHITESPACE}]+|[${WHITESPACE}]+$`, "g");
|
|
4013
|
+
WHITESPACE_RUN = new RegExp(`[${WHITESPACE}]+`, "g");
|
|
3848
4014
|
NOT_DIGIT = /[^0-9]/g;
|
|
3849
4015
|
STRIP_PREFIX = "strip_prefix";
|
|
3850
4016
|
trim = (value) => value.replace(WHITESPACE_EDGES, "");
|
|
@@ -3857,7 +4023,7 @@ var init_transforms = __esm({
|
|
|
3857
4023
|
return at === -1 ? "" : value.slice(at + 1).toLowerCase();
|
|
3858
4024
|
};
|
|
3859
4025
|
stripPrefix = (prefix) => (value) => value.startsWith(prefix) ? value.slice(prefix.length) : value;
|
|
3860
|
-
hex = (algorithm, value) =>
|
|
4026
|
+
hex = (algorithm, value) => createHash4(algorithm).update(value, "utf8").digest("hex");
|
|
3861
4027
|
SIMPLE = {
|
|
3862
4028
|
lower: (value) => value.toLowerCase(),
|
|
3863
4029
|
upper: (value) => value.toUpperCase(),
|
|
@@ -3881,7 +4047,7 @@ var init_transforms = __esm({
|
|
|
3881
4047
|
});
|
|
3882
4048
|
|
|
3883
4049
|
// src/worlds/redaction.ts
|
|
3884
|
-
import { createHash as
|
|
4050
|
+
import { createHash as createHash5 } from "crypto";
|
|
3885
4051
|
function pyText(value) {
|
|
3886
4052
|
if (typeof value === "string") return value;
|
|
3887
4053
|
if (Number.isInteger(value)) return String(value);
|
|
@@ -3890,7 +4056,7 @@ function pyText(value) {
|
|
|
3890
4056
|
return pyFloatRepr(value);
|
|
3891
4057
|
}
|
|
3892
4058
|
function hashForm(slug, plaintext) {
|
|
3893
|
-
return "sha256:" +
|
|
4059
|
+
return "sha256:" + createHash5("sha256").update(`${slug}:${pyText(plaintext)}`, "utf8").digest("hex");
|
|
3894
4060
|
}
|
|
3895
4061
|
function preserveForm(slug, plaintext) {
|
|
3896
4062
|
const kind = typeof plaintext === "string" ? "str" : isInt(plaintext) ? "int" : "float";
|
|
@@ -3899,12 +4065,12 @@ function preserveForm(slug, plaintext) {
|
|
|
3899
4065
|
}
|
|
3900
4066
|
function shapeText(slug, text2, kind) {
|
|
3901
4067
|
const seed = Buffer.from(`${slug}:${text2}`, "utf8");
|
|
3902
|
-
let digest =
|
|
4068
|
+
let digest = createHash5("sha256").update(seed).digest();
|
|
3903
4069
|
let cursor = 0;
|
|
3904
4070
|
const out = [];
|
|
3905
4071
|
for (const char of text2) {
|
|
3906
4072
|
if (cursor === digest.length) {
|
|
3907
|
-
digest =
|
|
4073
|
+
digest = createHash5("sha256").update(Buffer.concat([digest, seed])).digest();
|
|
3908
4074
|
cursor = 0;
|
|
3909
4075
|
}
|
|
3910
4076
|
const byte = digest[cursor];
|
|
@@ -3952,8 +4118,9 @@ var init_redaction = __esm({
|
|
|
3952
4118
|
});
|
|
3953
4119
|
|
|
3954
4120
|
// src/worlds/query.ts
|
|
3955
|
-
function queryRowsSearch(entity, opts = {}) {
|
|
4121
|
+
function queryRowsSearch(entity, opts = {}, ref) {
|
|
3956
4122
|
const search = new URLSearchParams();
|
|
4123
|
+
if (ref) search.set("ref", ref);
|
|
3957
4124
|
search.set("entity", entity);
|
|
3958
4125
|
if (opts.where && Object.keys(opts.where).length) search.set("where", JSON.stringify(opts.where));
|
|
3959
4126
|
if (opts.limit !== void 0) search.set("limit", String(opts.limit));
|
|
@@ -3963,17 +4130,15 @@ function queryRowsSearch(entity, opts = {}) {
|
|
|
3963
4130
|
if (opts.compare) search.set("compare", "true");
|
|
3964
4131
|
return `?${search.toString()}`;
|
|
3965
4132
|
}
|
|
3966
|
-
async function queryRows(
|
|
3967
|
-
|
|
3968
|
-
|
|
3969
|
-
`queryRows reads a world's head version: pass the slug without @ref (got ${slug})`
|
|
3970
|
-
);
|
|
4133
|
+
async function queryRows(slugRef, entity, opts = {}, options = {}) {
|
|
4134
|
+
const [slug, ref] = slugRef.split("@", 2);
|
|
4135
|
+
if (!slug || ref === "") throw new Error(`queryRows needs a slug, optionally slug@ref (got ${slugRef})`);
|
|
3971
4136
|
if (!entity) throw new Error("queryRows needs an entity");
|
|
3972
4137
|
if (opts.compare && !opts.across) throw new Error("queryRows: compare needs across");
|
|
3973
4138
|
return apiRequest(
|
|
3974
4139
|
options,
|
|
3975
4140
|
"GET",
|
|
3976
|
-
`/api/public/worlds/${encodeURIComponent(slug)}/data${queryRowsSearch(entity, opts)}`
|
|
4141
|
+
`/api/public/worlds/${encodeURIComponent(slug)}/data${queryRowsSearch(entity, opts, ref)}`
|
|
3977
4142
|
);
|
|
3978
4143
|
}
|
|
3979
4144
|
var init_query = __esm({
|
|
@@ -4029,14 +4194,23 @@ var init_router = __esm({
|
|
|
4029
4194
|
});
|
|
4030
4195
|
|
|
4031
4196
|
// src/cli/config.ts
|
|
4032
|
-
import * as
|
|
4197
|
+
import * as fs4 from "fs";
|
|
4033
4198
|
import * as os from "os";
|
|
4034
|
-
import * as
|
|
4199
|
+
import * as path4 from "path";
|
|
4200
|
+
function transportDetail(error, timeoutMs) {
|
|
4201
|
+
if (error instanceof Error && error.name === "AbortError") return `timed out after ${Math.round(timeoutMs / 1e3)}s`;
|
|
4202
|
+
const cause = error instanceof Error && error.cause instanceof Error ? error.cause : void 0;
|
|
4203
|
+
const code = cause && "code" in cause && typeof cause.code === "string" ? `${cause.code}: ` : "";
|
|
4204
|
+
const message = cause?.message ?? (error instanceof Error ? error.message : String(error));
|
|
4205
|
+
return `${code}${message}`;
|
|
4206
|
+
}
|
|
4035
4207
|
async function api(creds, method, route, body, init2 = {}) {
|
|
4036
4208
|
const controller = new AbortController();
|
|
4037
|
-
const
|
|
4209
|
+
const timeoutMs = init2.timeoutMs ?? 12e4;
|
|
4210
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
4211
|
+
let res;
|
|
4038
4212
|
try {
|
|
4039
|
-
|
|
4213
|
+
res = await fetch(`${creds.host}${route}`, {
|
|
4040
4214
|
method,
|
|
4041
4215
|
headers: {
|
|
4042
4216
|
"content-type": "application/json",
|
|
@@ -4046,6 +4220,11 @@ async function api(creds, method, route, body, init2 = {}) {
|
|
|
4046
4220
|
...body === void 0 ? {} : { body: JSON.stringify(body) },
|
|
4047
4221
|
signal: controller.signal
|
|
4048
4222
|
});
|
|
4223
|
+
} catch (error) {
|
|
4224
|
+
clearTimeout(timer);
|
|
4225
|
+
throw new UnreachableError(creds.host, method, route, transportDetail(error, timeoutMs));
|
|
4226
|
+
}
|
|
4227
|
+
try {
|
|
4049
4228
|
const text2 = await res.text();
|
|
4050
4229
|
if (!res.ok) throw new ApiError(res.status, method, route, text2);
|
|
4051
4230
|
return text2 ? JSON.parse(text2) : {};
|
|
@@ -4053,7 +4232,7 @@ async function api(creds, method, route, body, init2 = {}) {
|
|
|
4053
4232
|
clearTimeout(timer);
|
|
4054
4233
|
}
|
|
4055
4234
|
}
|
|
4056
|
-
var ApiError;
|
|
4235
|
+
var ApiError, UnreachableError;
|
|
4057
4236
|
var init_config = __esm({
|
|
4058
4237
|
"src/cli/config.ts"() {
|
|
4059
4238
|
"use strict";
|
|
@@ -4067,14 +4246,23 @@ var init_config = __esm({
|
|
|
4067
4246
|
this.text = text2;
|
|
4068
4247
|
}
|
|
4069
4248
|
};
|
|
4249
|
+
UnreachableError = class extends CliError {
|
|
4250
|
+
constructor(host, method, route, detail) {
|
|
4251
|
+
super(`${method} ${route}: no answer from ${host} (${detail}). Check the host and the network, then retry.`, 1);
|
|
4252
|
+
this.host = host;
|
|
4253
|
+
this.method = method;
|
|
4254
|
+
this.route = route;
|
|
4255
|
+
this.detail = detail;
|
|
4256
|
+
}
|
|
4257
|
+
};
|
|
4070
4258
|
}
|
|
4071
4259
|
});
|
|
4072
4260
|
|
|
4073
4261
|
// src/cli/runtime/local.ts
|
|
4074
4262
|
import { spawn, spawnSync } from "child_process";
|
|
4075
|
-
import * as
|
|
4263
|
+
import * as fs5 from "fs";
|
|
4076
4264
|
import * as net from "net";
|
|
4077
|
-
import * as
|
|
4265
|
+
import * as path5 from "path";
|
|
4078
4266
|
function tailOf(output) {
|
|
4079
4267
|
return output.length > DIAGNOSTIC_TAIL ? `\u2026${output.slice(-DIAGNOSTIC_TAIL)}` : output;
|
|
4080
4268
|
}
|
|
@@ -4089,16 +4277,16 @@ function runtimeRoot() {
|
|
|
4089
4277
|
const candidates = [];
|
|
4090
4278
|
if (process.env.GATEWAY_RUNTIME_PATH)
|
|
4091
4279
|
candidates.push(process.env.GATEWAY_RUNTIME_PATH);
|
|
4092
|
-
const entry = process.argv[1] &&
|
|
4280
|
+
const entry = process.argv[1] && fs5.existsSync(process.argv[1]) ? path5.dirname(fs5.realpathSync(process.argv[1])) : null;
|
|
4093
4281
|
if (entry) {
|
|
4094
|
-
candidates.push(
|
|
4095
|
-
candidates.push(
|
|
4096
|
-
candidates.push(
|
|
4282
|
+
candidates.push(path5.resolve(entry, "..", "runtime"));
|
|
4283
|
+
candidates.push(path5.resolve(entry, "..", "..", "runtime"));
|
|
4284
|
+
candidates.push(path5.resolve(entry, "..", "..", "sdk", "src"));
|
|
4097
4285
|
}
|
|
4098
|
-
candidates.push(
|
|
4286
|
+
candidates.push(path5.resolve(process.cwd(), "runtime"));
|
|
4099
4287
|
for (const root of candidates)
|
|
4100
|
-
if (
|
|
4101
|
-
|
|
4288
|
+
if (fs5.existsSync(
|
|
4289
|
+
path5.join(root, "gatewaysdk", "environment", "schema", "platform.py")
|
|
4102
4290
|
))
|
|
4103
4291
|
return root;
|
|
4104
4292
|
return null;
|
|
@@ -4266,28 +4454,28 @@ runpy.run_module('gatewaysdk.environment.schema.platform',run_name='__main__')`;
|
|
|
4266
4454
|
});
|
|
4267
4455
|
|
|
4268
4456
|
// src/cli/runtime/client.ts
|
|
4269
|
-
import * as
|
|
4270
|
-
import * as
|
|
4457
|
+
import * as fs6 from "fs";
|
|
4458
|
+
import * as path6 from "path";
|
|
4271
4459
|
function isBuildOutput(relPath) {
|
|
4272
4460
|
return BUILD_OUTPUT.test(relPath);
|
|
4273
4461
|
}
|
|
4274
4462
|
function readWorldFiles(dir, opts = {}) {
|
|
4275
|
-
if (!
|
|
4463
|
+
if (!fs6.existsSync(dir) || !fs6.statSync(dir).isDirectory())
|
|
4276
4464
|
throw new CliError(`Not a directory: ${dir}`);
|
|
4277
4465
|
const out = [];
|
|
4278
4466
|
const walk = (rel) => {
|
|
4279
|
-
const abs = rel ?
|
|
4280
|
-
const entries =
|
|
4467
|
+
const abs = rel ? path6.join(dir, rel) : dir;
|
|
4468
|
+
const entries = fs6.readdirSync(abs, { withFileTypes: true });
|
|
4281
4469
|
entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
4282
4470
|
for (const entry of entries) {
|
|
4283
4471
|
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
4284
4472
|
if (entry.isDirectory()) {
|
|
4285
|
-
if (!
|
|
4473
|
+
if (!SKIP_DIRS.has(entry.name)) walk(childRel);
|
|
4286
4474
|
} else if (entry.isFile()) {
|
|
4287
4475
|
if (!opts.buildOutputs && isBuildOutput(childRel)) continue;
|
|
4288
4476
|
out.push({
|
|
4289
4477
|
path: childRel,
|
|
4290
|
-
content:
|
|
4478
|
+
content: fs6.readFileSync(path6.join(dir, childRel)).toString("base64")
|
|
4291
4479
|
});
|
|
4292
4480
|
}
|
|
4293
4481
|
}
|
|
@@ -4302,19 +4490,19 @@ function assertSafeRelPath(rel) {
|
|
|
4302
4490
|
}
|
|
4303
4491
|
function writeWorldFiles(dir, files, opts = {}) {
|
|
4304
4492
|
for (const file of files) assertSafeRelPath(file.path);
|
|
4305
|
-
if (opts.clean &&
|
|
4493
|
+
if (opts.clean && fs6.existsSync(dir)) {
|
|
4306
4494
|
const keep = new Set(files.map((f) => f.path));
|
|
4307
4495
|
for (const existing of readWorldFiles(dir)) {
|
|
4308
|
-
if (!keep.has(existing.path))
|
|
4496
|
+
if (!keep.has(existing.path)) fs6.rmSync(path6.join(dir, existing.path));
|
|
4309
4497
|
}
|
|
4310
4498
|
pruneEmptyDirs(dir);
|
|
4311
4499
|
}
|
|
4312
|
-
|
|
4500
|
+
fs6.mkdirSync(dir, { recursive: true });
|
|
4313
4501
|
const written = [];
|
|
4314
4502
|
for (const file of files) {
|
|
4315
|
-
const target =
|
|
4316
|
-
|
|
4317
|
-
|
|
4503
|
+
const target = path6.join(dir, file.path);
|
|
4504
|
+
fs6.mkdirSync(path6.dirname(target), { recursive: true });
|
|
4505
|
+
fs6.writeFileSync(target, Buffer.from(file.content, "base64"));
|
|
4318
4506
|
written.push(file.path);
|
|
4319
4507
|
}
|
|
4320
4508
|
return written;
|
|
@@ -4322,14 +4510,14 @@ function writeWorldFiles(dir, files, opts = {}) {
|
|
|
4322
4510
|
function pruneEmptyDirs(root) {
|
|
4323
4511
|
const walk = (abs) => {
|
|
4324
4512
|
let empty = true;
|
|
4325
|
-
for (const entry of
|
|
4326
|
-
const child =
|
|
4513
|
+
for (const entry of fs6.readdirSync(abs, { withFileTypes: true })) {
|
|
4514
|
+
const child = path6.join(abs, entry.name);
|
|
4327
4515
|
if (entry.isDirectory()) {
|
|
4328
|
-
if (
|
|
4516
|
+
if (SKIP_DIRS.has(entry.name)) {
|
|
4329
4517
|
empty = false;
|
|
4330
4518
|
continue;
|
|
4331
4519
|
}
|
|
4332
|
-
if (walk(child))
|
|
4520
|
+
if (walk(child)) fs6.rmdirSync(child);
|
|
4333
4521
|
else empty = false;
|
|
4334
4522
|
} else empty = false;
|
|
4335
4523
|
}
|
|
@@ -4393,13 +4581,13 @@ async function runtime(creds, request, opts = {}) {
|
|
|
4393
4581
|
clearTimeout(timer);
|
|
4394
4582
|
}
|
|
4395
4583
|
}
|
|
4396
|
-
var
|
|
4584
|
+
var SKIP_DIRS, BUILD_OUTPUT, BridgeRejected;
|
|
4397
4585
|
var init_client = __esm({
|
|
4398
4586
|
"src/cli/runtime/client.ts"() {
|
|
4399
4587
|
"use strict";
|
|
4400
4588
|
init_router();
|
|
4401
4589
|
init_local();
|
|
4402
|
-
|
|
4590
|
+
SKIP_DIRS = /* @__PURE__ */ new Set(["__pycache__", ".git", "node_modules", ".gateway"]);
|
|
4403
4591
|
BUILD_OUTPUT = /(^|\/)state\.(sqlite|json)$/;
|
|
4404
4592
|
BridgeRejected = class extends CliError {
|
|
4405
4593
|
constructor(detail) {
|
|
@@ -4421,8 +4609,8 @@ __export(data_exports, {
|
|
|
4421
4609
|
queryLocalWorldDataAcross: () => queryLocalWorldDataAcross,
|
|
4422
4610
|
showLocalWorldData: () => showLocalWorldData
|
|
4423
4611
|
});
|
|
4424
|
-
import * as
|
|
4425
|
-
import * as
|
|
4612
|
+
import * as fs7 from "fs";
|
|
4613
|
+
import * as path7 from "path";
|
|
4426
4614
|
async function importLocalWorldData(dir, rows, opts, creds) {
|
|
4427
4615
|
const res = await runtime(creds, {
|
|
4428
4616
|
command: "import-data",
|
|
@@ -4448,7 +4636,7 @@ function dropBuildOutputs(dir) {
|
|
|
4448
4636
|
const dropped = [];
|
|
4449
4637
|
for (const file of readWorldFiles(dir, { buildOutputs: true })) {
|
|
4450
4638
|
if (!BUILD_OUTPUT2.test(file.path)) continue;
|
|
4451
|
-
|
|
4639
|
+
fs7.rmSync(path7.join(dir, file.path), { force: true });
|
|
4452
4640
|
dropped.push(file.path);
|
|
4453
4641
|
}
|
|
4454
4642
|
return dropped;
|
|
@@ -4459,6 +4647,7 @@ async function checkLocalWorldData(dir, rows, creds, opts = {}) {
|
|
|
4459
4647
|
files: readWorldFiles(dir),
|
|
4460
4648
|
rows,
|
|
4461
4649
|
mode: "append",
|
|
4650
|
+
dry_run: true,
|
|
4462
4651
|
...opts.redact && opts.redact !== "off" ? { redact: opts.redact } : {}
|
|
4463
4652
|
});
|
|
4464
4653
|
return {
|
|
@@ -4514,7 +4703,7 @@ async function queryLocalWorldDataAcross(dir, entity, where, opts) {
|
|
|
4514
4703
|
const payload = await localBridge(
|
|
4515
4704
|
{
|
|
4516
4705
|
command: "query",
|
|
4517
|
-
world:
|
|
4706
|
+
world: path7.resolve(dir),
|
|
4518
4707
|
entity,
|
|
4519
4708
|
across: true,
|
|
4520
4709
|
...opts.compare ? { compare: true } : {},
|
|
@@ -4585,9 +4774,9 @@ __export(transform_exports, {
|
|
|
4585
4774
|
transformRecords: () => transformRecords
|
|
4586
4775
|
});
|
|
4587
4776
|
import { spawn as spawn2 } from "child_process";
|
|
4588
|
-
import * as
|
|
4777
|
+
import * as fs8 from "fs";
|
|
4589
4778
|
import * as os2 from "os";
|
|
4590
|
-
import * as
|
|
4779
|
+
import * as path8 from "path";
|
|
4591
4780
|
import { createRequire } from "module";
|
|
4592
4781
|
function checkRows(value, where) {
|
|
4593
4782
|
if (!isObject(value)) throw new CliError(`${where} must return {entity: [rows]}; got ${value === null ? "null" : Array.isArray(value) ? "an array" : typeof value}`, 1);
|
|
@@ -4612,27 +4801,27 @@ function typescriptPrefix() {
|
|
|
4612
4801
|
}
|
|
4613
4802
|
}
|
|
4614
4803
|
function command(script, wrapperDir) {
|
|
4615
|
-
const suffix =
|
|
4804
|
+
const suffix = path8.extname(script).toLowerCase();
|
|
4616
4805
|
if (PYTHON_SUFFIXES.includes(suffix)) return { file: process.env.GATEWAY_PYTHON || "python3", args: ["-c", PY_WRAPPER, script] };
|
|
4617
4806
|
if (NODE_SUFFIXES.includes(suffix)) {
|
|
4618
|
-
const wrapper =
|
|
4619
|
-
|
|
4807
|
+
const wrapper = path8.join(wrapperDir, "transform-wrapper.mjs");
|
|
4808
|
+
fs8.writeFileSync(wrapper, NODE_WRAPPER);
|
|
4620
4809
|
const flags = suffix === ".ts" || suffix === ".mts" ? typescriptPrefix() : [];
|
|
4621
4810
|
return { file: process.env.GATEWAY_NODE || process.execPath, args: [...flags, wrapper, script] };
|
|
4622
4811
|
}
|
|
4623
|
-
throw new CliError(`${
|
|
4812
|
+
throw new CliError(`${path8.basename(script)}: a transform script is .py, .ts, .mjs, .js or .cjs`, 1);
|
|
4624
4813
|
}
|
|
4625
4814
|
async function runTransform(script, records, opts = {}) {
|
|
4626
|
-
if (!
|
|
4627
|
-
const resolved =
|
|
4628
|
-
const name =
|
|
4815
|
+
if (!fs8.existsSync(script) || !fs8.statSync(script).isFile()) throw new CliError(`no transform script at ${script}`, 1);
|
|
4816
|
+
const resolved = path8.resolve(script);
|
|
4817
|
+
const name = path8.basename(script);
|
|
4629
4818
|
const timeoutMs = opts.timeoutMs ?? DEFAULT_TRANSFORM_TIMEOUT_MS;
|
|
4630
|
-
const scratch =
|
|
4819
|
+
const scratch = fs8.mkdtempSync(path8.join(os2.tmpdir(), "gateway-transform-"));
|
|
4631
4820
|
try {
|
|
4632
4821
|
const { file, args } = command(resolved, scratch);
|
|
4633
4822
|
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !SCRUBBED.test(key)));
|
|
4634
4823
|
const done = await new Promise((resolve6, reject) => {
|
|
4635
|
-
const child = spawn2(file, args, { cwd:
|
|
4824
|
+
const child = spawn2(file, args, { cwd: path8.dirname(resolved), env, stdio: ["pipe", "pipe", "pipe"] });
|
|
4636
4825
|
let stdout = "";
|
|
4637
4826
|
let stderr = "";
|
|
4638
4827
|
let timedOut = false;
|
|
@@ -4663,7 +4852,7 @@ async function runTransform(script, records, opts = {}) {
|
|
|
4663
4852
|
}
|
|
4664
4853
|
return { rows: checkRows(value, name), stderr: done.stderr };
|
|
4665
4854
|
} finally {
|
|
4666
|
-
|
|
4855
|
+
fs8.rmSync(scratch, { recursive: true, force: true });
|
|
4667
4856
|
}
|
|
4668
4857
|
}
|
|
4669
4858
|
async function transformRecords(transform, records, opts = {}) {
|
|
@@ -4755,13 +4944,18 @@ if (fn) {
|
|
|
4755
4944
|
});
|
|
4756
4945
|
|
|
4757
4946
|
// src/cli/platform/worldData/batches.ts
|
|
4758
|
-
import { createHash as
|
|
4759
|
-
import * as
|
|
4947
|
+
import { createHash as createHash6 } from "crypto";
|
|
4948
|
+
import * as fs9 from "fs";
|
|
4949
|
+
function isSettled(status) {
|
|
4950
|
+
if (!TERMINAL_STATES.has(status.state)) return false;
|
|
4951
|
+
if (status.state !== "applied") return true;
|
|
4952
|
+
return status.dryRun === true || status.publish !== "now" || !status.snapshotId;
|
|
4953
|
+
}
|
|
4760
4954
|
function uploadHeadersFor(gz) {
|
|
4761
4955
|
return {
|
|
4762
4956
|
"content-type": "application/gzip",
|
|
4763
4957
|
"content-length": String(gz.length),
|
|
4764
|
-
"x-amz-checksum-sha256":
|
|
4958
|
+
"x-amz-checksum-sha256": createHash6("sha256").update(gz).digest("base64")
|
|
4765
4959
|
};
|
|
4766
4960
|
}
|
|
4767
4961
|
async function forEachLimited(items, limit, fn) {
|
|
@@ -4798,7 +4992,7 @@ async function waitForBatch(client, batchId, opts = {}) {
|
|
|
4798
4992
|
opts.onPhase?.(phase);
|
|
4799
4993
|
last = phase;
|
|
4800
4994
|
}
|
|
4801
|
-
if (
|
|
4995
|
+
if (isSettled(status)) return status;
|
|
4802
4996
|
await sleep2(delay);
|
|
4803
4997
|
delay = Math.min(1e4, Math.round(delay * 1.5));
|
|
4804
4998
|
}
|
|
@@ -4813,7 +5007,12 @@ var init_batches = __esm({
|
|
|
4813
5007
|
"use strict";
|
|
4814
5008
|
init_config();
|
|
4815
5009
|
init_router();
|
|
4816
|
-
TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
5010
|
+
TERMINAL_STATES = /* @__PURE__ */ new Set([
|
|
5011
|
+
"applied",
|
|
5012
|
+
"refused",
|
|
5013
|
+
"published",
|
|
5014
|
+
"failed"
|
|
5015
|
+
]);
|
|
4817
5016
|
sleepMs = (ms) => new Promise((resolve6) => setTimeout(resolve6, ms));
|
|
4818
5017
|
WorldDataApi = class {
|
|
4819
5018
|
constructor(creds, slug, sleep2 = sleepMs) {
|
|
@@ -4827,22 +5026,61 @@ var init_batches = __esm({
|
|
|
4827
5026
|
return api(this.creds, "POST", `${this.base}/batches`, body);
|
|
4828
5027
|
}
|
|
4829
5028
|
register(batchId, chunk) {
|
|
4830
|
-
return api(
|
|
5029
|
+
return api(
|
|
5030
|
+
this.creds,
|
|
5031
|
+
"POST",
|
|
5032
|
+
`${this.base}/batches/${batchId}/chunks`,
|
|
5033
|
+
{
|
|
5034
|
+
ordinal: chunk.ordinal,
|
|
5035
|
+
digest: chunk.digest,
|
|
5036
|
+
bytes: chunk.bytes,
|
|
5037
|
+
rows: chunk.rows,
|
|
5038
|
+
...chunk.entity ? { entity: chunk.entity } : {}
|
|
5039
|
+
}
|
|
5040
|
+
);
|
|
5041
|
+
}
|
|
5042
|
+
/**
|
|
5043
|
+
* Stage a project-drive rows file as one chunk, on the platform: the server
|
|
5044
|
+
* streams it out of drive storage into chunk storage and answers with the
|
|
5045
|
+
* digest and gzipped size it derived. No bytes pass through this process.
|
|
5046
|
+
*/
|
|
5047
|
+
async stageDrive(batchId, chunk) {
|
|
5048
|
+
const staged = await api(this.creds, "POST", `${this.base}/batches/${batchId}/chunks`, {
|
|
4831
5049
|
ordinal: chunk.ordinal,
|
|
4832
|
-
|
|
4833
|
-
bytes: chunk.bytes,
|
|
4834
|
-
rows: chunk.rows,
|
|
5050
|
+
drive: chunk.path,
|
|
4835
5051
|
...chunk.entity ? { entity: chunk.entity } : {}
|
|
4836
5052
|
});
|
|
5053
|
+
if (!staged.uploaded || !staged.digest || staged.bytes === void 0 || staged.rows === void 0)
|
|
5054
|
+
throw new CliError(
|
|
5055
|
+
`the server staged drive:${chunk.path} as chunk ${chunk.ordinal} without its digest, size and rows`,
|
|
5056
|
+
1
|
|
5057
|
+
);
|
|
5058
|
+
return {
|
|
5059
|
+
ordinal: chunk.ordinal,
|
|
5060
|
+
digest: staged.digest,
|
|
5061
|
+
bytes: staged.bytes,
|
|
5062
|
+
rows: staged.rows,
|
|
5063
|
+
entity: chunk.entity,
|
|
5064
|
+
chunkId: staged.chunkId,
|
|
5065
|
+
uploaded: true
|
|
5066
|
+
};
|
|
4837
5067
|
}
|
|
4838
5068
|
markUploaded(batchId, chunkId) {
|
|
4839
|
-
return api(
|
|
5069
|
+
return api(
|
|
5070
|
+
this.creds,
|
|
5071
|
+
"POST",
|
|
5072
|
+
`${this.base}/batches/${batchId}/chunks/${chunkId}/uploaded`
|
|
5073
|
+
);
|
|
4840
5074
|
}
|
|
4841
5075
|
complete(batchId) {
|
|
4842
5076
|
return api(this.creds, "POST", `${this.base}/batches/${batchId}/complete`);
|
|
4843
5077
|
}
|
|
4844
5078
|
get(batchId) {
|
|
4845
|
-
return api(
|
|
5079
|
+
return api(
|
|
5080
|
+
this.creds,
|
|
5081
|
+
"GET",
|
|
5082
|
+
`${this.base}/batches/${batchId}`
|
|
5083
|
+
);
|
|
4846
5084
|
}
|
|
4847
5085
|
list(limit, cursor) {
|
|
4848
5086
|
const params = new URLSearchParams();
|
|
@@ -4852,7 +5090,12 @@ var init_batches = __esm({
|
|
|
4852
5090
|
return api(this.creds, "GET", `${this.base}/batches${q ? `?${q}` : ""}`);
|
|
4853
5091
|
}
|
|
4854
5092
|
publish(message) {
|
|
4855
|
-
return api(
|
|
5093
|
+
return api(
|
|
5094
|
+
this.creds,
|
|
5095
|
+
"POST",
|
|
5096
|
+
`${this.base}/publish`,
|
|
5097
|
+
message ? { message } : {}
|
|
5098
|
+
);
|
|
4856
5099
|
}
|
|
4857
5100
|
/**
|
|
4858
5101
|
* PUT the gzipped chunk to its presigned URL with exactly the headers the
|
|
@@ -4864,11 +5107,23 @@ var init_batches = __esm({
|
|
|
4864
5107
|
const delays = [500, 1500, 4e3];
|
|
4865
5108
|
for (let attempt = 0; ; attempt++) {
|
|
4866
5109
|
try {
|
|
4867
|
-
const res = await fetch(uploadUrl, {
|
|
5110
|
+
const res = await fetch(uploadUrl, {
|
|
5111
|
+
method: "PUT",
|
|
5112
|
+
body: new Uint8Array(gz),
|
|
5113
|
+
headers
|
|
5114
|
+
});
|
|
4868
5115
|
if (res.ok) return;
|
|
4869
|
-
if (res.status < 500 || attempt >= delays.length)
|
|
5116
|
+
if (res.status < 500 || attempt >= delays.length)
|
|
5117
|
+
throw new CliError(
|
|
5118
|
+
`chunk upload -> ${res.status}: ${(await res.text()).slice(0, 300)}`,
|
|
5119
|
+
1
|
|
5120
|
+
);
|
|
4870
5121
|
} catch (error) {
|
|
4871
|
-
if (error instanceof CliError || attempt >= delays.length)
|
|
5122
|
+
if (error instanceof CliError || attempt >= delays.length)
|
|
5123
|
+
throw error instanceof CliError ? error : new CliError(
|
|
5124
|
+
`chunk upload failed: ${error.message}`,
|
|
5125
|
+
1
|
|
5126
|
+
);
|
|
4872
5127
|
}
|
|
4873
5128
|
await this.sleep(delays[attempt]);
|
|
4874
5129
|
}
|
|
@@ -4877,18 +5132,30 @@ var init_batches = __esm({
|
|
|
4877
5132
|
async upload(batchId, chunk) {
|
|
4878
5133
|
const registered = await this.register(batchId, chunk);
|
|
4879
5134
|
if (!registered.uploaded) {
|
|
4880
|
-
if (!registered.uploadUrl)
|
|
5135
|
+
if (!registered.uploadUrl)
|
|
5136
|
+
throw new CliError(
|
|
5137
|
+
`the server registered chunk ${chunk.ordinal} without an upload URL`,
|
|
5138
|
+
1
|
|
5139
|
+
);
|
|
4881
5140
|
await this.put(registered.uploadUrl, chunk.gz, registered.uploadHeaders);
|
|
4882
5141
|
await this.markUploaded(batchId, registered.chunkId);
|
|
4883
5142
|
}
|
|
4884
|
-
return {
|
|
5143
|
+
return {
|
|
5144
|
+
ordinal: chunk.ordinal,
|
|
5145
|
+
digest: chunk.digest,
|
|
5146
|
+
bytes: chunk.bytes,
|
|
5147
|
+
rows: chunk.rows,
|
|
5148
|
+
entity: chunk.entity,
|
|
5149
|
+
chunkId: registered.chunkId,
|
|
5150
|
+
uploaded: true
|
|
5151
|
+
};
|
|
4885
5152
|
}
|
|
4886
5153
|
};
|
|
4887
5154
|
}
|
|
4888
5155
|
});
|
|
4889
5156
|
|
|
4890
5157
|
// src/cli/platform/worldData/chunker.ts
|
|
4891
|
-
import { createHash as
|
|
5158
|
+
import { createHash as createHash7 } from "crypto";
|
|
4892
5159
|
import { deflateRawSync } from "zlib";
|
|
4893
5160
|
function chunkLine(row, declared) {
|
|
4894
5161
|
if (declared !== null) return dumps(row.row) + "\n";
|
|
@@ -4897,14 +5164,20 @@ function chunkLine(row, declared) {
|
|
|
4897
5164
|
tagged.set("row", row.row);
|
|
4898
5165
|
return dumps(tagged) + "\n";
|
|
4899
5166
|
}
|
|
4900
|
-
async function* buildChunks(rows, plan) {
|
|
5167
|
+
async function* buildChunks(rows, plan, ordinalStart = 0) {
|
|
4901
5168
|
let lines = [];
|
|
4902
5169
|
let rawBytes = 0;
|
|
4903
|
-
let ordinal =
|
|
5170
|
+
let ordinal = ordinalStart;
|
|
4904
5171
|
let seen = 0;
|
|
4905
5172
|
const flush2 = () => {
|
|
4906
5173
|
const raw = Buffer.concat(lines);
|
|
4907
|
-
const built = finishChunk(
|
|
5174
|
+
const built = finishChunk(
|
|
5175
|
+
ordinal++,
|
|
5176
|
+
raw,
|
|
5177
|
+
lines.length,
|
|
5178
|
+
plan.entity,
|
|
5179
|
+
seen - lines.length
|
|
5180
|
+
);
|
|
4908
5181
|
lines = [];
|
|
4909
5182
|
rawBytes = 0;
|
|
4910
5183
|
return built;
|
|
@@ -4923,12 +5196,12 @@ function finishChunk(ordinal, raw, rows, entity, firstRow = 0) {
|
|
|
4923
5196
|
const gz = gzipDeterministic(raw, { name: chunkName(ordinal) });
|
|
4924
5197
|
return {
|
|
4925
5198
|
ordinal,
|
|
4926
|
-
digest:
|
|
5199
|
+
digest: createHash7("sha256").update(gz).digest("hex"),
|
|
4927
5200
|
bytes: gz.length,
|
|
4928
5201
|
rows,
|
|
4929
5202
|
entity,
|
|
4930
5203
|
rawBytes: raw.length,
|
|
4931
|
-
contentSha256:
|
|
5204
|
+
contentSha256: createHash7("sha256").update(raw).digest("hex"),
|
|
4932
5205
|
firstRow,
|
|
4933
5206
|
gz
|
|
4934
5207
|
};
|
|
@@ -4936,7 +5209,18 @@ function finishChunk(ordinal, raw, rows, entity, firstRow = 0) {
|
|
|
4936
5209
|
function gzipDeterministic(raw, opts = {}) {
|
|
4937
5210
|
const level = opts.level ?? GZIP_LEVEL;
|
|
4938
5211
|
const name = opts.name ? Buffer.concat([Buffer.from(opts.name, "latin1"), Buffer.from([0])]) : Buffer.alloc(0);
|
|
4939
|
-
const header = Buffer.from([
|
|
5212
|
+
const header = Buffer.from([
|
|
5213
|
+
31,
|
|
5214
|
+
139,
|
|
5215
|
+
8,
|
|
5216
|
+
opts.name ? 8 : 0,
|
|
5217
|
+
0,
|
|
5218
|
+
0,
|
|
5219
|
+
0,
|
|
5220
|
+
0,
|
|
5221
|
+
level === 9 ? 2 : level === 1 ? 4 : 0,
|
|
5222
|
+
255
|
|
5223
|
+
]);
|
|
4940
5224
|
const body = deflateRawSync(raw, { level });
|
|
4941
5225
|
const trailer = Buffer.alloc(8);
|
|
4942
5226
|
trailer.writeUInt32LE(crc32(raw), 0);
|
|
@@ -4953,7 +5237,8 @@ function crc32(data) {
|
|
|
4953
5237
|
}
|
|
4954
5238
|
}
|
|
4955
5239
|
let crc = 4294967295;
|
|
4956
|
-
for (let i = 0; i < data.length; i++)
|
|
5240
|
+
for (let i = 0; i < data.length; i++)
|
|
5241
|
+
crc = table[(crc ^ data[i]) & 255] ^ crc >>> 8;
|
|
4957
5242
|
return (crc ^ 4294967295) >>> 0;
|
|
4958
5243
|
}
|
|
4959
5244
|
var DEFAULT_CHUNK_BYTES, GZIP_LEVEL, chunkName, table;
|
|
@@ -5011,9 +5296,9 @@ var init_push = __esm({
|
|
|
5011
5296
|
});
|
|
5012
5297
|
|
|
5013
5298
|
// src/worlds/ingest.ts
|
|
5014
|
-
import * as
|
|
5299
|
+
import * as fs10 from "fs";
|
|
5015
5300
|
import * as os3 from "os";
|
|
5016
|
-
import * as
|
|
5301
|
+
import * as path9 from "path";
|
|
5017
5302
|
function parseToolCallRecord(raw) {
|
|
5018
5303
|
const none = { record: null, skip: null };
|
|
5019
5304
|
if (!isObject2(raw)) return { ...none, refusal: "record is not an object" };
|
|
@@ -5034,7 +5319,7 @@ function parseToolCallRecord(raw) {
|
|
|
5034
5319
|
}
|
|
5035
5320
|
function readToolCallRecords(paths) {
|
|
5036
5321
|
const records = [];
|
|
5037
|
-
for (const file of paths) records.push(...toolCallRecordsIn(
|
|
5322
|
+
for (const file of paths) records.push(...toolCallRecordsIn(fs10.readFileSync(file, "utf8"), file));
|
|
5038
5323
|
return records;
|
|
5039
5324
|
}
|
|
5040
5325
|
function toolCallRecordsIn(text2, where) {
|
|
@@ -5070,7 +5355,7 @@ async function emit(sink, record) {
|
|
|
5070
5355
|
return;
|
|
5071
5356
|
}
|
|
5072
5357
|
if (typeof sink === "string") {
|
|
5073
|
-
|
|
5358
|
+
fs10.appendFileSync(sink, JSON.stringify(record) + "\n");
|
|
5074
5359
|
return;
|
|
5075
5360
|
}
|
|
5076
5361
|
if (typeof sink === "function") {
|
|
@@ -5101,14 +5386,6 @@ function captureToolCalls(tools, sink, opts = {}) {
|
|
|
5101
5386
|
}
|
|
5102
5387
|
return wrapped;
|
|
5103
5388
|
}
|
|
5104
|
-
function jsonish(value) {
|
|
5105
|
-
if (typeof value !== "string") return value;
|
|
5106
|
-
try {
|
|
5107
|
-
return JSON.parse(value);
|
|
5108
|
-
} catch {
|
|
5109
|
-
return value;
|
|
5110
|
-
}
|
|
5111
|
-
}
|
|
5112
5389
|
function toolCallsFromObservations(observations, opts = {}) {
|
|
5113
5390
|
const known = opts.tools ? new Set(opts.tools) : null;
|
|
5114
5391
|
const sinceMs = opts.since ? Date.parse(opts.since) : NaN;
|
|
@@ -5135,10 +5412,9 @@ function toolCallsFromObservations(observations, opts = {}) {
|
|
|
5135
5412
|
out.unknownTools[tool] = (out.unknownTools[tool] ?? 0) + 1;
|
|
5136
5413
|
continue;
|
|
5137
5414
|
}
|
|
5138
|
-
const
|
|
5139
|
-
const record = { tool, args: isObject2(input) ? input : {} };
|
|
5415
|
+
const record = { tool, args: isObject2(observation.input) ? observation.input : {} };
|
|
5140
5416
|
if (observation.level === "ERROR") record.error = observation.statusMessage || "ERROR";
|
|
5141
|
-
else record.result =
|
|
5417
|
+
else record.result = observation.output;
|
|
5142
5418
|
if (at !== void 0) record.at = at;
|
|
5143
5419
|
if (opts.session) record.session = opts.session;
|
|
5144
5420
|
record.source = `trace:${observation.traceId ?? "?"}/${observation.id ?? "?"}`;
|
|
@@ -5159,7 +5435,7 @@ async function ingest(world, records, options = {}) {
|
|
|
5159
5435
|
const mode = options.mode ?? "append";
|
|
5160
5436
|
const dryRun = Boolean(options.dryRun);
|
|
5161
5437
|
const creds = credentialsOf(options);
|
|
5162
|
-
const local =
|
|
5438
|
+
const local = fs10.existsSync(world) && fs10.statSync(world).isDirectory();
|
|
5163
5439
|
const map = options.map === void 0 ? void 0 : options.map;
|
|
5164
5440
|
let rows = null;
|
|
5165
5441
|
let stderr = "";
|
|
@@ -5175,9 +5451,9 @@ async function ingest(world, records, options = {}) {
|
|
|
5175
5451
|
if (!creds) throw new Error("Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey).");
|
|
5176
5452
|
const { pull: pull2 } = await Promise.resolve().then(() => (init_hub(), hub_exports));
|
|
5177
5453
|
const slug = world.split("@", 1)[0];
|
|
5178
|
-
const scratch =
|
|
5454
|
+
const scratch = fs10.mkdtempSync(path9.join(os3.tmpdir(), "gateway-ingest-"));
|
|
5179
5455
|
try {
|
|
5180
|
-
const pulled = await pull2(world,
|
|
5456
|
+
const pulled = await pull2(world, path9.join(scratch, slug), options);
|
|
5181
5457
|
let report;
|
|
5182
5458
|
let projected;
|
|
5183
5459
|
if (rows === null) {
|
|
@@ -5198,7 +5474,7 @@ async function ingest(world, records, options = {}) {
|
|
|
5198
5474
|
if (status.entityCounts) report.entityCounts = status.entityCounts;
|
|
5199
5475
|
return report;
|
|
5200
5476
|
} finally {
|
|
5201
|
-
|
|
5477
|
+
fs10.rmSync(scratch, { recursive: true, force: true });
|
|
5202
5478
|
}
|
|
5203
5479
|
}
|
|
5204
5480
|
async function ingestToolCalls(world, records, options = {}) {
|
|
@@ -5261,6 +5537,58 @@ var init_ingest = __esm({
|
|
|
5261
5537
|
}
|
|
5262
5538
|
});
|
|
5263
5539
|
|
|
5540
|
+
// src/worlds/sandbox.ts
|
|
5541
|
+
function parseSlugRef(slugRef) {
|
|
5542
|
+
const at = slugRef.indexOf("@");
|
|
5543
|
+
if (at === -1) return { slug: slugRef };
|
|
5544
|
+
return { slug: slugRef.slice(0, at), ref: slugRef.slice(at + 1) };
|
|
5545
|
+
}
|
|
5546
|
+
var sid, WorldSandboxClient;
|
|
5547
|
+
var init_sandbox = __esm({
|
|
5548
|
+
"src/worlds/sandbox.ts"() {
|
|
5549
|
+
"use strict";
|
|
5550
|
+
init_worlds();
|
|
5551
|
+
sid = encodeURIComponent;
|
|
5552
|
+
WorldSandboxClient = class {
|
|
5553
|
+
constructor(options = {}) {
|
|
5554
|
+
this.options = options;
|
|
5555
|
+
}
|
|
5556
|
+
open(target) {
|
|
5557
|
+
return apiRequest(this.options, "POST", "/api/public/benchmark-containers/sandboxes", target);
|
|
5558
|
+
}
|
|
5559
|
+
status(sandboxId) {
|
|
5560
|
+
return apiRequest(this.options, "GET", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}`);
|
|
5561
|
+
}
|
|
5562
|
+
async read(sandboxId, filePath) {
|
|
5563
|
+
const res = await apiRequest(
|
|
5564
|
+
this.options,
|
|
5565
|
+
"GET",
|
|
5566
|
+
`/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/file?path=${encodeURIComponent(filePath)}`
|
|
5567
|
+
);
|
|
5568
|
+
return { path: res.path, size: res.size, bytes: Buffer.from(res.contentBase64, "base64") };
|
|
5569
|
+
}
|
|
5570
|
+
write(sandboxId, filePath, bytes) {
|
|
5571
|
+
return apiRequest(this.options, "PUT", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/file`, {
|
|
5572
|
+
path: filePath,
|
|
5573
|
+
contentBase64: bytes.toString("base64")
|
|
5574
|
+
});
|
|
5575
|
+
}
|
|
5576
|
+
exec(sandboxId, input) {
|
|
5577
|
+
return apiRequest(this.options, "POST", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/exec`, input);
|
|
5578
|
+
}
|
|
5579
|
+
check(sandboxId) {
|
|
5580
|
+
return apiRequest(this.options, "POST", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/check`, {});
|
|
5581
|
+
}
|
|
5582
|
+
commit(sandboxId, input) {
|
|
5583
|
+
return apiRequest(this.options, "POST", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}/commit`, input);
|
|
5584
|
+
}
|
|
5585
|
+
close(sandboxId) {
|
|
5586
|
+
return apiRequest(this.options, "DELETE", `/api/public/benchmark-containers/sandboxes/${sid(sandboxId)}`);
|
|
5587
|
+
}
|
|
5588
|
+
};
|
|
5589
|
+
}
|
|
5590
|
+
});
|
|
5591
|
+
|
|
5264
5592
|
// src/worlds/index.ts
|
|
5265
5593
|
var worlds_exports = {};
|
|
5266
5594
|
__export(worlds_exports, {
|
|
@@ -5273,6 +5601,7 @@ __export(worlds_exports, {
|
|
|
5273
5601
|
EVENT_FLUSH_INTERVAL_MS: () => EVENT_FLUSH_INTERVAL_MS,
|
|
5274
5602
|
EVENT_MAX_RETRIES: () => EVENT_MAX_RETRIES,
|
|
5275
5603
|
GatewayEvalsClient: () => GatewayEvalsClient,
|
|
5604
|
+
PushSlugConflict: () => PushSlugConflict,
|
|
5276
5605
|
STRIP_PREFIX: () => STRIP_PREFIX,
|
|
5277
5606
|
SURFACE_KINDS: () => SURFACE_KINDS,
|
|
5278
5607
|
TRANSFORMS: () => TRANSFORMS,
|
|
@@ -5280,6 +5609,7 @@ __export(worlds_exports, {
|
|
|
5280
5609
|
TaskSessions: () => TaskSessions,
|
|
5281
5610
|
WHITESPACE: () => WHITESPACE,
|
|
5282
5611
|
WorldBrowser: () => WorldBrowser,
|
|
5612
|
+
WorldSandboxClient: () => WorldSandboxClient,
|
|
5283
5613
|
WorldSession: () => WorldSession,
|
|
5284
5614
|
WorldSessionCallError: () => WorldSessionCallError,
|
|
5285
5615
|
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
@@ -5289,9 +5619,11 @@ __export(worlds_exports, {
|
|
|
5289
5619
|
aggregateTaskGrade: () => aggregateTaskGrade,
|
|
5290
5620
|
apiRequest: () => apiRequest,
|
|
5291
5621
|
applyTransforms: () => applyTransforms,
|
|
5622
|
+
archiveMode: () => archiveMode,
|
|
5292
5623
|
attachManifest: () => attachManifest,
|
|
5293
5624
|
attachSession: () => attachSession,
|
|
5294
5625
|
attachTask: () => attachTask,
|
|
5626
|
+
buildFileManifest: () => buildFileManifest,
|
|
5295
5627
|
buildSourceTarball: () => buildSourceTarball,
|
|
5296
5628
|
captureToolCalls: () => captureToolCalls,
|
|
5297
5629
|
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
@@ -5303,12 +5635,12 @@ __export(worlds_exports, {
|
|
|
5303
5635
|
deleteTask: () => deleteTask,
|
|
5304
5636
|
describeWorld: () => describeWorld,
|
|
5305
5637
|
detectProvenance: () => detectProvenance,
|
|
5638
|
+
extractBundle: () => extractBundle,
|
|
5306
5639
|
getRun: () => getRun,
|
|
5307
5640
|
getTask: () => getTask,
|
|
5308
5641
|
hashForm: () => hashForm,
|
|
5309
5642
|
ingest: () => ingest,
|
|
5310
5643
|
ingestToolCalls: () => ingestToolCalls,
|
|
5311
|
-
isIncludedInContentHash: () => isIncludedInContentHash,
|
|
5312
5644
|
isTaskIdRef: () => isTaskIdRef,
|
|
5313
5645
|
isTaskManifest: () => isTaskManifest,
|
|
5314
5646
|
isTaskSpec: () => isTaskSpec,
|
|
@@ -5324,6 +5656,8 @@ __export(worlds_exports, {
|
|
|
5324
5656
|
openSession: () => openSession,
|
|
5325
5657
|
openTask: () => openTask,
|
|
5326
5658
|
openWorld: () => openWorld,
|
|
5659
|
+
openWorldVersion: () => openWorldVersion,
|
|
5660
|
+
parseSlugRef: () => parseSlugRef,
|
|
5327
5661
|
parseToolCallRecord: () => parseToolCallRecord,
|
|
5328
5662
|
parseTransform: () => parseTransform,
|
|
5329
5663
|
parseWorldTaskRefs: () => parseWorldTaskRefs,
|
|
@@ -5340,6 +5674,7 @@ __export(worlds_exports, {
|
|
|
5340
5674
|
redactActionArgs: () => redactActionArgs,
|
|
5341
5675
|
redactionOf: () => redactionOf,
|
|
5342
5676
|
resolveBrowserMode: () => resolveBrowserMode,
|
|
5677
|
+
resolvePushSlug: () => resolvePushSlug,
|
|
5343
5678
|
roundForm: () => roundForm,
|
|
5344
5679
|
runSessions: () => runSessions,
|
|
5345
5680
|
runTask: () => runTask,
|
|
@@ -5374,9 +5709,9 @@ function resolveEnv(options) {
|
|
|
5374
5709
|
}
|
|
5375
5710
|
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
5376
5711
|
}
|
|
5377
|
-
async function apiRequest(options, method,
|
|
5712
|
+
async function apiRequest(options, method, path10, body) {
|
|
5378
5713
|
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
5379
|
-
const res = await fetch(`${host}${
|
|
5714
|
+
const res = await fetch(`${host}${path10}`, {
|
|
5380
5715
|
method,
|
|
5381
5716
|
headers: {
|
|
5382
5717
|
"content-type": "application/json",
|
|
@@ -5385,7 +5720,7 @@ async function apiRequest(options, method, path9, body) {
|
|
|
5385
5720
|
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
5386
5721
|
});
|
|
5387
5722
|
const text2 = await res.text();
|
|
5388
|
-
if (!res.ok) throw new ApiError2(res.status, method,
|
|
5723
|
+
if (!res.ok) throw new ApiError2(res.status, method, path10, text2.slice(0, 400));
|
|
5389
5724
|
return JSON.parse(text2);
|
|
5390
5725
|
}
|
|
5391
5726
|
async function setEnvSecret(slug, key, value, options = {}) {
|
|
@@ -5514,9 +5849,21 @@ async function openWorld(slugRef, options = {}) {
|
|
|
5514
5849
|
const at = slugRef.indexOf("@");
|
|
5515
5850
|
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
5516
5851
|
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
5852
|
+
return resolveWorld(slugRef, slug, ref, ref ? { ref } : {}, options);
|
|
5853
|
+
}
|
|
5854
|
+
async function openWorldVersion(slug, versionId, options = {}) {
|
|
5855
|
+
return resolveWorld(
|
|
5856
|
+
`${slug}@${versionId}`,
|
|
5857
|
+
slug,
|
|
5858
|
+
null,
|
|
5859
|
+
{ versionId },
|
|
5860
|
+
options
|
|
5861
|
+
);
|
|
5862
|
+
}
|
|
5863
|
+
async function resolveWorld(slugRef, slug, ref, pin, options) {
|
|
5517
5864
|
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
5518
5865
|
slug,
|
|
5519
|
-
...
|
|
5866
|
+
...pin
|
|
5520
5867
|
});
|
|
5521
5868
|
const world = new WorldSet(
|
|
5522
5869
|
slugRef,
|
|
@@ -5554,9 +5901,10 @@ var init_worlds = __esm({
|
|
|
5554
5901
|
init_query();
|
|
5555
5902
|
init_migrate();
|
|
5556
5903
|
init_ingest();
|
|
5904
|
+
init_sandbox();
|
|
5557
5905
|
ApiError2 = class extends Error {
|
|
5558
|
-
constructor(status, method,
|
|
5559
|
-
super(`${method} ${
|
|
5906
|
+
constructor(status, method, path10, body) {
|
|
5907
|
+
super(`${method} ${path10} -> ${status}: ${body}`);
|
|
5560
5908
|
this.status = status;
|
|
5561
5909
|
this.name = "ApiError";
|
|
5562
5910
|
}
|
|
@@ -5636,6 +5984,7 @@ export {
|
|
|
5636
5984
|
EVENT_FLUSH_INTERVAL_MS,
|
|
5637
5985
|
EVENT_MAX_RETRIES,
|
|
5638
5986
|
GatewayEvalsClient,
|
|
5987
|
+
PushSlugConflict,
|
|
5639
5988
|
STRIP_PREFIX,
|
|
5640
5989
|
SURFACE_KINDS,
|
|
5641
5990
|
TRANSFORMS,
|
|
@@ -5643,6 +5992,7 @@ export {
|
|
|
5643
5992
|
TaskSessions,
|
|
5644
5993
|
WHITESPACE,
|
|
5645
5994
|
WorldBrowser,
|
|
5995
|
+
WorldSandboxClient,
|
|
5646
5996
|
WorldSession,
|
|
5647
5997
|
WorldSessionCallError,
|
|
5648
5998
|
WorldSessionTimeout,
|
|
@@ -5652,9 +6002,11 @@ export {
|
|
|
5652
6002
|
aggregateTaskGrade,
|
|
5653
6003
|
apiRequest,
|
|
5654
6004
|
applyTransforms,
|
|
6005
|
+
archiveMode,
|
|
5655
6006
|
attachManifest,
|
|
5656
6007
|
attachSession,
|
|
5657
6008
|
attachTask,
|
|
6009
|
+
buildFileManifest,
|
|
5658
6010
|
buildSourceTarball,
|
|
5659
6011
|
captureToolCalls,
|
|
5660
6012
|
collectArchiveRelPaths,
|
|
@@ -5666,12 +6018,12 @@ export {
|
|
|
5666
6018
|
deleteTask,
|
|
5667
6019
|
describeWorld,
|
|
5668
6020
|
detectProvenance,
|
|
6021
|
+
extractBundle,
|
|
5669
6022
|
getRun,
|
|
5670
6023
|
getTask,
|
|
5671
6024
|
hashForm,
|
|
5672
6025
|
ingest,
|
|
5673
6026
|
ingestToolCalls,
|
|
5674
|
-
isIncludedInContentHash,
|
|
5675
6027
|
isTaskIdRef,
|
|
5676
6028
|
isTaskManifest,
|
|
5677
6029
|
isTaskSpec,
|
|
@@ -5687,6 +6039,8 @@ export {
|
|
|
5687
6039
|
openSession,
|
|
5688
6040
|
openTask,
|
|
5689
6041
|
openWorld,
|
|
6042
|
+
openWorldVersion,
|
|
6043
|
+
parseSlugRef,
|
|
5690
6044
|
parseToolCallRecord,
|
|
5691
6045
|
parseTransform,
|
|
5692
6046
|
parseWorldTaskRefs,
|
|
@@ -5703,6 +6057,7 @@ export {
|
|
|
5703
6057
|
redactActionArgs,
|
|
5704
6058
|
redactionOf,
|
|
5705
6059
|
resolveBrowserMode,
|
|
6060
|
+
resolvePushSlug,
|
|
5706
6061
|
roundForm,
|
|
5707
6062
|
runSessions,
|
|
5708
6063
|
runTask,
|