@withgateway/sdk 0.1.13 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -4
- package/dist/cli.d.mts +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +1489 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.mjs +1476 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/index.d.mts +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1213 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1170 -54
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +27 -2
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +27 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +508 -0
- package/dist/worlds/index.d.ts +508 -0
- package/dist/worlds/index.js +1120 -0
- package/dist/worlds/index.js.map +1 -0
- package/dist/worlds/index.mjs +1104 -0
- package/dist/worlds/index.mjs.map +1 -0
- package/package.json +27 -2
package/dist/index.js
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
2
3
|
var __defProp = Object.defineProperty;
|
|
3
4
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
5
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __esm = (fn, res) => function __init() {
|
|
9
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
10
|
+
};
|
|
6
11
|
var __commonJS = (cb, mod) => function __require() {
|
|
7
12
|
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
8
13
|
};
|
|
@@ -18,14 +23,1074 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
18
23
|
}
|
|
19
24
|
return to;
|
|
20
25
|
};
|
|
26
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
27
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
28
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
29
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
30
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
31
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
32
|
+
mod
|
|
33
|
+
));
|
|
21
34
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
22
35
|
|
|
36
|
+
// src/worlds/session.ts
|
|
37
|
+
var session_exports = {};
|
|
38
|
+
__export(session_exports, {
|
|
39
|
+
WorldSession: () => WorldSession,
|
|
40
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
41
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
42
|
+
attachSession: () => attachSession,
|
|
43
|
+
openSession: () => openSession,
|
|
44
|
+
runSessions: () => runSessions,
|
|
45
|
+
schemaFromSignature: () => schemaFromSignature
|
|
46
|
+
});
|
|
47
|
+
function schemaFromSignature(spec) {
|
|
48
|
+
const inside = /\(([\s\S]*)\)\s*$/.exec(spec.signature.trim())?.[1] ?? "";
|
|
49
|
+
const properties = {};
|
|
50
|
+
const required = [];
|
|
51
|
+
const params = [];
|
|
52
|
+
let depth = 0;
|
|
53
|
+
let current = "";
|
|
54
|
+
for (const ch of inside) {
|
|
55
|
+
if ("([{".includes(ch)) depth++;
|
|
56
|
+
if (")]}".includes(ch)) depth--;
|
|
57
|
+
if (ch === "," && depth === 0) {
|
|
58
|
+
params.push(current);
|
|
59
|
+
current = "";
|
|
60
|
+
} else current += ch;
|
|
61
|
+
}
|
|
62
|
+
if (current.trim()) params.push(current);
|
|
63
|
+
for (const raw of params) {
|
|
64
|
+
const param = raw.trim();
|
|
65
|
+
if (!param || param.startsWith("*") || param === "self") continue;
|
|
66
|
+
const eq = param.indexOf("=");
|
|
67
|
+
const name = (eq === -1 ? param : param.slice(0, eq)).split(":")[0].trim();
|
|
68
|
+
if (!name) continue;
|
|
69
|
+
if (eq === -1) {
|
|
70
|
+
properties[name] = { type: "string" };
|
|
71
|
+
required.push(name);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
const value = param.slice(eq + 1).trim();
|
|
75
|
+
if (/^-?\d+(\.\d+)?$/.test(value)) properties[name] = { type: "number" };
|
|
76
|
+
else if (value === "True" || value === "False")
|
|
77
|
+
properties[name] = { type: "boolean" };
|
|
78
|
+
else properties[name] = { type: "string" };
|
|
79
|
+
}
|
|
80
|
+
return {
|
|
81
|
+
type: "function",
|
|
82
|
+
function: {
|
|
83
|
+
name: spec.name,
|
|
84
|
+
description: spec.doc ?? spec.name,
|
|
85
|
+
parameters: { type: "object", properties, required }
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
async function attachSession(sessionId, options = {}) {
|
|
90
|
+
const state2 = await apiRequest(
|
|
91
|
+
options,
|
|
92
|
+
"GET",
|
|
93
|
+
`/api/public/world-sessions/${encodeURIComponent(sessionId)}`
|
|
94
|
+
);
|
|
95
|
+
return new WorldSession(sessionId, state2, options);
|
|
96
|
+
}
|
|
97
|
+
async function openSession(slugOrRef, task, opts = {}) {
|
|
98
|
+
const options = opts.options ?? {};
|
|
99
|
+
const at = slugOrRef.indexOf("@");
|
|
100
|
+
const slug = at === -1 ? slugOrRef : slugOrRef.slice(0, at);
|
|
101
|
+
const state2 = await apiRequest(
|
|
102
|
+
options,
|
|
103
|
+
"POST",
|
|
104
|
+
"/api/public/world-sessions",
|
|
105
|
+
{
|
|
106
|
+
...opts.versionId ? { versionId: opts.versionId } : { slug },
|
|
107
|
+
task,
|
|
108
|
+
...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm }
|
|
109
|
+
}
|
|
110
|
+
);
|
|
111
|
+
return new WorldSession(state2.sessionId, state2, options);
|
|
112
|
+
}
|
|
113
|
+
async function runSessions(opts) {
|
|
114
|
+
const options = opts.options ?? {};
|
|
115
|
+
const concurrency = Math.max(1, opts.concurrency ?? 3);
|
|
116
|
+
let tasks = opts.tasks;
|
|
117
|
+
let versionId = opts.versionId;
|
|
118
|
+
if (!tasks || !versionId) {
|
|
119
|
+
const { openWorld: openWorld2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
|
|
120
|
+
const world = await openWorld2(opts.world, options);
|
|
121
|
+
tasks = tasks ?? world.tasks.map((t) => t.name);
|
|
122
|
+
versionId = versionId ?? world.version.versionId;
|
|
123
|
+
}
|
|
124
|
+
const results = [];
|
|
125
|
+
let next = 0;
|
|
126
|
+
await Promise.all(
|
|
127
|
+
Array.from({ length: Math.min(concurrency, tasks.length) }, async () => {
|
|
128
|
+
while (next < tasks.length) {
|
|
129
|
+
const task = tasks[next++];
|
|
130
|
+
let session = null;
|
|
131
|
+
try {
|
|
132
|
+
session = await openSession(opts.world, task, {
|
|
133
|
+
versionId,
|
|
134
|
+
options
|
|
135
|
+
});
|
|
136
|
+
await session.ready();
|
|
137
|
+
const toolkit = await session.toolkit();
|
|
138
|
+
await opts.agent({
|
|
139
|
+
session,
|
|
140
|
+
toolkit,
|
|
141
|
+
instruction: session.instruction ?? task,
|
|
142
|
+
task
|
|
143
|
+
});
|
|
144
|
+
const graded = await session.grade();
|
|
145
|
+
const result = {
|
|
146
|
+
task,
|
|
147
|
+
sessionId: session.sessionId,
|
|
148
|
+
reward: graded.reward,
|
|
149
|
+
rewards: graded.rewards
|
|
150
|
+
};
|
|
151
|
+
results.push(result);
|
|
152
|
+
opts.onResult?.(result);
|
|
153
|
+
} catch (error) {
|
|
154
|
+
const result = {
|
|
155
|
+
task,
|
|
156
|
+
sessionId: session?.sessionId ?? "",
|
|
157
|
+
reward: 0,
|
|
158
|
+
rewards: {},
|
|
159
|
+
// An Error with an empty message would read as "no error" to any
|
|
160
|
+
// caller checking truthiness; name the type instead.
|
|
161
|
+
error: (error instanceof Error ? error.message : String(error)) || (error instanceof Error ? error.name : "Unknown error")
|
|
162
|
+
};
|
|
163
|
+
results.push(result);
|
|
164
|
+
opts.onResult?.(result);
|
|
165
|
+
} finally {
|
|
166
|
+
await session?.close({ keepWarm: opts.keepWarm ?? true }).catch(() => void 0);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
})
|
|
170
|
+
);
|
|
171
|
+
const mean = results.reduce((sum, r) => sum + (r.reward ?? 0), 0) / (results.length || 1);
|
|
172
|
+
return {
|
|
173
|
+
results: results.sort((a, b) => a.task.localeCompare(b.task)),
|
|
174
|
+
mean,
|
|
175
|
+
passed: opts.failUnder === void 0 ? true : mean >= opts.failUnder
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
var READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, sleep2, WorldSessionTimeout, WorldSessionCallError, WorldSession;
|
|
179
|
+
var init_session = __esm({
|
|
180
|
+
"src/worlds/session.ts"() {
|
|
181
|
+
"use strict";
|
|
182
|
+
init_worlds();
|
|
183
|
+
READY_TIMEOUT_MS = 10 * 6e4;
|
|
184
|
+
READY_POLL_MS = 2e3;
|
|
185
|
+
CALL_TIMEOUT_MS = 3 * 6e4;
|
|
186
|
+
CALL_POLL_MS = 400;
|
|
187
|
+
sleep2 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
188
|
+
WorldSessionTimeout = class extends Error {
|
|
189
|
+
};
|
|
190
|
+
WorldSessionCallError = class extends Error {
|
|
191
|
+
};
|
|
192
|
+
WorldSession = class {
|
|
193
|
+
constructor(sessionId, state2, options) {
|
|
194
|
+
this.sessionId = sessionId;
|
|
195
|
+
this.state = state2;
|
|
196
|
+
this.options = options;
|
|
197
|
+
}
|
|
198
|
+
get task() {
|
|
199
|
+
return this.state.taskName;
|
|
200
|
+
}
|
|
201
|
+
/** What this task asks for — the bundle's own instruction.md. */
|
|
202
|
+
get instruction() {
|
|
203
|
+
return this.state.instruction ?? null;
|
|
204
|
+
}
|
|
205
|
+
get versionId() {
|
|
206
|
+
return this.state.versionId;
|
|
207
|
+
}
|
|
208
|
+
/** The session's latest platform state (one request, no waiting). */
|
|
209
|
+
async status() {
|
|
210
|
+
const fresh = await apiRequest(
|
|
211
|
+
this.options,
|
|
212
|
+
"GET",
|
|
213
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`
|
|
214
|
+
);
|
|
215
|
+
this.state = {
|
|
216
|
+
instruction: this.state.instruction,
|
|
217
|
+
tools: this.state.tools,
|
|
218
|
+
...fresh
|
|
219
|
+
};
|
|
220
|
+
return this.state;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Wait until the container is warm — meaning the process inside it has
|
|
224
|
+
* asked the platform for work, not merely that a job was accepted.
|
|
225
|
+
*/
|
|
226
|
+
async ready(opts = {}) {
|
|
227
|
+
const deadline = Date.now() + (opts.timeoutMs ?? READY_TIMEOUT_MS);
|
|
228
|
+
for (; ; ) {
|
|
229
|
+
const state2 = await this.status();
|
|
230
|
+
if (state2.ready) return state2;
|
|
231
|
+
if (state2.state === "FAILED" || state2.state === "STOPPED")
|
|
232
|
+
throw new WorldSessionCallError(
|
|
233
|
+
`World session ${state2.state.toLowerCase()} before it was ready.`
|
|
234
|
+
);
|
|
235
|
+
if (Date.now() > deadline)
|
|
236
|
+
throw new WorldSessionTimeout(
|
|
237
|
+
`World session still ${state2.state} after ${Math.round((opts.timeoutMs ?? READY_TIMEOUT_MS) / 1e3)}s.`
|
|
238
|
+
);
|
|
239
|
+
await sleep2(READY_POLL_MS);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
/** Enqueue one interaction and wait for the container's answer. */
|
|
243
|
+
async exchange(body, timeoutMs = CALL_TIMEOUT_MS) {
|
|
244
|
+
const { callId } = await apiRequest(
|
|
245
|
+
this.options,
|
|
246
|
+
"POST",
|
|
247
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls`,
|
|
248
|
+
body
|
|
249
|
+
);
|
|
250
|
+
const deadline = Date.now() + timeoutMs;
|
|
251
|
+
for (; ; ) {
|
|
252
|
+
const call = await apiRequest(
|
|
253
|
+
this.options,
|
|
254
|
+
"GET",
|
|
255
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls/${encodeURIComponent(callId)}`
|
|
256
|
+
);
|
|
257
|
+
if (call.done) {
|
|
258
|
+
if (call.error) throw new WorldSessionCallError(call.error);
|
|
259
|
+
return call.result;
|
|
260
|
+
}
|
|
261
|
+
if (Date.now() > deadline)
|
|
262
|
+
throw new WorldSessionTimeout(
|
|
263
|
+
`Call ${callId} still ${call.state} after ${Math.round(timeoutMs / 1e3)}s \u2014 the container may have died.`
|
|
264
|
+
);
|
|
265
|
+
await sleep2(CALL_POLL_MS);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Run one of the world's tools INSIDE the container. The arguments and the
|
|
270
|
+
* result are the world's own contract — this layer neither validates nor
|
|
271
|
+
* reshapes them, so a tool error reads exactly as it would in a hosted run.
|
|
272
|
+
*/
|
|
273
|
+
async call(tool, args = {}, opts = {}) {
|
|
274
|
+
return this.exchange({ kind: "call", tool, args }, opts.timeoutMs);
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* THE WORLD'S TOOLS, READY TO HAND TO A MODEL. Schemas come from the
|
|
278
|
+
* world's own signatures and implementations call into this session, so a
|
|
279
|
+
* caller never writes — and never has to re-sync — a tool map of their own:
|
|
280
|
+
*
|
|
281
|
+
* const { schemas, impls } = await session.toolkit();
|
|
282
|
+
* await runMyAgent(impls, schemas, session.instruction);
|
|
283
|
+
*
|
|
284
|
+
* A tool added to the world shows up here on the next run; a tool renamed
|
|
285
|
+
* in the world renames here too, instead of failing at call time against a
|
|
286
|
+
* hand-written map that still remembers the old name.
|
|
287
|
+
*/
|
|
288
|
+
async toolkit() {
|
|
289
|
+
let specs = this.state.tools;
|
|
290
|
+
if (!specs) specs = (await this.status()).tools ?? [];
|
|
291
|
+
const impls = {};
|
|
292
|
+
for (const spec of specs) {
|
|
293
|
+
impls[spec.name] = (args = {}) => this.call(spec.name, args);
|
|
294
|
+
}
|
|
295
|
+
return { schemas: specs.map(schemaFromSignature), impls, specs };
|
|
296
|
+
}
|
|
297
|
+
/** Run the task's grader against the container's END STATE. */
|
|
298
|
+
async grade(opts = {}) {
|
|
299
|
+
const raw = await this.exchange(
|
|
300
|
+
{ kind: "grade" },
|
|
301
|
+
opts.timeoutMs
|
|
302
|
+
);
|
|
303
|
+
const record = raw && typeof raw === "object" ? raw : {};
|
|
304
|
+
const rewards = {};
|
|
305
|
+
for (const [key, value] of Object.entries(record)) {
|
|
306
|
+
if (key !== "reward" && typeof value === "number") rewards[key] = value;
|
|
307
|
+
}
|
|
308
|
+
return {
|
|
309
|
+
reward: typeof record.reward === "number" ? record.reward : null,
|
|
310
|
+
rewards,
|
|
311
|
+
raw
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* End the session. `keepWarm` hands the container back to the pool instead
|
|
316
|
+
* of tearing it down, so the next session for the same task starts warm —
|
|
317
|
+
* the pooled instance still obeys the fleet's idle TTL and pinning.
|
|
318
|
+
*/
|
|
319
|
+
async close(opts = {}) {
|
|
320
|
+
return apiRequest(
|
|
321
|
+
this.options,
|
|
322
|
+
"DELETE",
|
|
323
|
+
`/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`,
|
|
324
|
+
{ keepWarm: opts.keepWarm ?? false }
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
// src/worlds/hub.ts
|
|
332
|
+
function isExcludedDirComponent(component) {
|
|
333
|
+
return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
|
|
334
|
+
}
|
|
335
|
+
function isIncludedInContentHash(relPath, includeAllRootFiles) {
|
|
336
|
+
const parts = relPath.split("/");
|
|
337
|
+
const name = parts[parts.length - 1];
|
|
338
|
+
const dirParts = parts.slice(0, -1);
|
|
339
|
+
if (name.startsWith(".")) return false;
|
|
340
|
+
if (parts.includes("__pycache__")) return false;
|
|
341
|
+
if (dirParts.length === 0) {
|
|
342
|
+
return includeAllRootFiles || name === "README.md" || name === "pyproject.toml" || name === "gateway-env.toml" || name.endsWith(".py");
|
|
343
|
+
}
|
|
344
|
+
return !dirParts.some(isExcludedDirComponent);
|
|
345
|
+
}
|
|
346
|
+
function collectArchiveRelPaths(dir) {
|
|
347
|
+
if (fs.existsSync(path.join(dir, ".gitignore"))) {
|
|
348
|
+
throw new Error(
|
|
349
|
+
"This bundle has a root .gitignore, which the Python CLI honors via gitignore_parser. Reimplementing that imperfectly here would silently produce a different content hash, so the TS CLI refuses. Remove the .gitignore (or bake its effect into the tree) and retry."
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
const out = [];
|
|
353
|
+
const includeAllRootFiles = fs.existsSync(path.join(dir, "gateway-env.toml"));
|
|
354
|
+
const walk = (rel) => {
|
|
355
|
+
const abs = rel ? path.join(dir, rel) : dir;
|
|
356
|
+
for (const entry of fs.readdirSync(abs, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
357
|
+
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
358
|
+
if (entry.isDirectory()) {
|
|
359
|
+
if (!isExcludedDirComponent(entry.name)) walk(childRel);
|
|
360
|
+
} else if (entry.isFile()) {
|
|
361
|
+
if (isIncludedInContentHash(childRel, includeAllRootFiles)) {
|
|
362
|
+
out.push(childRel);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
};
|
|
367
|
+
walk("");
|
|
368
|
+
return out.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
369
|
+
}
|
|
370
|
+
function computeContentHash(dir) {
|
|
371
|
+
const hasher = (0, import_node_crypto.createHash)("sha256");
|
|
372
|
+
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
373
|
+
hasher.update(Buffer.from(`file:${relPath}`, "utf-8"));
|
|
374
|
+
hasher.update(fs.readFileSync(path.join(dir, relPath)));
|
|
375
|
+
}
|
|
376
|
+
return hasher.digest("hex");
|
|
377
|
+
}
|
|
378
|
+
function octal(value, length) {
|
|
379
|
+
return Buffer.from(value.toString(8).padStart(length - 1, "0") + "\0", "ascii");
|
|
380
|
+
}
|
|
381
|
+
function tarHeader(name, size) {
|
|
382
|
+
const header = Buffer.alloc(512);
|
|
383
|
+
header.write(name, 0, 100, "utf-8");
|
|
384
|
+
octal(420, 8).copy(header, 100);
|
|
385
|
+
octal(0, 8).copy(header, 108);
|
|
386
|
+
octal(0, 8).copy(header, 116);
|
|
387
|
+
octal(size, 12).copy(header, 124);
|
|
388
|
+
octal(0, 12).copy(header, 136);
|
|
389
|
+
header.write(" ", 148, "ascii");
|
|
390
|
+
header.write("0", 156, "ascii");
|
|
391
|
+
header.write("ustar", 257, "ascii");
|
|
392
|
+
header.write("00", 263, "ascii");
|
|
393
|
+
let sum = 0;
|
|
394
|
+
for (const byte of header) sum += byte;
|
|
395
|
+
header.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
|
|
396
|
+
return header;
|
|
397
|
+
}
|
|
398
|
+
function tarMember(name, payload) {
|
|
399
|
+
const parts = [];
|
|
400
|
+
const nameBytes = Buffer.byteLength(name, "utf-8");
|
|
401
|
+
if (nameBytes > 100) {
|
|
402
|
+
const body = Buffer.from(name + "\0", "utf-8");
|
|
403
|
+
const lHeader = tarHeader("././@LongLink", body.length);
|
|
404
|
+
lHeader.write("L", 156, "ascii");
|
|
405
|
+
let sum = 0;
|
|
406
|
+
lHeader.write(" ", 148, "ascii");
|
|
407
|
+
for (const byte of lHeader) sum += byte;
|
|
408
|
+
lHeader.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
|
|
409
|
+
parts.push(lHeader, body);
|
|
410
|
+
const pad2 = body.length % 512 === 0 ? 0 : 512 - body.length % 512;
|
|
411
|
+
if (pad2) parts.push(Buffer.alloc(pad2));
|
|
412
|
+
parts.push(tarHeader(name.slice(0, 100), payload.length));
|
|
413
|
+
} else {
|
|
414
|
+
parts.push(tarHeader(name, payload.length));
|
|
415
|
+
}
|
|
416
|
+
parts.push(payload);
|
|
417
|
+
const pad = payload.length % 512 === 0 ? 0 : 512 - payload.length % 512;
|
|
418
|
+
if (pad) parts.push(Buffer.alloc(pad));
|
|
419
|
+
return parts;
|
|
420
|
+
}
|
|
421
|
+
function buildSourceTarball(dir) {
|
|
422
|
+
const parts = [];
|
|
423
|
+
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
424
|
+
parts.push(...tarMember(relPath, fs.readFileSync(path.join(dir, relPath))));
|
|
425
|
+
}
|
|
426
|
+
parts.push(Buffer.alloc(1024));
|
|
427
|
+
return (0, import_node_zlib.gzipSync)(Buffer.concat(parts), { level: 9 });
|
|
428
|
+
}
|
|
429
|
+
function readLocalMeta(dir) {
|
|
430
|
+
try {
|
|
431
|
+
const raw = fs.readFileSync(path.join(dir, META_RELPATH), "utf-8");
|
|
432
|
+
const parsed = JSON.parse(raw);
|
|
433
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
434
|
+
} catch {
|
|
435
|
+
return null;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
function writeLocalMeta(dir, meta) {
|
|
439
|
+
fs.mkdirSync(path.join(dir, ".gateway"), { recursive: true });
|
|
440
|
+
fs.writeFileSync(path.join(dir, META_RELPATH), JSON.stringify(meta, null, 2));
|
|
441
|
+
}
|
|
442
|
+
function snapshotTreeHashes(dir) {
|
|
443
|
+
const out = {};
|
|
444
|
+
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
445
|
+
out[relPath] = (0, import_node_crypto.createHash)("sha256").update(fs.readFileSync(path.join(dir, relPath))).digest("hex");
|
|
446
|
+
}
|
|
447
|
+
return out;
|
|
448
|
+
}
|
|
449
|
+
function localTreeState(dir) {
|
|
450
|
+
const meta = readLocalMeta(dir);
|
|
451
|
+
const base = meta?.baseFiles;
|
|
452
|
+
if (!base || typeof base !== "object") {
|
|
453
|
+
return { meta, dirty: [], clean: false };
|
|
454
|
+
}
|
|
455
|
+
const now = snapshotTreeHashes(dir);
|
|
456
|
+
const dirty = /* @__PURE__ */ new Set();
|
|
457
|
+
for (const [p, h] of Object.entries(now)) if (base[p] !== h) dirty.add(p);
|
|
458
|
+
for (const p of Object.keys(base)) if (!(p in now)) dirty.add(p);
|
|
459
|
+
return { meta, dirty: [...dirty].sort(), clean: dirty.size === 0 };
|
|
460
|
+
}
|
|
461
|
+
function readPyproject(dir) {
|
|
462
|
+
try {
|
|
463
|
+
const text = fs.readFileSync(path.join(dir, "pyproject.toml"), "utf-8");
|
|
464
|
+
const section = /\[project\]([\s\S]*?)(\n\[|$)/.exec(text)?.[1] ?? "";
|
|
465
|
+
const name = /\bname\s*=\s*"([^"]+)"/.exec(section)?.[1];
|
|
466
|
+
const version = /\bversion\s*=\s*"([^"]+)"/.exec(section)?.[1];
|
|
467
|
+
return { name, version };
|
|
468
|
+
} catch {
|
|
469
|
+
return {};
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
function buildFileManifest(dir) {
|
|
473
|
+
const entries = [];
|
|
474
|
+
const walk = (rel) => {
|
|
475
|
+
const abs = rel ? path.join(dir, rel) : dir;
|
|
476
|
+
for (const entry of fs.readdirSync(abs, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
477
|
+
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
478
|
+
if (entry.isDirectory()) {
|
|
479
|
+
if (!SKIP_DIRS.has(entry.name) && !entry.name.endsWith(".egg-info")) {
|
|
480
|
+
walk(childRel);
|
|
481
|
+
}
|
|
482
|
+
} else if (entry.isFile()) {
|
|
483
|
+
const payload = fs.readFileSync(path.join(dir, childRel));
|
|
484
|
+
entries.push({
|
|
485
|
+
path: childRel,
|
|
486
|
+
isDirectory: false,
|
|
487
|
+
size: payload.length,
|
|
488
|
+
sha256: (0, import_node_crypto.createHash)("sha256").update(payload).digest("hex")
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
};
|
|
493
|
+
walk("");
|
|
494
|
+
return entries;
|
|
495
|
+
}
|
|
496
|
+
async function putObject(url, data, shaB64) {
|
|
497
|
+
const resp = await fetch(url, {
|
|
498
|
+
method: "PUT",
|
|
499
|
+
headers: {
|
|
500
|
+
"Content-Type": "application/gzip",
|
|
501
|
+
"x-amz-checksum-sha256": shaB64
|
|
502
|
+
},
|
|
503
|
+
body: new Uint8Array(data)
|
|
504
|
+
});
|
|
505
|
+
if (!resp.ok) {
|
|
506
|
+
throw new Error(`object PUT -> ${resp.status}: ${(await resp.text()).slice(0, 300)}`);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
async function push(dir, opts = {}) {
|
|
510
|
+
const options = opts.options ?? {};
|
|
511
|
+
if (!fs.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
512
|
+
const localMeta = readLocalMeta(dir) ?? {};
|
|
513
|
+
const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
|
|
514
|
+
const baseVersionId = localMeta.versionId ?? null;
|
|
515
|
+
const contentHash = computeContentHash(dir);
|
|
516
|
+
const pyproject = readPyproject(dir);
|
|
517
|
+
const slug = opts.slug ?? pyproject.name ?? path.basename(path.resolve(dir));
|
|
518
|
+
const resolve2 = await apiRequest(
|
|
519
|
+
options,
|
|
520
|
+
"POST",
|
|
521
|
+
"/api/public/benchmark-containers/resolve",
|
|
522
|
+
{ slug, name: pyproject.name ?? slug }
|
|
523
|
+
);
|
|
524
|
+
const tarball = buildSourceTarball(dir);
|
|
525
|
+
const bundleSha256 = sha256b64(tarball);
|
|
526
|
+
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/versions", {
|
|
527
|
+
containerId: resolve2.containerId,
|
|
528
|
+
contentHash,
|
|
529
|
+
bundleSha256,
|
|
530
|
+
bundleSizeBytes: tarball.length,
|
|
531
|
+
...pyproject.version ? { semanticVersion: pyproject.version } : {},
|
|
532
|
+
pyprojectMeta: pyproject,
|
|
533
|
+
fileManifest: buildFileManifest(dir),
|
|
534
|
+
branch,
|
|
535
|
+
baseVersionId,
|
|
536
|
+
...opts.force ? { force: true } : {},
|
|
537
|
+
...opts.message ? { changeReason: opts.message } : {},
|
|
538
|
+
...opts.author ? { author: opts.author } : {}
|
|
539
|
+
});
|
|
540
|
+
if (version.uploadUrl) await putObject(version.uploadUrl, tarball, bundleSha256);
|
|
541
|
+
await apiRequest(
|
|
542
|
+
options,
|
|
543
|
+
"POST",
|
|
544
|
+
`/api/public/benchmark-containers/versions/${version.versionId}/finalize`,
|
|
545
|
+
{ branch, baseVersionId, ...opts.force ? { force: true } : {} }
|
|
546
|
+
);
|
|
547
|
+
writeLocalMeta(dir, {
|
|
548
|
+
...localMeta,
|
|
549
|
+
slug,
|
|
550
|
+
ref: branch,
|
|
551
|
+
branch,
|
|
552
|
+
versionId: version.versionId,
|
|
553
|
+
contentHash,
|
|
554
|
+
semanticVersion: pyproject.version,
|
|
555
|
+
forkedFromVersionId: baseVersionId,
|
|
556
|
+
baseFiles: snapshotTreeHashes(dir)
|
|
557
|
+
});
|
|
558
|
+
return { slug, versionId: version.versionId, contentHash, branch };
|
|
559
|
+
}
|
|
560
|
+
function extractBundle(bundle, dir) {
|
|
561
|
+
const tar = (0, import_node_zlib.gunzipSync)(bundle);
|
|
562
|
+
let offset = 0;
|
|
563
|
+
let longName = null;
|
|
564
|
+
let written = 0;
|
|
565
|
+
while (offset + 512 <= tar.length) {
|
|
566
|
+
const header = tar.subarray(offset, offset + 512);
|
|
567
|
+
offset += 512;
|
|
568
|
+
if (header.every((b) => b === 0)) break;
|
|
569
|
+
const rawName = header.subarray(0, 100).toString("utf-8").replace(/\0.*$/, "");
|
|
570
|
+
const size = parseInt(header.subarray(124, 136).toString("ascii").replace(/\0.*$/, "").trim(), 8) || 0;
|
|
571
|
+
const typeflag = header.subarray(156, 157).toString("ascii");
|
|
572
|
+
const body = tar.subarray(offset, offset + size);
|
|
573
|
+
offset += size + (size % 512 === 0 ? 0 : 512 - size % 512);
|
|
574
|
+
if (typeflag === "L") {
|
|
575
|
+
longName = body.toString("utf-8").replace(/\0.*$/, "");
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
const name = (longName ?? rawName).replace(/^\.\//, "");
|
|
579
|
+
longName = null;
|
|
580
|
+
if (typeflag !== "0" && typeflag !== "") continue;
|
|
581
|
+
if (name.startsWith("/") || name.split("/").includes("..")) continue;
|
|
582
|
+
const target = path.join(dir, name);
|
|
583
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
584
|
+
fs.writeFileSync(target, body);
|
|
585
|
+
written++;
|
|
586
|
+
}
|
|
587
|
+
return written;
|
|
588
|
+
}
|
|
589
|
+
async function pull(slugRef, target, options = {}) {
|
|
590
|
+
const at = slugRef.indexOf("@");
|
|
591
|
+
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
592
|
+
const ref = at === -1 ? "latest" : slugRef.slice(at + 1);
|
|
593
|
+
const details = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
594
|
+
slug,
|
|
595
|
+
ref
|
|
596
|
+
});
|
|
597
|
+
if (!details.downloadUrl) {
|
|
598
|
+
throw new Error(`${slug}@${ref} resolved but has no uploaded bundle`);
|
|
599
|
+
}
|
|
600
|
+
let dir = target ?? path.join(process.cwd(), slug);
|
|
601
|
+
if (!target) {
|
|
602
|
+
let i = 1;
|
|
603
|
+
const base = dir;
|
|
604
|
+
while (fs.existsSync(dir)) dir = `${base}-${i++}`;
|
|
605
|
+
}
|
|
606
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
607
|
+
const resp = await fetch(details.downloadUrl);
|
|
608
|
+
if (!resp.ok) throw new Error(`bundle download -> ${resp.status}`);
|
|
609
|
+
const bundle = Buffer.from(await resp.arrayBuffer());
|
|
610
|
+
if (details.bundleSha256 && sha256b64(bundle) !== details.bundleSha256) {
|
|
611
|
+
throw new Error(
|
|
612
|
+
`bundle integrity check failed for ${slug}@${ref}: downloaded sha256 != recorded bundleSha256`
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
const files = extractBundle(bundle, dir);
|
|
616
|
+
const branch = ref === "latest" || /^\d|^[0-9a-f]{8,}$/.test(ref) ? DEFAULT_BRANCH : ref;
|
|
617
|
+
writeLocalMeta(dir, {
|
|
618
|
+
slug,
|
|
619
|
+
ref,
|
|
620
|
+
branch,
|
|
621
|
+
versionId: details.versionId,
|
|
622
|
+
contentHash: details.contentHash,
|
|
623
|
+
semanticVersion: details.semanticVersion ?? void 0,
|
|
624
|
+
baseFiles: snapshotTreeHashes(dir)
|
|
625
|
+
});
|
|
626
|
+
return { slug, dir, versionId: details.versionId, files };
|
|
627
|
+
}
|
|
628
|
+
async function propose(dir, opts) {
|
|
629
|
+
const meta = readLocalMeta(dir);
|
|
630
|
+
if (!meta?.slug || !meta.baseFiles) {
|
|
631
|
+
throw new Error("This dir has no pull/push base \u2014 run pull first, then edit, then propose.");
|
|
632
|
+
}
|
|
633
|
+
const state2 = localTreeState(dir);
|
|
634
|
+
if (state2.clean) throw new Error("Nothing to propose \u2014 the tree matches its base.");
|
|
635
|
+
const changes = state2.dirty.map((relPath) => {
|
|
636
|
+
const abs = path.join(dir, relPath);
|
|
637
|
+
if (!fs.existsSync(abs)) return { path: relPath, action: "delete" };
|
|
638
|
+
return {
|
|
639
|
+
path: relPath,
|
|
640
|
+
action: "write",
|
|
641
|
+
newContent: fs.readFileSync(abs, "utf-8")
|
|
642
|
+
};
|
|
643
|
+
});
|
|
644
|
+
const res = await apiRequest(
|
|
645
|
+
opts.options ?? {},
|
|
646
|
+
"POST",
|
|
647
|
+
"/api/public/benchmark-containers/proposals",
|
|
648
|
+
{
|
|
649
|
+
slug: meta.slug,
|
|
650
|
+
title: opts.title,
|
|
651
|
+
changeReason: opts.reason,
|
|
652
|
+
baseVersionId: meta.versionId,
|
|
653
|
+
...opts.targetBranch ? { targetBranch: opts.targetBranch } : {},
|
|
654
|
+
changes
|
|
655
|
+
}
|
|
656
|
+
);
|
|
657
|
+
return res;
|
|
658
|
+
}
|
|
659
|
+
async function createRef(slug, name, kind, versionId, options = {}) {
|
|
660
|
+
return apiRequest(options, "POST", "/api/public/benchmark-containers/refs", {
|
|
661
|
+
slug,
|
|
662
|
+
name,
|
|
663
|
+
kind,
|
|
664
|
+
...versionId ? { versionId } : {}
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
var import_node_crypto, import_node_zlib, fs, path, DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, sha256b64;
|
|
668
|
+
var init_hub = __esm({
|
|
669
|
+
"src/worlds/hub.ts"() {
|
|
670
|
+
"use strict";
|
|
671
|
+
import_node_crypto = require("crypto");
|
|
672
|
+
import_node_zlib = require("zlib");
|
|
673
|
+
fs = __toESM(require("fs"));
|
|
674
|
+
path = __toESM(require("path"));
|
|
675
|
+
init_worlds();
|
|
676
|
+
DEFAULT_BRANCH = "main";
|
|
677
|
+
META_RELPATH = path.join(".gateway", ".env-metadata.json");
|
|
678
|
+
SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
679
|
+
".git",
|
|
680
|
+
"__pycache__",
|
|
681
|
+
"node_modules",
|
|
682
|
+
".venv",
|
|
683
|
+
".mypy_cache",
|
|
684
|
+
"dist",
|
|
685
|
+
"build",
|
|
686
|
+
"outputs"
|
|
687
|
+
]);
|
|
688
|
+
ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
689
|
+
sha256b64 = (data) => (0, import_node_crypto.createHash)("sha256").update(data).digest("base64");
|
|
690
|
+
}
|
|
691
|
+
});
|
|
692
|
+
|
|
693
|
+
// src/worlds/validate.ts
|
|
694
|
+
function fmtScore(score) {
|
|
695
|
+
return score === null ? "no score" : score.toFixed(3);
|
|
696
|
+
}
|
|
697
|
+
function buildMarkdown(report, minScore) {
|
|
698
|
+
const lines = [];
|
|
699
|
+
lines.push(
|
|
700
|
+
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
701
|
+
);
|
|
702
|
+
lines.push("");
|
|
703
|
+
lines.push(
|
|
704
|
+
`Ran the pull request's own world source (version \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
705
|
+
);
|
|
706
|
+
lines.push("");
|
|
707
|
+
lines.push("| Model | Status | Score | Run |");
|
|
708
|
+
lines.push("| --- | --- | ---: | --- |");
|
|
709
|
+
for (const run of report.runs) {
|
|
710
|
+
const r = run.report;
|
|
711
|
+
const link = r?.url ? `[open](${r.url})` : "\u2014";
|
|
712
|
+
const status = run.failure ? `\u274C ${run.failure}` : `\u2705 ${r?.status ?? "COMPLETED"}`;
|
|
713
|
+
lines.push(
|
|
714
|
+
`| \`${run.model}\` | ${status} | ${fmtScore(r?.score ?? null)} | ${link} |`
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
const taskRows = report.runs.flatMap(
|
|
718
|
+
(run) => (run.report?.samples ?? []).map((sample) => ({
|
|
719
|
+
model: run.model,
|
|
720
|
+
task: sample.task ?? "?",
|
|
721
|
+
reward: sample.reward
|
|
722
|
+
}))
|
|
723
|
+
);
|
|
724
|
+
if (taskRows.length > 0 && taskRows.length <= 40) {
|
|
725
|
+
lines.push("");
|
|
726
|
+
lines.push("<details><summary>Per-task rewards</summary>");
|
|
727
|
+
lines.push("");
|
|
728
|
+
lines.push("| Task | Model | Reward |");
|
|
729
|
+
lines.push("| --- | --- | ---: |");
|
|
730
|
+
for (const row of taskRows) {
|
|
731
|
+
lines.push(
|
|
732
|
+
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
733
|
+
);
|
|
734
|
+
}
|
|
735
|
+
lines.push("");
|
|
736
|
+
lines.push("</details>");
|
|
737
|
+
}
|
|
738
|
+
return lines.join("\n");
|
|
739
|
+
}
|
|
740
|
+
async function validateWorld(dir, opts) {
|
|
741
|
+
const options = opts.options ?? {};
|
|
742
|
+
const progress = opts.onProgress ?? (() => {
|
|
743
|
+
});
|
|
744
|
+
const minScore = opts.minScore ?? null;
|
|
745
|
+
const provenance = detectProvenance();
|
|
746
|
+
const branch = opts.branch ?? provenance.branch;
|
|
747
|
+
if (!branch)
|
|
748
|
+
throw new Error(
|
|
749
|
+
"No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
|
|
750
|
+
);
|
|
751
|
+
progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
|
|
752
|
+
const pushed = await push(dir, {
|
|
753
|
+
slug: opts.slug,
|
|
754
|
+
branch,
|
|
755
|
+
message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
|
|
756
|
+
...provenance.actor?.name ? { author: provenance.actor.name } : {},
|
|
757
|
+
options
|
|
758
|
+
});
|
|
759
|
+
progress(`pushed version ${pushed.versionId} (${pushed.contentHash.slice(0, 12)})`);
|
|
760
|
+
const configs = await listRunConfigs(options);
|
|
761
|
+
const runConfig = configs.find((c) => c.id === opts.runConfig) ?? configs.find((c) => c.name === opts.runConfig);
|
|
762
|
+
if (!runConfig) {
|
|
763
|
+
const known = configs.map((c) => c.name).join(", ") || "(none)";
|
|
764
|
+
throw new Error(
|
|
765
|
+
`Run config "${opts.runConfig}" not found in this project. Have: ${known}`
|
|
766
|
+
);
|
|
767
|
+
}
|
|
768
|
+
progress(`dispatching run config "${runConfig.name}" (${runConfig.models.join(", ")})\u2026`);
|
|
769
|
+
const dispatched = await apiRequest(options, "POST", "/api/public/benchmark-containers/run-configs/dispatch", {
|
|
770
|
+
runConfigId: runConfig.id,
|
|
771
|
+
versionId: pushed.versionId,
|
|
772
|
+
...provenance
|
|
773
|
+
});
|
|
774
|
+
const runs = [];
|
|
775
|
+
for (const run of dispatched.runs) {
|
|
776
|
+
if (!run.ok) {
|
|
777
|
+
runs.push({
|
|
778
|
+
model: run.model,
|
|
779
|
+
runId: run.runId,
|
|
780
|
+
report: null,
|
|
781
|
+
failure: `dispatch failed: ${run.error ?? "unknown error"}`
|
|
782
|
+
});
|
|
783
|
+
continue;
|
|
784
|
+
}
|
|
785
|
+
progress(`waiting on ${run.model} (run ${run.runId})\u2026`);
|
|
786
|
+
let report = null;
|
|
787
|
+
let failure = null;
|
|
788
|
+
try {
|
|
789
|
+
report = await waitForRun(run.runId, {
|
|
790
|
+
timeoutMinutes: opts.timeoutMinutes ?? 45,
|
|
791
|
+
options,
|
|
792
|
+
onTick: (r) => progress(
|
|
793
|
+
`\u2026${run.model}: ${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
|
|
794
|
+
)
|
|
795
|
+
});
|
|
796
|
+
if (report.status.toUpperCase() !== "COMPLETED") {
|
|
797
|
+
failure = report.status;
|
|
798
|
+
} else if (minScore !== null) {
|
|
799
|
+
if (report.score === null) failure = "completed without a score";
|
|
800
|
+
else if (report.score < minScore)
|
|
801
|
+
failure = `score ${fmtScore(report.score)} < ${minScore}`;
|
|
802
|
+
}
|
|
803
|
+
} catch (error) {
|
|
804
|
+
failure = error instanceof Error ? error.message : String(error);
|
|
805
|
+
}
|
|
806
|
+
runs.push({ model: run.model, runId: run.runId, report, failure });
|
|
807
|
+
}
|
|
808
|
+
const base = {
|
|
809
|
+
ok: runs.length > 0 && runs.every((r) => r.failure === null),
|
|
810
|
+
slug: pushed.slug,
|
|
811
|
+
branch,
|
|
812
|
+
versionId: pushed.versionId,
|
|
813
|
+
contentHash: pushed.contentHash,
|
|
814
|
+
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
815
|
+
runs
|
|
816
|
+
};
|
|
817
|
+
return { ...base, markdown: buildMarkdown(base, minScore) };
|
|
818
|
+
}
|
|
819
|
+
var init_validate = __esm({
|
|
820
|
+
"src/worlds/validate.ts"() {
|
|
821
|
+
"use strict";
|
|
822
|
+
init_worlds();
|
|
823
|
+
init_hub();
|
|
824
|
+
}
|
|
825
|
+
});
|
|
826
|
+
|
|
827
|
+
// src/worlds/index.ts
|
|
828
|
+
var worlds_exports = {};
|
|
829
|
+
__export(worlds_exports, {
|
|
830
|
+
DEFAULT_BRANCH: () => DEFAULT_BRANCH,
|
|
831
|
+
WorldSession: () => WorldSession,
|
|
832
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
833
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
834
|
+
WorldSet: () => WorldSet,
|
|
835
|
+
WorldTask: () => WorldTask,
|
|
836
|
+
apiRequest: () => apiRequest,
|
|
837
|
+
attachSession: () => attachSession,
|
|
838
|
+
buildSourceTarball: () => buildSourceTarball,
|
|
839
|
+
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
840
|
+
computeContentHash: () => computeContentHash,
|
|
841
|
+
createRef: () => createRef,
|
|
842
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
843
|
+
detectProvenance: () => detectProvenance,
|
|
844
|
+
getRun: () => getRun,
|
|
845
|
+
isIncludedInContentHash: () => isIncludedInContentHash,
|
|
846
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
847
|
+
listRunConfigs: () => listRunConfigs,
|
|
848
|
+
localTreeState: () => localTreeState,
|
|
849
|
+
openSession: () => openSession,
|
|
850
|
+
openWorld: () => openWorld,
|
|
851
|
+
propose: () => propose,
|
|
852
|
+
pull: () => pull,
|
|
853
|
+
push: () => push,
|
|
854
|
+
readLocalMeta: () => readLocalMeta,
|
|
855
|
+
runSessions: () => runSessions,
|
|
856
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
857
|
+
setEnvSecret: () => setEnvSecret,
|
|
858
|
+
snapshotTreeHashes: () => snapshotTreeHashes,
|
|
859
|
+
validateWorld: () => validateWorld,
|
|
860
|
+
waitForRun: () => waitForRun,
|
|
861
|
+
writeLocalMeta: () => writeLocalMeta
|
|
862
|
+
});
|
|
863
|
+
function resolveEnv(options) {
|
|
864
|
+
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
865
|
+
const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
866
|
+
const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
867
|
+
if (!publicKey || !secretKey) {
|
|
868
|
+
throw new Error(
|
|
869
|
+
"Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
|
|
870
|
+
);
|
|
871
|
+
}
|
|
872
|
+
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
873
|
+
}
|
|
874
|
+
async function apiRequest(options, method, path2, body) {
|
|
875
|
+
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
876
|
+
const res = await fetch(`${host}${path2}`, {
|
|
877
|
+
method,
|
|
878
|
+
headers: {
|
|
879
|
+
"content-type": "application/json",
|
|
880
|
+
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
881
|
+
},
|
|
882
|
+
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
883
|
+
});
|
|
884
|
+
const text = await res.text();
|
|
885
|
+
if (!res.ok) {
|
|
886
|
+
throw new Error(
|
|
887
|
+
`${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
|
|
888
|
+
);
|
|
889
|
+
}
|
|
890
|
+
return JSON.parse(text);
|
|
891
|
+
}
|
|
892
|
+
async function setEnvSecret(slug, key, value, options = {}) {
|
|
893
|
+
return apiRequest(
|
|
894
|
+
options,
|
|
895
|
+
"PUT",
|
|
896
|
+
"/api/public/benchmark-containers/secrets",
|
|
897
|
+
{
|
|
898
|
+
slug,
|
|
899
|
+
key,
|
|
900
|
+
value
|
|
901
|
+
}
|
|
902
|
+
);
|
|
903
|
+
}
|
|
904
|
+
async function listEnvSecretKeys(slug, options = {}) {
|
|
905
|
+
const res = await apiRequest(
|
|
906
|
+
options,
|
|
907
|
+
"GET",
|
|
908
|
+
`/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
|
|
909
|
+
);
|
|
910
|
+
return res.secrets;
|
|
911
|
+
}
|
|
912
|
+
async function deleteEnvSecret(slug, key, options = {}) {
|
|
913
|
+
await apiRequest(
|
|
914
|
+
options,
|
|
915
|
+
"DELETE",
|
|
916
|
+
"/api/public/benchmark-containers/secrets",
|
|
917
|
+
{
|
|
918
|
+
slug,
|
|
919
|
+
key
|
|
920
|
+
}
|
|
921
|
+
);
|
|
922
|
+
}
|
|
923
|
+
async function listRunConfigs(options = {}) {
|
|
924
|
+
const res = await apiRequest(
|
|
925
|
+
options,
|
|
926
|
+
"GET",
|
|
927
|
+
"/api/public/benchmark-containers/run-configs"
|
|
928
|
+
);
|
|
929
|
+
return res.runConfigs;
|
|
930
|
+
}
|
|
931
|
+
async function getRun(runId, options = {}) {
|
|
932
|
+
return apiRequest(
|
|
933
|
+
options,
|
|
934
|
+
"GET",
|
|
935
|
+
`/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
async function waitForRun(runId, opts = {}) {
|
|
939
|
+
const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
|
|
940
|
+
for (; ; ) {
|
|
941
|
+
const run = await getRun(runId, opts.options ?? {});
|
|
942
|
+
if (run.terminal) return run;
|
|
943
|
+
if (Date.now() > deadline)
|
|
944
|
+
throw new Error(
|
|
945
|
+
`Run ${runId} still ${run.status} after the wait timeout.`
|
|
946
|
+
);
|
|
947
|
+
opts.onTick?.(run);
|
|
948
|
+
await new Promise(
|
|
949
|
+
(resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
|
|
950
|
+
);
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
function detectProvenance() {
|
|
954
|
+
const env = globalThis.process?.env ?? {};
|
|
955
|
+
const out = {};
|
|
956
|
+
const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
957
|
+
if (ref) out.branch = ref;
|
|
958
|
+
if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
|
|
959
|
+
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
960
|
+
if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
|
|
961
|
+
if (env.GITHUB_ACTOR) {
|
|
962
|
+
out.actor = {
|
|
963
|
+
kind: "ci",
|
|
964
|
+
id: env.GITHUB_ACTOR,
|
|
965
|
+
name: env.GITHUB_ACTOR,
|
|
966
|
+
// GitHub serves every account's avatar at /<login>.png.
|
|
967
|
+
imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
|
|
968
|
+
};
|
|
969
|
+
}
|
|
970
|
+
if (!out.branch || !out.commit) {
|
|
971
|
+
try {
|
|
972
|
+
const { execFileSync } = require("child_process");
|
|
973
|
+
const git = (args) => execFileSync("git", args, {
|
|
974
|
+
encoding: "utf-8",
|
|
975
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
976
|
+
}).toString().trim();
|
|
977
|
+
out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
978
|
+
out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
|
|
979
|
+
if (!out.repoUrl) {
|
|
980
|
+
const remote = git(["config", "--get", "remote.origin.url"]);
|
|
981
|
+
if (remote) out.repoUrl = remote.replace(/\.git$/, "");
|
|
982
|
+
}
|
|
983
|
+
if (!out.actor) {
|
|
984
|
+
const name = git(["config", "--get", "user.name"]);
|
|
985
|
+
if (name) out.actor = { kind: "user", name };
|
|
986
|
+
}
|
|
987
|
+
} catch {
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
return out;
|
|
991
|
+
}
|
|
992
|
+
async function openWorld(slugRef, options = {}) {
|
|
993
|
+
const at = slugRef.indexOf("@");
|
|
994
|
+
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
995
|
+
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
996
|
+
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
997
|
+
slug,
|
|
998
|
+
...ref ? { ref } : {}
|
|
999
|
+
});
|
|
1000
|
+
const world = new WorldSet(
|
|
1001
|
+
slugRef,
|
|
1002
|
+
slug,
|
|
1003
|
+
ref,
|
|
1004
|
+
{
|
|
1005
|
+
versionId: version.versionId,
|
|
1006
|
+
semanticVersion: version.semanticVersion ?? null,
|
|
1007
|
+
contentHash: version.contentHash,
|
|
1008
|
+
state: version.state
|
|
1009
|
+
},
|
|
1010
|
+
[],
|
|
1011
|
+
options
|
|
1012
|
+
);
|
|
1013
|
+
for (const name of version.taskNames ?? []) {
|
|
1014
|
+
const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
|
|
1015
|
+
world.tasks.push(new WorldTask(world, name, variant));
|
|
1016
|
+
}
|
|
1017
|
+
return world;
|
|
1018
|
+
}
|
|
1019
|
+
var WorldTask, WorldSet;
|
|
1020
|
+
var init_worlds = __esm({
|
|
1021
|
+
"src/worlds/index.ts"() {
|
|
1022
|
+
"use strict";
|
|
1023
|
+
init_session();
|
|
1024
|
+
init_hub();
|
|
1025
|
+
init_validate();
|
|
1026
|
+
WorldTask = class {
|
|
1027
|
+
constructor(world, name, variant) {
|
|
1028
|
+
this.world = world;
|
|
1029
|
+
this.name = name;
|
|
1030
|
+
this.variant = variant;
|
|
1031
|
+
}
|
|
1032
|
+
};
|
|
1033
|
+
WorldSet = class {
|
|
1034
|
+
constructor(slugRef, slug, ref, version, tasks, options) {
|
|
1035
|
+
this.slugRef = slugRef;
|
|
1036
|
+
this.slug = slug;
|
|
1037
|
+
this.ref = ref;
|
|
1038
|
+
this.version = version;
|
|
1039
|
+
this.tasks = tasks;
|
|
1040
|
+
this.options = options;
|
|
1041
|
+
}
|
|
1042
|
+
task(variant) {
|
|
1043
|
+
const hit = this.tasks.find(
|
|
1044
|
+
(t) => t.variant === variant || t.name.includes(variant)
|
|
1045
|
+
);
|
|
1046
|
+
if (!hit) {
|
|
1047
|
+
const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
|
|
1048
|
+
throw new Error(`no task matches ${variant}; have: ${known}`);
|
|
1049
|
+
}
|
|
1050
|
+
return hit;
|
|
1051
|
+
}
|
|
1052
|
+
/**
|
|
1053
|
+
* Open a LIVE session against one of this world's tasks: the platform boots
|
|
1054
|
+
* (or reuses) a warm container, and your agent calls its tools through the
|
|
1055
|
+
* platform. See worlds/session.ts.
|
|
1056
|
+
*/
|
|
1057
|
+
async open(task, opts = {}) {
|
|
1058
|
+
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
1059
|
+
return openSession2(this.slug, task, {
|
|
1060
|
+
versionId: this.version.versionId,
|
|
1061
|
+
reuseWarm: opts.reuseWarm,
|
|
1062
|
+
options: this.options
|
|
1063
|
+
});
|
|
1064
|
+
}
|
|
1065
|
+
/**
|
|
1066
|
+
* Start a HOSTED run of a run config against this world — the platform
|
|
1067
|
+
* executes the (Python) world in parallel containers per the config's
|
|
1068
|
+
* `parallelism` and files one graded run per model. Returns those runs.
|
|
1069
|
+
*/
|
|
1070
|
+
async dispatch(runConfigId, opts = {}) {
|
|
1071
|
+
const provenance = opts.provenance ?? detectProvenance();
|
|
1072
|
+
return apiRequest(
|
|
1073
|
+
this.options,
|
|
1074
|
+
"POST",
|
|
1075
|
+
"/api/public/benchmark-containers/run-configs/dispatch",
|
|
1076
|
+
{
|
|
1077
|
+
runConfigId,
|
|
1078
|
+
...opts.containerId ? { containerId: opts.containerId } : {},
|
|
1079
|
+
versionId: opts.versionId ?? this.version.versionId,
|
|
1080
|
+
...provenance
|
|
1081
|
+
}
|
|
1082
|
+
);
|
|
1083
|
+
}
|
|
1084
|
+
};
|
|
1085
|
+
}
|
|
1086
|
+
});
|
|
1087
|
+
|
|
23
1088
|
// package.json
|
|
24
1089
|
var require_package = __commonJS({
|
|
25
1090
|
"package.json"(exports2, module2) {
|
|
26
1091
|
module2.exports = {
|
|
27
1092
|
name: "@withgateway/sdk",
|
|
28
|
-
version: "0.1.
|
|
1093
|
+
version: "0.1.16",
|
|
29
1094
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
30
1095
|
main: "./dist/index.js",
|
|
31
1096
|
module: "./dist/index.mjs",
|
|
@@ -85,6 +1150,11 @@ var require_package = __commonJS({
|
|
|
85
1150
|
types: "./dist/scores/index.d.ts",
|
|
86
1151
|
import: "./dist/scores/index.mjs",
|
|
87
1152
|
require: "./dist/scores/index.js"
|
|
1153
|
+
},
|
|
1154
|
+
"./worlds": {
|
|
1155
|
+
types: "./dist/worlds/index.d.ts",
|
|
1156
|
+
import: "./dist/worlds/index.mjs",
|
|
1157
|
+
require: "./dist/worlds/index.js"
|
|
88
1158
|
}
|
|
89
1159
|
},
|
|
90
1160
|
files: [
|
|
@@ -98,6 +1168,7 @@ var require_package = __commonJS({
|
|
|
98
1168
|
build: "tsup",
|
|
99
1169
|
dev: "tsup --watch",
|
|
100
1170
|
typecheck: "tsc --noEmit",
|
|
1171
|
+
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
101
1172
|
test: "vitest",
|
|
102
1173
|
"test:run": "vitest run",
|
|
103
1174
|
lint: "eslint src --ext .ts",
|
|
@@ -158,10 +1229,29 @@ var require_package = __commonJS({
|
|
|
158
1229
|
"@opentelemetry/sdk-trace-node": "^2.6.0",
|
|
159
1230
|
"@opentelemetry/semantic-conventions": "^1.40.0",
|
|
160
1231
|
"@types/node": "^20.0.0",
|
|
1232
|
+
openai: "^4.70.0",
|
|
161
1233
|
tsup: "^8.0.0",
|
|
162
1234
|
tsx: "^4.0.0",
|
|
163
1235
|
typescript: "^5.0.0",
|
|
164
|
-
vitest: "^2.
|
|
1236
|
+
vitest: "^3.2.6"
|
|
1237
|
+
},
|
|
1238
|
+
overrides: {
|
|
1239
|
+
protobufjs: "^8.4.1",
|
|
1240
|
+
"vite@6": "^6.4.3",
|
|
1241
|
+
postcss: "^8.5.18",
|
|
1242
|
+
"picomatch@4": "^4.0.4"
|
|
1243
|
+
},
|
|
1244
|
+
pnpm: {
|
|
1245
|
+
overrides: {
|
|
1246
|
+
protobufjs: "^8.4.1",
|
|
1247
|
+
vitest: "^3.2.6",
|
|
1248
|
+
"vite@6": "^6.4.3",
|
|
1249
|
+
postcss: "^8.5.18",
|
|
1250
|
+
"picomatch@4": "^4.0.4"
|
|
1251
|
+
}
|
|
1252
|
+
},
|
|
1253
|
+
bin: {
|
|
1254
|
+
gateway: "./dist/cli.js"
|
|
165
1255
|
}
|
|
166
1256
|
};
|
|
167
1257
|
}
|
|
@@ -171,6 +1261,7 @@ var require_package = __commonJS({
|
|
|
171
1261
|
var src_exports = {};
|
|
172
1262
|
__export(src_exports, {
|
|
173
1263
|
AGENT_ID_KEY: () => AGENT_ID_KEY,
|
|
1264
|
+
DEFAULT_BRANCH: () => DEFAULT_BRANCH,
|
|
174
1265
|
DEFAULT_TIMEOUT_SECONDS: () => DEFAULT_TIMEOUT_SECONDS,
|
|
175
1266
|
DriftError: () => DriftError,
|
|
176
1267
|
ENV_GATEWAY_HOST: () => ENV_GATEWAY_HOST,
|
|
@@ -212,17 +1303,30 @@ __export(src_exports, {
|
|
|
212
1303
|
TraceDiff: () => TraceDiff,
|
|
213
1304
|
UserProfileError: () => UserProfileError,
|
|
214
1305
|
UsersClient: () => UsersClient,
|
|
1306
|
+
WorldSession: () => WorldSession,
|
|
1307
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
1308
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
1309
|
+
WorldSet: () => WorldSet,
|
|
1310
|
+
WorldTask: () => WorldTask,
|
|
215
1311
|
addExperimentAttributes: () => addExperimentAttributes,
|
|
1312
|
+
apiRequest: () => apiRequest,
|
|
216
1313
|
assembleManifest: () => assembleManifest,
|
|
217
1314
|
assignVariant: () => assignVariant,
|
|
1315
|
+
attachSession: () => attachSession,
|
|
218
1316
|
basicAuthHeader: () => basicAuthHeader,
|
|
1317
|
+
buildSourceTarball: () => buildSourceTarball,
|
|
219
1318
|
canonicalJson: () => canonicalJson,
|
|
220
1319
|
clearExperimentContext: () => clearExperimentContext,
|
|
1320
|
+
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
221
1321
|
computeBuildHash: () => computeBuildHash,
|
|
1322
|
+
computeContentHash: () => computeContentHash,
|
|
222
1323
|
computeHash: () => computeHash,
|
|
223
1324
|
context: () => context,
|
|
1325
|
+
createRef: () => createRef,
|
|
224
1326
|
currentTraceId: () => currentTraceId,
|
|
225
1327
|
defineTool: () => defineTool,
|
|
1328
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
1329
|
+
detectProvenance: () => detectProvenance,
|
|
226
1330
|
experimentContextStorage: () => experimentContextStorage,
|
|
227
1331
|
experiments: () => singleton_exports,
|
|
228
1332
|
gateway: () => gateway,
|
|
@@ -231,10 +1335,21 @@ __export(src_exports, {
|
|
|
231
1335
|
getRegisteredPrompts: () => getRegisteredPrompts,
|
|
232
1336
|
getRegisteredSkills: () => getRegisteredSkills,
|
|
233
1337
|
getRegisteredTools: () => getRegisteredTools,
|
|
1338
|
+
getRun: () => getRun,
|
|
234
1339
|
identify: () => identify,
|
|
1340
|
+
isIncludedInContentHash: () => isIncludedInContentHash,
|
|
1341
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
1342
|
+
listRunConfigs: () => listRunConfigs,
|
|
235
1343
|
loadSession: () => loadSession,
|
|
1344
|
+
localTreeState: () => localTreeState,
|
|
236
1345
|
memory: () => memory,
|
|
1346
|
+
openSession: () => openSession,
|
|
1347
|
+
openWorld: () => openWorld,
|
|
237
1348
|
personas: () => personas,
|
|
1349
|
+
propose: () => propose,
|
|
1350
|
+
pull: () => pull,
|
|
1351
|
+
push: () => push,
|
|
1352
|
+
readLocalMeta: () => readLocalMeta,
|
|
238
1353
|
registerArtifact: () => registerArtifact,
|
|
239
1354
|
registerPrompt: () => registerPrompt,
|
|
240
1355
|
registerSkill: () => registerSkill,
|
|
@@ -242,16 +1357,23 @@ __export(src_exports, {
|
|
|
242
1357
|
replay: () => replay,
|
|
243
1358
|
resetBuildRegistry: () => resetBuildRegistry,
|
|
244
1359
|
resolveUserId: () => resolveUserId,
|
|
1360
|
+
runSessions: () => runSessions,
|
|
245
1361
|
runSuite: () => runSuite,
|
|
246
1362
|
runWithExperimentContext: () => runWithExperimentContext,
|
|
1363
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
247
1364
|
scores: () => scores,
|
|
248
1365
|
sessionObservationFromApi: () => sessionObservationFromApi,
|
|
249
1366
|
sessions: () => sessions,
|
|
1367
|
+
setEnvSecret: () => setEnvSecret,
|
|
250
1368
|
setExperimentContext: () => setExperimentContext,
|
|
1369
|
+
snapshotTreeHashes: () => snapshotTreeHashes,
|
|
251
1370
|
success: () => success,
|
|
252
1371
|
suite: () => suite,
|
|
253
1372
|
tracing: () => tracing,
|
|
254
|
-
users: () => users
|
|
1373
|
+
users: () => users,
|
|
1374
|
+
validateWorld: () => validateWorld,
|
|
1375
|
+
waitForRun: () => waitForRun,
|
|
1376
|
+
writeLocalMeta: () => writeLocalMeta
|
|
255
1377
|
});
|
|
256
1378
|
module.exports = __toCommonJS(src_exports);
|
|
257
1379
|
|
|
@@ -825,8 +1947,8 @@ var GatewayContext = class _GatewayContext {
|
|
|
825
1947
|
}
|
|
826
1948
|
return resolved;
|
|
827
1949
|
}
|
|
828
|
-
async requestJson(method,
|
|
829
|
-
const response = await fetch(`${this.host}${
|
|
1950
|
+
async requestJson(method, path2, payload) {
|
|
1951
|
+
const response = await fetch(`${this.host}${path2}`, {
|
|
830
1952
|
method,
|
|
831
1953
|
headers: {
|
|
832
1954
|
Authorization: this.authHeader(),
|
|
@@ -837,7 +1959,7 @@ var GatewayContext = class _GatewayContext {
|
|
|
837
1959
|
const text = await response.text();
|
|
838
1960
|
const body = text ? safeJson(text) : {};
|
|
839
1961
|
if (!response.ok) {
|
|
840
|
-
const message = typeof body.message === "string" ? body.message : `Gateway API ${response.status} on ${
|
|
1962
|
+
const message = typeof body.message === "string" ? body.message : `Gateway API ${response.status} on ${path2}`;
|
|
841
1963
|
if (response.status === 409) {
|
|
842
1964
|
throw new GatewayContextConflictError(
|
|
843
1965
|
message,
|
|
@@ -846,7 +1968,7 @@ var GatewayContext = class _GatewayContext {
|
|
|
846
1968
|
}
|
|
847
1969
|
throw new GatewayContextError(message);
|
|
848
1970
|
}
|
|
849
|
-
this.log("%s %s -> %s", method,
|
|
1971
|
+
this.log("%s %s -> %s", method, path2, response.status);
|
|
850
1972
|
return body;
|
|
851
1973
|
}
|
|
852
1974
|
authHeader() {
|
|
@@ -953,11 +2075,11 @@ function primaryPathForKind(kind, snapshot) {
|
|
|
953
2075
|
const preferred = kind === "agent" ? "AGENTS.md" : kind === "prompt" ? "PROMPT.md" : kind === "skill" ? "SKILL.md" : "MEMORY.md";
|
|
954
2076
|
if (readFile(snapshot, preferred)) return preferred;
|
|
955
2077
|
return Object.entries(snapshot.files).find(
|
|
956
|
-
([
|
|
2078
|
+
([path2, entry]) => entry.type === "file" && path2.endsWith(".md")
|
|
957
2079
|
)?.[0] ?? null;
|
|
958
2080
|
}
|
|
959
|
-
function parseJsonFile(snapshot,
|
|
960
|
-
const content = readFile(snapshot,
|
|
2081
|
+
function parseJsonFile(snapshot, path2) {
|
|
2082
|
+
const content = readFile(snapshot, path2);
|
|
961
2083
|
if (!content) return {};
|
|
962
2084
|
try {
|
|
963
2085
|
const parsed = JSON.parse(content);
|
|
@@ -966,8 +2088,8 @@ function parseJsonFile(snapshot, path) {
|
|
|
966
2088
|
return {};
|
|
967
2089
|
}
|
|
968
2090
|
}
|
|
969
|
-
function readFile(snapshot,
|
|
970
|
-
const entry = snapshot.files[
|
|
2091
|
+
function readFile(snapshot, path2) {
|
|
2092
|
+
const entry = snapshot.files[path2];
|
|
971
2093
|
return entry?.type === "file" ? entry.content : null;
|
|
972
2094
|
}
|
|
973
2095
|
function provenanceFor(snapshot, label) {
|
|
@@ -1069,16 +2191,16 @@ var GatewayMemory = class _GatewayMemory {
|
|
|
1069
2191
|
`/api/public/context/memory/${encodeIdentifier2(identifier)}${suffix}`
|
|
1070
2192
|
);
|
|
1071
2193
|
}
|
|
1072
|
-
async read(target,
|
|
2194
|
+
async read(target, path2, options = {}) {
|
|
1073
2195
|
const snapshot = typeof target === "string" ? await this.pull(target, options) : target;
|
|
1074
|
-
const file = snapshot.files[
|
|
2196
|
+
const file = snapshot.files[path2];
|
|
1075
2197
|
return file?.type === "file" ? file.content : null;
|
|
1076
2198
|
}
|
|
1077
|
-
async write(target,
|
|
2199
|
+
async write(target, path2, content, options = {}) {
|
|
1078
2200
|
const snapshot = typeof target === "string" ? await this.pull(target) : target;
|
|
1079
2201
|
return this.push(snapshot.repo.repoHandle, {
|
|
1080
2202
|
files: {
|
|
1081
|
-
[
|
|
2203
|
+
[path2]: {
|
|
1082
2204
|
content,
|
|
1083
2205
|
...options.mimeType ? { mimeType: options.mimeType } : {}
|
|
1084
2206
|
}
|
|
@@ -1088,10 +2210,10 @@ var GatewayMemory = class _GatewayMemory {
|
|
|
1088
2210
|
...runtimeContextPayload2(options)
|
|
1089
2211
|
});
|
|
1090
2212
|
}
|
|
1091
|
-
async delete(target,
|
|
2213
|
+
async delete(target, path2, options = {}) {
|
|
1092
2214
|
const snapshot = typeof target === "string" ? await this.pull(target) : target;
|
|
1093
2215
|
return this.push(snapshot.repo.repoHandle, {
|
|
1094
|
-
files: { [
|
|
2216
|
+
files: { [path2]: null },
|
|
1095
2217
|
parentCommit: options.parentCommit ?? snapshot.commit.commitHash,
|
|
1096
2218
|
message: options.message,
|
|
1097
2219
|
...runtimeContextPayload2(options)
|
|
@@ -1104,8 +2226,8 @@ var GatewayMemory = class _GatewayMemory {
|
|
|
1104
2226
|
payload
|
|
1105
2227
|
);
|
|
1106
2228
|
}
|
|
1107
|
-
async requestJson(method,
|
|
1108
|
-
const response = await fetch(`${this.host}${
|
|
2229
|
+
async requestJson(method, path2, payload) {
|
|
2230
|
+
const response = await fetch(`${this.host}${path2}`, {
|
|
1109
2231
|
method,
|
|
1110
2232
|
headers: {
|
|
1111
2233
|
Authorization: this.authHeader(),
|
|
@@ -1116,7 +2238,7 @@ var GatewayMemory = class _GatewayMemory {
|
|
|
1116
2238
|
const text = await response.text();
|
|
1117
2239
|
const body = text ? safeJson2(text) : {};
|
|
1118
2240
|
if (!response.ok) {
|
|
1119
|
-
const message = typeof body.message === "string" ? body.message : `Gateway API ${response.status} on ${
|
|
2241
|
+
const message = typeof body.message === "string" ? body.message : `Gateway API ${response.status} on ${path2}`;
|
|
1120
2242
|
if (response.status === 409) {
|
|
1121
2243
|
throw new GatewayMemoryConflictError(
|
|
1122
2244
|
message,
|
|
@@ -1125,7 +2247,7 @@ var GatewayMemory = class _GatewayMemory {
|
|
|
1125
2247
|
}
|
|
1126
2248
|
throw new GatewayMemoryError(message);
|
|
1127
2249
|
}
|
|
1128
|
-
this.log("%s %s -> %s", method,
|
|
2250
|
+
this.log("%s %s -> %s", method, path2, response.status);
|
|
1129
2251
|
return body;
|
|
1130
2252
|
}
|
|
1131
2253
|
authHeader() {
|
|
@@ -1287,10 +2409,10 @@ var PersonasClient = class _PersonasClient {
|
|
|
1287
2409
|
done: Boolean(payload.done)
|
|
1288
2410
|
};
|
|
1289
2411
|
}
|
|
1290
|
-
async requestJson(method,
|
|
2412
|
+
async requestJson(method, path2, payload) {
|
|
1291
2413
|
let response;
|
|
1292
2414
|
try {
|
|
1293
|
-
response = await fetch(`${this.host}${
|
|
2415
|
+
response = await fetch(`${this.host}${path2}`, {
|
|
1294
2416
|
method,
|
|
1295
2417
|
headers: {
|
|
1296
2418
|
Authorization: this.authHeader(),
|
|
@@ -1301,7 +2423,7 @@ var PersonasClient = class _PersonasClient {
|
|
|
1301
2423
|
});
|
|
1302
2424
|
} catch (error) {
|
|
1303
2425
|
const reason = error instanceof Error ? error.message : String(error);
|
|
1304
|
-
throw new PersonaError(`${method} ${
|
|
2426
|
+
throw new PersonaError(`${method} ${path2} failed: ${reason}`, {
|
|
1305
2427
|
cause: error
|
|
1306
2428
|
});
|
|
1307
2429
|
}
|
|
@@ -1309,10 +2431,10 @@ var PersonasClient = class _PersonasClient {
|
|
|
1309
2431
|
if (!response.ok) {
|
|
1310
2432
|
const detail = text.slice(0, 500);
|
|
1311
2433
|
throw new PersonaError(
|
|
1312
|
-
`${method} ${
|
|
2434
|
+
`${method} ${path2} failed (${response.status}): ${detail}`
|
|
1313
2435
|
);
|
|
1314
2436
|
}
|
|
1315
|
-
this.log("%s %s -> %s", method,
|
|
2437
|
+
this.log("%s %s -> %s", method, path2, response.status);
|
|
1316
2438
|
return safeJson3(text);
|
|
1317
2439
|
}
|
|
1318
2440
|
authHeader() {
|
|
@@ -1537,10 +2659,10 @@ var SessionRecording = class _SessionRecording {
|
|
|
1537
2659
|
};
|
|
1538
2660
|
}
|
|
1539
2661
|
/** Dump the full session as a JSON fixture (for regression tests). */
|
|
1540
|
-
toFixture(
|
|
1541
|
-
(0, import_node_fs.mkdirSync)((0, import_node_path.dirname)(
|
|
1542
|
-
(0, import_node_fs.writeFileSync)(
|
|
1543
|
-
return
|
|
2662
|
+
toFixture(path2) {
|
|
2663
|
+
(0, import_node_fs.mkdirSync)((0, import_node_path.dirname)(path2), { recursive: true });
|
|
2664
|
+
(0, import_node_fs.writeFileSync)(path2, JSON.stringify(this.toJSON(), null, 2), "utf-8");
|
|
2665
|
+
return path2;
|
|
1544
2666
|
}
|
|
1545
2667
|
static fromApi(raw) {
|
|
1546
2668
|
return new _SessionRecording({
|
|
@@ -2375,7 +3497,7 @@ function randomHex() {
|
|
|
2375
3497
|
return crypto.randomUUID().replace(/-/g, "");
|
|
2376
3498
|
}
|
|
2377
3499
|
function sleep(ms) {
|
|
2378
|
-
return new Promise((
|
|
3500
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
2379
3501
|
}
|
|
2380
3502
|
var replay = {
|
|
2381
3503
|
suite,
|
|
@@ -2462,7 +3584,7 @@ var GatewaySecurity = class _GatewaySecurity {
|
|
|
2462
3584
|
const descriptors = entries.map(
|
|
2463
3585
|
([key, tool]) => this.describeTool(key, tool, options)
|
|
2464
3586
|
);
|
|
2465
|
-
const
|
|
3587
|
+
const resolve2 = await this.resolveToolset({
|
|
2466
3588
|
agent: this.agent,
|
|
2467
3589
|
toolsetId: options.toolsetId,
|
|
2468
3590
|
toolsetName: options.toolsetName,
|
|
@@ -2472,8 +3594,8 @@ var GatewaySecurity = class _GatewaySecurity {
|
|
|
2472
3594
|
});
|
|
2473
3595
|
const securedEntries = entries.flatMap(([key, tool]) => {
|
|
2474
3596
|
const descriptor = descriptors.find((item) => item.name === key);
|
|
2475
|
-
if (!this.shouldExposeTool(
|
|
2476
|
-
this.log("Hiding tool %s in %s mode", descriptor.id,
|
|
3597
|
+
if (!this.shouldExposeTool(resolve2, descriptor.id)) {
|
|
3598
|
+
this.log("Hiding tool %s in %s mode", descriptor.id, resolve2.mode);
|
|
2477
3599
|
return [];
|
|
2478
3600
|
}
|
|
2479
3601
|
return [
|
|
@@ -2483,7 +3605,7 @@ var GatewaySecurity = class _GatewaySecurity {
|
|
|
2483
3605
|
toolId: descriptor.id,
|
|
2484
3606
|
toolsetId: options.toolsetId,
|
|
2485
3607
|
context: this.normalizeContext(options.context),
|
|
2486
|
-
resolve
|
|
3608
|
+
resolve: resolve2
|
|
2487
3609
|
})
|
|
2488
3610
|
]
|
|
2489
3611
|
];
|
|
@@ -2571,8 +3693,8 @@ var GatewaySecurity = class _GatewaySecurity {
|
|
|
2571
3693
|
);
|
|
2572
3694
|
}
|
|
2573
3695
|
}
|
|
2574
|
-
async postJson(
|
|
2575
|
-
const response = await fetch(`${this.host}${
|
|
3696
|
+
async postJson(path2, payload) {
|
|
3697
|
+
const response = await fetch(`${this.host}${path2}`, {
|
|
2576
3698
|
method: "POST",
|
|
2577
3699
|
headers: {
|
|
2578
3700
|
Authorization: this.authHeader(),
|
|
@@ -2583,7 +3705,7 @@ var GatewaySecurity = class _GatewaySecurity {
|
|
|
2583
3705
|
if (!response.ok) {
|
|
2584
3706
|
const body = await response.text();
|
|
2585
3707
|
throw new GatewaySecurityError(
|
|
2586
|
-
`Gateway API ${response.status} on ${
|
|
3708
|
+
`Gateway API ${response.status} on ${path2}: ${body || response.statusText}`
|
|
2587
3709
|
);
|
|
2588
3710
|
}
|
|
2589
3711
|
return await response.json();
|
|
@@ -2598,9 +3720,9 @@ var GatewaySecurity = class _GatewaySecurity {
|
|
|
2598
3720
|
metadata: options.toolMetadata?.[key] ?? {}
|
|
2599
3721
|
};
|
|
2600
3722
|
}
|
|
2601
|
-
shouldExposeTool(
|
|
2602
|
-
if (
|
|
2603
|
-
if (
|
|
3723
|
+
shouldExposeTool(resolve2, toolId) {
|
|
3724
|
+
if (resolve2.mode === "observe") return true;
|
|
3725
|
+
if (resolve2.blockedToolIds.includes(toolId)) return false;
|
|
2604
3726
|
return true;
|
|
2605
3727
|
}
|
|
2606
3728
|
normalizeContext(context4) {
|
|
@@ -2764,10 +3886,10 @@ var UsersClient = class _UsersClient {
|
|
|
2764
3886
|
totalItems: Number(meta.totalItems ?? rows.length)
|
|
2765
3887
|
};
|
|
2766
3888
|
}
|
|
2767
|
-
async requestJson(method,
|
|
3889
|
+
async requestJson(method, path2, payload) {
|
|
2768
3890
|
let response;
|
|
2769
3891
|
try {
|
|
2770
|
-
response = await fetch(`${this.host}${
|
|
3892
|
+
response = await fetch(`${this.host}${path2}`, {
|
|
2771
3893
|
method,
|
|
2772
3894
|
headers: {
|
|
2773
3895
|
Authorization: this.authHeader(),
|
|
@@ -2778,17 +3900,17 @@ var UsersClient = class _UsersClient {
|
|
|
2778
3900
|
});
|
|
2779
3901
|
} catch (error) {
|
|
2780
3902
|
throw new UserProfileError(
|
|
2781
|
-
`${method} ${
|
|
3903
|
+
`${method} ${path2} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2782
3904
|
{ cause: error }
|
|
2783
3905
|
);
|
|
2784
3906
|
}
|
|
2785
3907
|
const text = await response.text();
|
|
2786
3908
|
if (!response.ok) {
|
|
2787
3909
|
throw new UserProfileError(
|
|
2788
|
-
`${method} ${
|
|
3910
|
+
`${method} ${path2} failed (${response.status}): ${text.slice(0, 500)}`
|
|
2789
3911
|
);
|
|
2790
3912
|
}
|
|
2791
|
-
this.log("%s %s -> %s", method,
|
|
3913
|
+
this.log("%s %s -> %s", method, path2, response.status);
|
|
2792
3914
|
return safeJson4(text);
|
|
2793
3915
|
}
|
|
2794
3916
|
authHeader() {
|
|
@@ -2992,10 +4114,10 @@ var ScoresClient = class _ScoresClient {
|
|
|
2992
4114
|
metadata: signalMetadata
|
|
2993
4115
|
});
|
|
2994
4116
|
}
|
|
2995
|
-
async request(method,
|
|
4117
|
+
async request(method, path2, payload) {
|
|
2996
4118
|
let response;
|
|
2997
4119
|
try {
|
|
2998
|
-
response = await fetch(`${this.host}${
|
|
4120
|
+
response = await fetch(`${this.host}${path2}`, {
|
|
2999
4121
|
method,
|
|
3000
4122
|
headers: {
|
|
3001
4123
|
Authorization: this.authHeader(),
|
|
@@ -3006,17 +4128,17 @@ var ScoresClient = class _ScoresClient {
|
|
|
3006
4128
|
});
|
|
3007
4129
|
} catch (error) {
|
|
3008
4130
|
throw new ScoreError(
|
|
3009
|
-
`${method} ${
|
|
4131
|
+
`${method} ${path2} failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
3010
4132
|
{ cause: error }
|
|
3011
4133
|
);
|
|
3012
4134
|
}
|
|
3013
4135
|
const text = await response.text();
|
|
3014
4136
|
if (!response.ok) {
|
|
3015
4137
|
throw new ScoreError(
|
|
3016
|
-
`${method} ${
|
|
4138
|
+
`${method} ${path2} failed (${response.status}): ${text.slice(0, 500)}`
|
|
3017
4139
|
);
|
|
3018
4140
|
}
|
|
3019
|
-
this.log("%s %s -> %s", method,
|
|
4141
|
+
this.log("%s %s -> %s", method, path2, response.status);
|
|
3020
4142
|
}
|
|
3021
4143
|
authHeader() {
|
|
3022
4144
|
return `Basic ${Buffer.from(`${this.publicKey}:${this.secretKey}`).toString(
|
|
@@ -3053,8 +4175,12 @@ var scores = {
|
|
|
3053
4175
|
success
|
|
3054
4176
|
};
|
|
3055
4177
|
|
|
4178
|
+
// src/index.ts
|
|
4179
|
+
init_worlds();
|
|
4180
|
+
init_session();
|
|
4181
|
+
|
|
3056
4182
|
// src/build/index.ts
|
|
3057
|
-
var
|
|
4183
|
+
var import_node_crypto2 = require("crypto");
|
|
3058
4184
|
function canonicalJson(value) {
|
|
3059
4185
|
if (value === null || typeof value !== "object") {
|
|
3060
4186
|
return JSON.stringify(value);
|
|
@@ -3066,7 +4192,7 @@ function canonicalJson(value) {
|
|
|
3066
4192
|
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`;
|
|
3067
4193
|
}
|
|
3068
4194
|
function sha256(text) {
|
|
3069
|
-
return `sha256:${(0,
|
|
4195
|
+
return `sha256:${(0, import_node_crypto2.createHash)("sha256").update(text).digest("hex")}`;
|
|
3070
4196
|
}
|
|
3071
4197
|
var registeredTools = /* @__PURE__ */ new Map();
|
|
3072
4198
|
var registeredPrompts = /* @__PURE__ */ new Map();
|
|
@@ -3935,6 +5061,7 @@ var tracing = {
|
|
|
3935
5061
|
// Annotate the CommonJS export names for ESM import in node:
|
|
3936
5062
|
0 && (module.exports = {
|
|
3937
5063
|
AGENT_ID_KEY,
|
|
5064
|
+
DEFAULT_BRANCH,
|
|
3938
5065
|
DEFAULT_TIMEOUT_SECONDS,
|
|
3939
5066
|
DriftError,
|
|
3940
5067
|
ENV_GATEWAY_HOST,
|
|
@@ -3976,17 +5103,30 @@ var tracing = {
|
|
|
3976
5103
|
TraceDiff,
|
|
3977
5104
|
UserProfileError,
|
|
3978
5105
|
UsersClient,
|
|
5106
|
+
WorldSession,
|
|
5107
|
+
WorldSessionCallError,
|
|
5108
|
+
WorldSessionTimeout,
|
|
5109
|
+
WorldSet,
|
|
5110
|
+
WorldTask,
|
|
3979
5111
|
addExperimentAttributes,
|
|
5112
|
+
apiRequest,
|
|
3980
5113
|
assembleManifest,
|
|
3981
5114
|
assignVariant,
|
|
5115
|
+
attachSession,
|
|
3982
5116
|
basicAuthHeader,
|
|
5117
|
+
buildSourceTarball,
|
|
3983
5118
|
canonicalJson,
|
|
3984
5119
|
clearExperimentContext,
|
|
5120
|
+
collectArchiveRelPaths,
|
|
3985
5121
|
computeBuildHash,
|
|
5122
|
+
computeContentHash,
|
|
3986
5123
|
computeHash,
|
|
3987
5124
|
context,
|
|
5125
|
+
createRef,
|
|
3988
5126
|
currentTraceId,
|
|
3989
5127
|
defineTool,
|
|
5128
|
+
deleteEnvSecret,
|
|
5129
|
+
detectProvenance,
|
|
3990
5130
|
experimentContextStorage,
|
|
3991
5131
|
experiments,
|
|
3992
5132
|
gateway,
|
|
@@ -3995,10 +5135,21 @@ var tracing = {
|
|
|
3995
5135
|
getRegisteredPrompts,
|
|
3996
5136
|
getRegisteredSkills,
|
|
3997
5137
|
getRegisteredTools,
|
|
5138
|
+
getRun,
|
|
3998
5139
|
identify,
|
|
5140
|
+
isIncludedInContentHash,
|
|
5141
|
+
listEnvSecretKeys,
|
|
5142
|
+
listRunConfigs,
|
|
3999
5143
|
loadSession,
|
|
5144
|
+
localTreeState,
|
|
4000
5145
|
memory,
|
|
5146
|
+
openSession,
|
|
5147
|
+
openWorld,
|
|
4001
5148
|
personas,
|
|
5149
|
+
propose,
|
|
5150
|
+
pull,
|
|
5151
|
+
push,
|
|
5152
|
+
readLocalMeta,
|
|
4002
5153
|
registerArtifact,
|
|
4003
5154
|
registerPrompt,
|
|
4004
5155
|
registerSkill,
|
|
@@ -4006,15 +5157,22 @@ var tracing = {
|
|
|
4006
5157
|
replay,
|
|
4007
5158
|
resetBuildRegistry,
|
|
4008
5159
|
resolveUserId,
|
|
5160
|
+
runSessions,
|
|
4009
5161
|
runSuite,
|
|
4010
5162
|
runWithExperimentContext,
|
|
5163
|
+
schemaFromSignature,
|
|
4011
5164
|
scores,
|
|
4012
5165
|
sessionObservationFromApi,
|
|
4013
5166
|
sessions,
|
|
5167
|
+
setEnvSecret,
|
|
4014
5168
|
setExperimentContext,
|
|
5169
|
+
snapshotTreeHashes,
|
|
4015
5170
|
success,
|
|
4016
5171
|
suite,
|
|
4017
5172
|
tracing,
|
|
4018
|
-
users
|
|
5173
|
+
users,
|
|
5174
|
+
validateWorld,
|
|
5175
|
+
waitForRun,
|
|
5176
|
+
writeLocalMeta
|
|
4019
5177
|
});
|
|
4020
5178
|
//# sourceMappingURL=index.js.map
|