@withgateway/sdk 0.1.13 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs ADDED
@@ -0,0 +1,1476 @@
1
+ #!/usr/bin/env node
2
+ import { createRequire as __gatewayCreateRequire } from "module";
3
+ const require = __gatewayCreateRequire(import.meta.url);
4
+ var __defProp = Object.defineProperty;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
7
+ get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
8
+ }) : x)(function(x) {
9
+ if (typeof require !== "undefined") return require.apply(this, arguments);
10
+ throw Error('Dynamic require of "' + x + '" is not supported');
11
+ });
12
+ var __esm = (fn, res) => function __init() {
13
+ return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
14
+ };
15
+ var __export = (target, all) => {
16
+ for (var name in all)
17
+ __defProp(target, name, { get: all[name], enumerable: true });
18
+ };
19
+
20
+ // src/worlds/session.ts
21
+ var session_exports = {};
22
+ __export(session_exports, {
23
+ WorldSession: () => WorldSession,
24
+ WorldSessionCallError: () => WorldSessionCallError,
25
+ WorldSessionTimeout: () => WorldSessionTimeout,
26
+ attachSession: () => attachSession,
27
+ openSession: () => openSession,
28
+ runSessions: () => runSessions,
29
+ schemaFromSignature: () => schemaFromSignature
30
+ });
31
+ function schemaFromSignature(spec) {
32
+ const inside = /\(([\s\S]*)\)\s*$/.exec(spec.signature.trim())?.[1] ?? "";
33
+ const properties = {};
34
+ const required = [];
35
+ const params = [];
36
+ let depth = 0;
37
+ let current = "";
38
+ for (const ch of inside) {
39
+ if ("([{".includes(ch)) depth++;
40
+ if (")]}".includes(ch)) depth--;
41
+ if (ch === "," && depth === 0) {
42
+ params.push(current);
43
+ current = "";
44
+ } else current += ch;
45
+ }
46
+ if (current.trim()) params.push(current);
47
+ for (const raw of params) {
48
+ const param = raw.trim();
49
+ if (!param || param.startsWith("*") || param === "self") continue;
50
+ const eq = param.indexOf("=");
51
+ const name = (eq === -1 ? param : param.slice(0, eq)).split(":")[0].trim();
52
+ if (!name) continue;
53
+ if (eq === -1) {
54
+ properties[name] = { type: "string" };
55
+ required.push(name);
56
+ continue;
57
+ }
58
+ const value = param.slice(eq + 1).trim();
59
+ if (/^-?\d+(\.\d+)?$/.test(value)) properties[name] = { type: "number" };
60
+ else if (value === "True" || value === "False")
61
+ properties[name] = { type: "boolean" };
62
+ else properties[name] = { type: "string" };
63
+ }
64
+ return {
65
+ type: "function",
66
+ function: {
67
+ name: spec.name,
68
+ description: spec.doc ?? spec.name,
69
+ parameters: { type: "object", properties, required }
70
+ }
71
+ };
72
+ }
73
+ async function attachSession(sessionId, options = {}) {
74
+ const state = await apiRequest(
75
+ options,
76
+ "GET",
77
+ `/api/public/world-sessions/${encodeURIComponent(sessionId)}`
78
+ );
79
+ return new WorldSession(sessionId, state, options);
80
+ }
81
+ async function openSession(slugOrRef, task, opts = {}) {
82
+ const options = opts.options ?? {};
83
+ const at = slugOrRef.indexOf("@");
84
+ const slug = at === -1 ? slugOrRef : slugOrRef.slice(0, at);
85
+ const state = await apiRequest(
86
+ options,
87
+ "POST",
88
+ "/api/public/world-sessions",
89
+ {
90
+ ...opts.versionId ? { versionId: opts.versionId } : { slug },
91
+ task,
92
+ ...opts.reuseWarm === void 0 ? {} : { reuseWarm: opts.reuseWarm }
93
+ }
94
+ );
95
+ return new WorldSession(state.sessionId, state, options);
96
+ }
97
+ async function runSessions(opts) {
98
+ const options = opts.options ?? {};
99
+ const concurrency = Math.max(1, opts.concurrency ?? 3);
100
+ let tasks = opts.tasks;
101
+ let versionId = opts.versionId;
102
+ if (!tasks || !versionId) {
103
+ const { openWorld: openWorld2 } = await Promise.resolve().then(() => (init_worlds(), worlds_exports));
104
+ const world = await openWorld2(opts.world, options);
105
+ tasks = tasks ?? world.tasks.map((t) => t.name);
106
+ versionId = versionId ?? world.version.versionId;
107
+ }
108
+ const results = [];
109
+ let next = 0;
110
+ await Promise.all(
111
+ Array.from({ length: Math.min(concurrency, tasks.length) }, async () => {
112
+ while (next < tasks.length) {
113
+ const task = tasks[next++];
114
+ let session = null;
115
+ try {
116
+ session = await openSession(opts.world, task, {
117
+ versionId,
118
+ options
119
+ });
120
+ await session.ready();
121
+ const toolkit = await session.toolkit();
122
+ await opts.agent({
123
+ session,
124
+ toolkit,
125
+ instruction: session.instruction ?? task,
126
+ task
127
+ });
128
+ const graded = await session.grade();
129
+ const result = {
130
+ task,
131
+ sessionId: session.sessionId,
132
+ reward: graded.reward,
133
+ rewards: graded.rewards
134
+ };
135
+ results.push(result);
136
+ opts.onResult?.(result);
137
+ } catch (error) {
138
+ const result = {
139
+ task,
140
+ sessionId: session?.sessionId ?? "",
141
+ reward: 0,
142
+ rewards: {},
143
+ // An Error with an empty message would read as "no error" to any
144
+ // caller checking truthiness; name the type instead.
145
+ error: (error instanceof Error ? error.message : String(error)) || (error instanceof Error ? error.name : "Unknown error")
146
+ };
147
+ results.push(result);
148
+ opts.onResult?.(result);
149
+ } finally {
150
+ await session?.close({ keepWarm: opts.keepWarm ?? true }).catch(() => void 0);
151
+ }
152
+ }
153
+ })
154
+ );
155
+ const mean = results.reduce((sum, r) => sum + (r.reward ?? 0), 0) / (results.length || 1);
156
+ return {
157
+ results: results.sort((a, b) => a.task.localeCompare(b.task)),
158
+ mean,
159
+ passed: opts.failUnder === void 0 ? true : mean >= opts.failUnder
160
+ };
161
+ }
162
+ var READY_TIMEOUT_MS, READY_POLL_MS, CALL_TIMEOUT_MS, CALL_POLL_MS, sleep, WorldSessionTimeout, WorldSessionCallError, WorldSession;
163
+ var init_session = __esm({
164
+ "src/worlds/session.ts"() {
165
+ "use strict";
166
+ init_worlds();
167
+ READY_TIMEOUT_MS = 10 * 6e4;
168
+ READY_POLL_MS = 2e3;
169
+ CALL_TIMEOUT_MS = 3 * 6e4;
170
+ CALL_POLL_MS = 400;
171
+ sleep = (ms) => new Promise((r) => setTimeout(r, ms));
172
+ WorldSessionTimeout = class extends Error {
173
+ };
174
+ WorldSessionCallError = class extends Error {
175
+ };
176
+ WorldSession = class {
177
+ constructor(sessionId, state, options) {
178
+ this.sessionId = sessionId;
179
+ this.state = state;
180
+ this.options = options;
181
+ }
182
+ get task() {
183
+ return this.state.taskName;
184
+ }
185
+ /** What this task asks for — the bundle's own instruction.md. */
186
+ get instruction() {
187
+ return this.state.instruction ?? null;
188
+ }
189
+ get versionId() {
190
+ return this.state.versionId;
191
+ }
192
+ /** The session's latest platform state (one request, no waiting). */
193
+ async status() {
194
+ const fresh = await apiRequest(
195
+ this.options,
196
+ "GET",
197
+ `/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`
198
+ );
199
+ this.state = {
200
+ instruction: this.state.instruction,
201
+ tools: this.state.tools,
202
+ ...fresh
203
+ };
204
+ return this.state;
205
+ }
206
+ /**
207
+ * Wait until the container is warm — meaning the process inside it has
208
+ * asked the platform for work, not merely that a job was accepted.
209
+ */
210
+ async ready(opts = {}) {
211
+ const deadline = Date.now() + (opts.timeoutMs ?? READY_TIMEOUT_MS);
212
+ for (; ; ) {
213
+ const state = await this.status();
214
+ if (state.ready) return state;
215
+ if (state.state === "FAILED" || state.state === "STOPPED")
216
+ throw new WorldSessionCallError(
217
+ `World session ${state.state.toLowerCase()} before it was ready.`
218
+ );
219
+ if (Date.now() > deadline)
220
+ throw new WorldSessionTimeout(
221
+ `World session still ${state.state} after ${Math.round((opts.timeoutMs ?? READY_TIMEOUT_MS) / 1e3)}s.`
222
+ );
223
+ await sleep(READY_POLL_MS);
224
+ }
225
+ }
226
+ /** Enqueue one interaction and wait for the container's answer. */
227
+ async exchange(body, timeoutMs = CALL_TIMEOUT_MS) {
228
+ const { callId } = await apiRequest(
229
+ this.options,
230
+ "POST",
231
+ `/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls`,
232
+ body
233
+ );
234
+ const deadline = Date.now() + timeoutMs;
235
+ for (; ; ) {
236
+ const call = await apiRequest(
237
+ this.options,
238
+ "GET",
239
+ `/api/public/world-sessions/${encodeURIComponent(this.sessionId)}/calls/${encodeURIComponent(callId)}`
240
+ );
241
+ if (call.done) {
242
+ if (call.error) throw new WorldSessionCallError(call.error);
243
+ return call.result;
244
+ }
245
+ if (Date.now() > deadline)
246
+ throw new WorldSessionTimeout(
247
+ `Call ${callId} still ${call.state} after ${Math.round(timeoutMs / 1e3)}s \u2014 the container may have died.`
248
+ );
249
+ await sleep(CALL_POLL_MS);
250
+ }
251
+ }
252
+ /**
253
+ * Run one of the world's tools INSIDE the container. The arguments and the
254
+ * result are the world's own contract — this layer neither validates nor
255
+ * reshapes them, so a tool error reads exactly as it would in a hosted run.
256
+ */
257
+ async call(tool, args = {}, opts = {}) {
258
+ return this.exchange({ kind: "call", tool, args }, opts.timeoutMs);
259
+ }
260
+ /**
261
+ * THE WORLD'S TOOLS, READY TO HAND TO A MODEL. Schemas come from the
262
+ * world's own signatures and implementations call into this session, so a
263
+ * caller never writes — and never has to re-sync — a tool map of their own:
264
+ *
265
+ * const { schemas, impls } = await session.toolkit();
266
+ * await runMyAgent(impls, schemas, session.instruction);
267
+ *
268
+ * A tool added to the world shows up here on the next run; a tool renamed
269
+ * in the world renames here too, instead of failing at call time against a
270
+ * hand-written map that still remembers the old name.
271
+ */
272
+ async toolkit() {
273
+ let specs = this.state.tools;
274
+ if (!specs) specs = (await this.status()).tools ?? [];
275
+ const impls = {};
276
+ for (const spec of specs) {
277
+ impls[spec.name] = (args = {}) => this.call(spec.name, args);
278
+ }
279
+ return { schemas: specs.map(schemaFromSignature), impls, specs };
280
+ }
281
+ /** Run the task's grader against the container's END STATE. */
282
+ async grade(opts = {}) {
283
+ const raw = await this.exchange(
284
+ { kind: "grade" },
285
+ opts.timeoutMs
286
+ );
287
+ const record = raw && typeof raw === "object" ? raw : {};
288
+ const rewards = {};
289
+ for (const [key, value] of Object.entries(record)) {
290
+ if (key !== "reward" && typeof value === "number") rewards[key] = value;
291
+ }
292
+ return {
293
+ reward: typeof record.reward === "number" ? record.reward : null,
294
+ rewards,
295
+ raw
296
+ };
297
+ }
298
+ /**
299
+ * End the session. `keepWarm` hands the container back to the pool instead
300
+ * of tearing it down, so the next session for the same task starts warm —
301
+ * the pooled instance still obeys the fleet's idle TTL and pinning.
302
+ */
303
+ async close(opts = {}) {
304
+ return apiRequest(
305
+ this.options,
306
+ "DELETE",
307
+ `/api/public/world-sessions/${encodeURIComponent(this.sessionId)}`,
308
+ { keepWarm: opts.keepWarm ?? false }
309
+ );
310
+ }
311
+ };
312
+ }
313
+ });
314
+
315
+ // src/worlds/hub.ts
316
+ import { createHash } from "crypto";
317
+ import { gzipSync, gunzipSync } from "zlib";
318
+ import * as fs from "fs";
319
+ import * as path from "path";
320
+ function isExcludedDirComponent(component) {
321
+ return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
322
+ }
323
+ function isIncludedInContentHash(relPath, includeAllRootFiles) {
324
+ const parts = relPath.split("/");
325
+ const name = parts[parts.length - 1];
326
+ const dirParts = parts.slice(0, -1);
327
+ if (name.startsWith(".")) return false;
328
+ if (parts.includes("__pycache__")) return false;
329
+ if (dirParts.length === 0) {
330
+ return includeAllRootFiles || name === "README.md" || name === "pyproject.toml" || name === "gateway-env.toml" || name.endsWith(".py");
331
+ }
332
+ return !dirParts.some(isExcludedDirComponent);
333
+ }
334
+ function collectArchiveRelPaths(dir) {
335
+ if (fs.existsSync(path.join(dir, ".gitignore"))) {
336
+ throw new Error(
337
+ "This bundle has a root .gitignore, which the Python CLI honors via gitignore_parser. Reimplementing that imperfectly here would silently produce a different content hash, so the TS CLI refuses. Remove the .gitignore (or bake its effect into the tree) and retry."
338
+ );
339
+ }
340
+ const out = [];
341
+ const includeAllRootFiles = fs.existsSync(path.join(dir, "gateway-env.toml"));
342
+ const walk = (rel) => {
343
+ const abs = rel ? path.join(dir, rel) : dir;
344
+ for (const entry of fs.readdirSync(abs, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
345
+ const childRel = rel ? `${rel}/${entry.name}` : entry.name;
346
+ if (entry.isDirectory()) {
347
+ if (!isExcludedDirComponent(entry.name)) walk(childRel);
348
+ } else if (entry.isFile()) {
349
+ if (isIncludedInContentHash(childRel, includeAllRootFiles)) {
350
+ out.push(childRel);
351
+ }
352
+ }
353
+ }
354
+ };
355
+ walk("");
356
+ return out.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
357
+ }
358
+ function computeContentHash(dir) {
359
+ const hasher = createHash("sha256");
360
+ for (const relPath of collectArchiveRelPaths(dir)) {
361
+ hasher.update(Buffer.from(`file:${relPath}`, "utf-8"));
362
+ hasher.update(fs.readFileSync(path.join(dir, relPath)));
363
+ }
364
+ return hasher.digest("hex");
365
+ }
366
+ function octal(value, length) {
367
+ return Buffer.from(value.toString(8).padStart(length - 1, "0") + "\0", "ascii");
368
+ }
369
+ function tarHeader(name, size) {
370
+ const header = Buffer.alloc(512);
371
+ header.write(name, 0, 100, "utf-8");
372
+ octal(420, 8).copy(header, 100);
373
+ octal(0, 8).copy(header, 108);
374
+ octal(0, 8).copy(header, 116);
375
+ octal(size, 12).copy(header, 124);
376
+ octal(0, 12).copy(header, 136);
377
+ header.write(" ", 148, "ascii");
378
+ header.write("0", 156, "ascii");
379
+ header.write("ustar", 257, "ascii");
380
+ header.write("00", 263, "ascii");
381
+ let sum = 0;
382
+ for (const byte of header) sum += byte;
383
+ header.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
384
+ return header;
385
+ }
386
+ function tarMember(name, payload) {
387
+ const parts = [];
388
+ const nameBytes = Buffer.byteLength(name, "utf-8");
389
+ if (nameBytes > 100) {
390
+ const body = Buffer.from(name + "\0", "utf-8");
391
+ const lHeader = tarHeader("././@LongLink", body.length);
392
+ lHeader.write("L", 156, "ascii");
393
+ let sum = 0;
394
+ lHeader.write(" ", 148, "ascii");
395
+ for (const byte of lHeader) sum += byte;
396
+ lHeader.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
397
+ parts.push(lHeader, body);
398
+ const pad2 = body.length % 512 === 0 ? 0 : 512 - body.length % 512;
399
+ if (pad2) parts.push(Buffer.alloc(pad2));
400
+ parts.push(tarHeader(name.slice(0, 100), payload.length));
401
+ } else {
402
+ parts.push(tarHeader(name, payload.length));
403
+ }
404
+ parts.push(payload);
405
+ const pad = payload.length % 512 === 0 ? 0 : 512 - payload.length % 512;
406
+ if (pad) parts.push(Buffer.alloc(pad));
407
+ return parts;
408
+ }
409
+ function buildSourceTarball(dir) {
410
+ const parts = [];
411
+ for (const relPath of collectArchiveRelPaths(dir)) {
412
+ parts.push(...tarMember(relPath, fs.readFileSync(path.join(dir, relPath))));
413
+ }
414
+ parts.push(Buffer.alloc(1024));
415
+ return gzipSync(Buffer.concat(parts), { level: 9 });
416
+ }
417
+ function readLocalMeta(dir) {
418
+ try {
419
+ const raw = fs.readFileSync(path.join(dir, META_RELPATH), "utf-8");
420
+ const parsed = JSON.parse(raw);
421
+ return parsed && typeof parsed === "object" ? parsed : null;
422
+ } catch {
423
+ return null;
424
+ }
425
+ }
426
+ function writeLocalMeta(dir, meta) {
427
+ fs.mkdirSync(path.join(dir, ".gateway"), { recursive: true });
428
+ fs.writeFileSync(path.join(dir, META_RELPATH), JSON.stringify(meta, null, 2));
429
+ }
430
+ function snapshotTreeHashes(dir) {
431
+ const out = {};
432
+ for (const relPath of collectArchiveRelPaths(dir)) {
433
+ out[relPath] = createHash("sha256").update(fs.readFileSync(path.join(dir, relPath))).digest("hex");
434
+ }
435
+ return out;
436
+ }
437
+ function localTreeState(dir) {
438
+ const meta = readLocalMeta(dir);
439
+ const base = meta?.baseFiles;
440
+ if (!base || typeof base !== "object") {
441
+ return { meta, dirty: [], clean: false };
442
+ }
443
+ const now = snapshotTreeHashes(dir);
444
+ const dirty = /* @__PURE__ */ new Set();
445
+ for (const [p, h] of Object.entries(now)) if (base[p] !== h) dirty.add(p);
446
+ for (const p of Object.keys(base)) if (!(p in now)) dirty.add(p);
447
+ return { meta, dirty: [...dirty].sort(), clean: dirty.size === 0 };
448
+ }
449
+ function readPyproject(dir) {
450
+ try {
451
+ const text = fs.readFileSync(path.join(dir, "pyproject.toml"), "utf-8");
452
+ const section = /\[project\]([\s\S]*?)(\n\[|$)/.exec(text)?.[1] ?? "";
453
+ const name = /\bname\s*=\s*"([^"]+)"/.exec(section)?.[1];
454
+ const version = /\bversion\s*=\s*"([^"]+)"/.exec(section)?.[1];
455
+ return { name, version };
456
+ } catch {
457
+ return {};
458
+ }
459
+ }
460
+ function buildFileManifest(dir) {
461
+ const entries = [];
462
+ const walk = (rel) => {
463
+ const abs = rel ? path.join(dir, rel) : dir;
464
+ for (const entry of fs.readdirSync(abs, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
465
+ const childRel = rel ? `${rel}/${entry.name}` : entry.name;
466
+ if (entry.isDirectory()) {
467
+ if (!SKIP_DIRS.has(entry.name) && !entry.name.endsWith(".egg-info")) {
468
+ walk(childRel);
469
+ }
470
+ } else if (entry.isFile()) {
471
+ const payload = fs.readFileSync(path.join(dir, childRel));
472
+ entries.push({
473
+ path: childRel,
474
+ isDirectory: false,
475
+ size: payload.length,
476
+ sha256: createHash("sha256").update(payload).digest("hex")
477
+ });
478
+ }
479
+ }
480
+ };
481
+ walk("");
482
+ return entries;
483
+ }
484
+ async function putObject(url, data, shaB64) {
485
+ const resp = await fetch(url, {
486
+ method: "PUT",
487
+ headers: {
488
+ "Content-Type": "application/gzip",
489
+ "x-amz-checksum-sha256": shaB64
490
+ },
491
+ body: new Uint8Array(data)
492
+ });
493
+ if (!resp.ok) {
494
+ throw new Error(`object PUT -> ${resp.status}: ${(await resp.text()).slice(0, 300)}`);
495
+ }
496
+ }
497
+ async function push(dir, opts = {}) {
498
+ const options = opts.options ?? {};
499
+ if (!fs.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
500
+ const localMeta = readLocalMeta(dir) ?? {};
501
+ const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
502
+ const baseVersionId = localMeta.versionId ?? null;
503
+ const contentHash = computeContentHash(dir);
504
+ const pyproject = readPyproject(dir);
505
+ const slug = opts.slug ?? pyproject.name ?? path.basename(path.resolve(dir));
506
+ const resolve2 = await apiRequest(
507
+ options,
508
+ "POST",
509
+ "/api/public/benchmark-containers/resolve",
510
+ { slug, name: pyproject.name ?? slug }
511
+ );
512
+ const tarball = buildSourceTarball(dir);
513
+ const bundleSha256 = sha256b64(tarball);
514
+ const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/versions", {
515
+ containerId: resolve2.containerId,
516
+ contentHash,
517
+ bundleSha256,
518
+ bundleSizeBytes: tarball.length,
519
+ ...pyproject.version ? { semanticVersion: pyproject.version } : {},
520
+ pyprojectMeta: pyproject,
521
+ fileManifest: buildFileManifest(dir),
522
+ branch,
523
+ baseVersionId,
524
+ ...opts.force ? { force: true } : {},
525
+ ...opts.message ? { changeReason: opts.message } : {},
526
+ ...opts.author ? { author: opts.author } : {}
527
+ });
528
+ if (version.uploadUrl) await putObject(version.uploadUrl, tarball, bundleSha256);
529
+ await apiRequest(
530
+ options,
531
+ "POST",
532
+ `/api/public/benchmark-containers/versions/${version.versionId}/finalize`,
533
+ { branch, baseVersionId, ...opts.force ? { force: true } : {} }
534
+ );
535
+ writeLocalMeta(dir, {
536
+ ...localMeta,
537
+ slug,
538
+ ref: branch,
539
+ branch,
540
+ versionId: version.versionId,
541
+ contentHash,
542
+ semanticVersion: pyproject.version,
543
+ forkedFromVersionId: baseVersionId,
544
+ baseFiles: snapshotTreeHashes(dir)
545
+ });
546
+ return { slug, versionId: version.versionId, contentHash, branch };
547
+ }
548
+ function extractBundle(bundle, dir) {
549
+ const tar = gunzipSync(bundle);
550
+ let offset = 0;
551
+ let longName = null;
552
+ let written = 0;
553
+ while (offset + 512 <= tar.length) {
554
+ const header = tar.subarray(offset, offset + 512);
555
+ offset += 512;
556
+ if (header.every((b) => b === 0)) break;
557
+ const rawName = header.subarray(0, 100).toString("utf-8").replace(/\0.*$/, "");
558
+ const size = parseInt(header.subarray(124, 136).toString("ascii").replace(/\0.*$/, "").trim(), 8) || 0;
559
+ const typeflag = header.subarray(156, 157).toString("ascii");
560
+ const body = tar.subarray(offset, offset + size);
561
+ offset += size + (size % 512 === 0 ? 0 : 512 - size % 512);
562
+ if (typeflag === "L") {
563
+ longName = body.toString("utf-8").replace(/\0.*$/, "");
564
+ continue;
565
+ }
566
+ const name = (longName ?? rawName).replace(/^\.\//, "");
567
+ longName = null;
568
+ if (typeflag !== "0" && typeflag !== "") continue;
569
+ if (name.startsWith("/") || name.split("/").includes("..")) continue;
570
+ const target = path.join(dir, name);
571
+ fs.mkdirSync(path.dirname(target), { recursive: true });
572
+ fs.writeFileSync(target, body);
573
+ written++;
574
+ }
575
+ return written;
576
+ }
577
+ async function pull(slugRef, target, options = {}) {
578
+ const at = slugRef.indexOf("@");
579
+ const slug = at === -1 ? slugRef : slugRef.slice(0, at);
580
+ const ref = at === -1 ? "latest" : slugRef.slice(at + 1);
581
+ const details = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
582
+ slug,
583
+ ref
584
+ });
585
+ if (!details.downloadUrl) {
586
+ throw new Error(`${slug}@${ref} resolved but has no uploaded bundle`);
587
+ }
588
+ let dir = target ?? path.join(process.cwd(), slug);
589
+ if (!target) {
590
+ let i = 1;
591
+ const base = dir;
592
+ while (fs.existsSync(dir)) dir = `${base}-${i++}`;
593
+ }
594
+ fs.mkdirSync(dir, { recursive: true });
595
+ const resp = await fetch(details.downloadUrl);
596
+ if (!resp.ok) throw new Error(`bundle download -> ${resp.status}`);
597
+ const bundle = Buffer.from(await resp.arrayBuffer());
598
+ if (details.bundleSha256 && sha256b64(bundle) !== details.bundleSha256) {
599
+ throw new Error(
600
+ `bundle integrity check failed for ${slug}@${ref}: downloaded sha256 != recorded bundleSha256`
601
+ );
602
+ }
603
+ const files = extractBundle(bundle, dir);
604
+ const branch = ref === "latest" || /^\d|^[0-9a-f]{8,}$/.test(ref) ? DEFAULT_BRANCH : ref;
605
+ writeLocalMeta(dir, {
606
+ slug,
607
+ ref,
608
+ branch,
609
+ versionId: details.versionId,
610
+ contentHash: details.contentHash,
611
+ semanticVersion: details.semanticVersion ?? void 0,
612
+ baseFiles: snapshotTreeHashes(dir)
613
+ });
614
+ return { slug, dir, versionId: details.versionId, files };
615
+ }
616
+ async function propose(dir, opts) {
617
+ const meta = readLocalMeta(dir);
618
+ if (!meta?.slug || !meta.baseFiles) {
619
+ throw new Error("This dir has no pull/push base \u2014 run pull first, then edit, then propose.");
620
+ }
621
+ const state = localTreeState(dir);
622
+ if (state.clean) throw new Error("Nothing to propose \u2014 the tree matches its base.");
623
+ const changes = state.dirty.map((relPath) => {
624
+ const abs = path.join(dir, relPath);
625
+ if (!fs.existsSync(abs)) return { path: relPath, action: "delete" };
626
+ return {
627
+ path: relPath,
628
+ action: "write",
629
+ newContent: fs.readFileSync(abs, "utf-8")
630
+ };
631
+ });
632
+ const res = await apiRequest(
633
+ opts.options ?? {},
634
+ "POST",
635
+ "/api/public/benchmark-containers/proposals",
636
+ {
637
+ slug: meta.slug,
638
+ title: opts.title,
639
+ changeReason: opts.reason,
640
+ baseVersionId: meta.versionId,
641
+ ...opts.targetBranch ? { targetBranch: opts.targetBranch } : {},
642
+ changes
643
+ }
644
+ );
645
+ return res;
646
+ }
647
+ async function createRef(slug, name, kind, versionId, options = {}) {
648
+ return apiRequest(options, "POST", "/api/public/benchmark-containers/refs", {
649
+ slug,
650
+ name,
651
+ kind,
652
+ ...versionId ? { versionId } : {}
653
+ });
654
+ }
655
+ var DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, sha256b64;
656
+ var init_hub = __esm({
657
+ "src/worlds/hub.ts"() {
658
+ "use strict";
659
+ init_worlds();
660
+ DEFAULT_BRANCH = "main";
661
+ META_RELPATH = path.join(".gateway", ".env-metadata.json");
662
+ SKIP_DIRS = /* @__PURE__ */ new Set([
663
+ ".git",
664
+ "__pycache__",
665
+ "node_modules",
666
+ ".venv",
667
+ ".mypy_cache",
668
+ "dist",
669
+ "build",
670
+ "outputs"
671
+ ]);
672
+ ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
673
+ sha256b64 = (data) => createHash("sha256").update(data).digest("base64");
674
+ }
675
+ });
676
+
677
+ // src/worlds/validate.ts
678
+ function fmtScore(score) {
679
+ return score === null ? "no score" : score.toFixed(3);
680
+ }
681
+ function buildMarkdown(report, minScore) {
682
+ const lines = [];
683
+ lines.push(
684
+ `### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
685
+ );
686
+ lines.push("");
687
+ lines.push(
688
+ `Ran the pull request's own world source (version \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
689
+ );
690
+ lines.push("");
691
+ lines.push("| Model | Status | Score | Run |");
692
+ lines.push("| --- | --- | ---: | --- |");
693
+ for (const run of report.runs) {
694
+ const r = run.report;
695
+ const link = r?.url ? `[open](${r.url})` : "\u2014";
696
+ const status = run.failure ? `\u274C ${run.failure}` : `\u2705 ${r?.status ?? "COMPLETED"}`;
697
+ lines.push(
698
+ `| \`${run.model}\` | ${status} | ${fmtScore(r?.score ?? null)} | ${link} |`
699
+ );
700
+ }
701
+ const taskRows = report.runs.flatMap(
702
+ (run) => (run.report?.samples ?? []).map((sample) => ({
703
+ model: run.model,
704
+ task: sample.task ?? "?",
705
+ reward: sample.reward
706
+ }))
707
+ );
708
+ if (taskRows.length > 0 && taskRows.length <= 40) {
709
+ lines.push("");
710
+ lines.push("<details><summary>Per-task rewards</summary>");
711
+ lines.push("");
712
+ lines.push("| Task | Model | Reward |");
713
+ lines.push("| --- | --- | ---: |");
714
+ for (const row of taskRows) {
715
+ lines.push(
716
+ `| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
717
+ );
718
+ }
719
+ lines.push("");
720
+ lines.push("</details>");
721
+ }
722
+ return lines.join("\n");
723
+ }
724
+ async function validateWorld(dir, opts) {
725
+ const options = opts.options ?? {};
726
+ const progress = opts.onProgress ?? (() => {
727
+ });
728
+ const minScore = opts.minScore ?? null;
729
+ const provenance = detectProvenance();
730
+ const branch = opts.branch ?? provenance.branch;
731
+ if (!branch)
732
+ throw new Error(
733
+ "No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
734
+ );
735
+ progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
736
+ const pushed = await push(dir, {
737
+ slug: opts.slug,
738
+ branch,
739
+ message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
740
+ ...provenance.actor?.name ? { author: provenance.actor.name } : {},
741
+ options
742
+ });
743
+ progress(`pushed version ${pushed.versionId} (${pushed.contentHash.slice(0, 12)})`);
744
+ const configs = await listRunConfigs(options);
745
+ const runConfig = configs.find((c) => c.id === opts.runConfig) ?? configs.find((c) => c.name === opts.runConfig);
746
+ if (!runConfig) {
747
+ const known = configs.map((c) => c.name).join(", ") || "(none)";
748
+ throw new Error(
749
+ `Run config "${opts.runConfig}" not found in this project. Have: ${known}`
750
+ );
751
+ }
752
+ progress(`dispatching run config "${runConfig.name}" (${runConfig.models.join(", ")})\u2026`);
753
+ const dispatched = await apiRequest(options, "POST", "/api/public/benchmark-containers/run-configs/dispatch", {
754
+ runConfigId: runConfig.id,
755
+ versionId: pushed.versionId,
756
+ ...provenance
757
+ });
758
+ const runs = [];
759
+ for (const run of dispatched.runs) {
760
+ if (!run.ok) {
761
+ runs.push({
762
+ model: run.model,
763
+ runId: run.runId,
764
+ report: null,
765
+ failure: `dispatch failed: ${run.error ?? "unknown error"}`
766
+ });
767
+ continue;
768
+ }
769
+ progress(`waiting on ${run.model} (run ${run.runId})\u2026`);
770
+ let report = null;
771
+ let failure = null;
772
+ try {
773
+ report = await waitForRun(run.runId, {
774
+ timeoutMinutes: opts.timeoutMinutes ?? 45,
775
+ options,
776
+ onTick: (r) => progress(
777
+ `\u2026${run.model}: ${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
778
+ )
779
+ });
780
+ if (report.status.toUpperCase() !== "COMPLETED") {
781
+ failure = report.status;
782
+ } else if (minScore !== null) {
783
+ if (report.score === null) failure = "completed without a score";
784
+ else if (report.score < minScore)
785
+ failure = `score ${fmtScore(report.score)} < ${minScore}`;
786
+ }
787
+ } catch (error) {
788
+ failure = error instanceof Error ? error.message : String(error);
789
+ }
790
+ runs.push({ model: run.model, runId: run.runId, report, failure });
791
+ }
792
+ const base = {
793
+ ok: runs.length > 0 && runs.every((r) => r.failure === null),
794
+ slug: pushed.slug,
795
+ branch,
796
+ versionId: pushed.versionId,
797
+ contentHash: pushed.contentHash,
798
+ runConfig: { id: runConfig.id, name: runConfig.name },
799
+ runs
800
+ };
801
+ return { ...base, markdown: buildMarkdown(base, minScore) };
802
+ }
803
+ var init_validate = __esm({
804
+ "src/worlds/validate.ts"() {
805
+ "use strict";
806
+ init_worlds();
807
+ init_hub();
808
+ }
809
+ });
810
+
811
+ // src/worlds/index.ts
812
+ var worlds_exports = {};
813
+ __export(worlds_exports, {
814
+ DEFAULT_BRANCH: () => DEFAULT_BRANCH,
815
+ WorldSession: () => WorldSession,
816
+ WorldSessionCallError: () => WorldSessionCallError,
817
+ WorldSessionTimeout: () => WorldSessionTimeout,
818
+ WorldSet: () => WorldSet,
819
+ WorldTask: () => WorldTask,
820
+ apiRequest: () => apiRequest,
821
+ attachSession: () => attachSession,
822
+ buildSourceTarball: () => buildSourceTarball,
823
+ collectArchiveRelPaths: () => collectArchiveRelPaths,
824
+ computeContentHash: () => computeContentHash,
825
+ createRef: () => createRef,
826
+ deleteEnvSecret: () => deleteEnvSecret,
827
+ detectProvenance: () => detectProvenance,
828
+ getRun: () => getRun,
829
+ isIncludedInContentHash: () => isIncludedInContentHash,
830
+ listEnvSecretKeys: () => listEnvSecretKeys,
831
+ listRunConfigs: () => listRunConfigs,
832
+ localTreeState: () => localTreeState,
833
+ openSession: () => openSession,
834
+ openWorld: () => openWorld,
835
+ propose: () => propose,
836
+ pull: () => pull,
837
+ push: () => push,
838
+ readLocalMeta: () => readLocalMeta,
839
+ runSessions: () => runSessions,
840
+ schemaFromSignature: () => schemaFromSignature,
841
+ setEnvSecret: () => setEnvSecret,
842
+ snapshotTreeHashes: () => snapshotTreeHashes,
843
+ validateWorld: () => validateWorld,
844
+ waitForRun: () => waitForRun,
845
+ writeLocalMeta: () => writeLocalMeta
846
+ });
847
+ function resolveEnv(options) {
848
+ const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
849
+ const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
850
+ const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
851
+ if (!publicKey || !secretKey) {
852
+ throw new Error(
853
+ "Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
854
+ );
855
+ }
856
+ return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
857
+ }
858
+ async function apiRequest(options, method, path2, body) {
859
+ const { host, publicKey, secretKey } = resolveEnv(options);
860
+ const res = await fetch(`${host}${path2}`, {
861
+ method,
862
+ headers: {
863
+ "content-type": "application/json",
864
+ authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
865
+ },
866
+ ...body === void 0 ? {} : { body: JSON.stringify(body) }
867
+ });
868
+ const text = await res.text();
869
+ if (!res.ok) {
870
+ throw new Error(
871
+ `${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
872
+ );
873
+ }
874
+ return JSON.parse(text);
875
+ }
876
+ async function setEnvSecret(slug, key, value, options = {}) {
877
+ return apiRequest(
878
+ options,
879
+ "PUT",
880
+ "/api/public/benchmark-containers/secrets",
881
+ {
882
+ slug,
883
+ key,
884
+ value
885
+ }
886
+ );
887
+ }
888
+ async function listEnvSecretKeys(slug, options = {}) {
889
+ const res = await apiRequest(
890
+ options,
891
+ "GET",
892
+ `/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
893
+ );
894
+ return res.secrets;
895
+ }
896
+ async function deleteEnvSecret(slug, key, options = {}) {
897
+ await apiRequest(
898
+ options,
899
+ "DELETE",
900
+ "/api/public/benchmark-containers/secrets",
901
+ {
902
+ slug,
903
+ key
904
+ }
905
+ );
906
+ }
907
+ async function listRunConfigs(options = {}) {
908
+ const res = await apiRequest(
909
+ options,
910
+ "GET",
911
+ "/api/public/benchmark-containers/run-configs"
912
+ );
913
+ return res.runConfigs;
914
+ }
915
+ async function getRun(runId, options = {}) {
916
+ return apiRequest(
917
+ options,
918
+ "GET",
919
+ `/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
920
+ );
921
+ }
922
+ async function waitForRun(runId, opts = {}) {
923
+ const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
924
+ for (; ; ) {
925
+ const run = await getRun(runId, opts.options ?? {});
926
+ if (run.terminal) return run;
927
+ if (Date.now() > deadline)
928
+ throw new Error(
929
+ `Run ${runId} still ${run.status} after the wait timeout.`
930
+ );
931
+ opts.onTick?.(run);
932
+ await new Promise(
933
+ (resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
934
+ );
935
+ }
936
+ }
937
+ function detectProvenance() {
938
+ const env = globalThis.process?.env ?? {};
939
+ const out = {};
940
+ const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
941
+ if (ref) out.branch = ref;
942
+ if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
943
+ const server = env.GITHUB_SERVER_URL ?? "https://github.com";
944
+ if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
945
+ if (env.GITHUB_ACTOR) {
946
+ out.actor = {
947
+ kind: "ci",
948
+ id: env.GITHUB_ACTOR,
949
+ name: env.GITHUB_ACTOR,
950
+ // GitHub serves every account's avatar at /<login>.png.
951
+ imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
952
+ };
953
+ }
954
+ if (!out.branch || !out.commit) {
955
+ try {
956
+ const { execFileSync } = __require("child_process");
957
+ const git = (args) => execFileSync("git", args, {
958
+ encoding: "utf-8",
959
+ stdio: ["ignore", "pipe", "ignore"]
960
+ }).toString().trim();
961
+ out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
962
+ out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
963
+ if (!out.repoUrl) {
964
+ const remote = git(["config", "--get", "remote.origin.url"]);
965
+ if (remote) out.repoUrl = remote.replace(/\.git$/, "");
966
+ }
967
+ if (!out.actor) {
968
+ const name = git(["config", "--get", "user.name"]);
969
+ if (name) out.actor = { kind: "user", name };
970
+ }
971
+ } catch {
972
+ }
973
+ }
974
+ return out;
975
+ }
976
+ async function openWorld(slugRef, options = {}) {
977
+ const at = slugRef.indexOf("@");
978
+ const slug = at === -1 ? slugRef : slugRef.slice(0, at);
979
+ const ref = at === -1 ? null : slugRef.slice(at + 1);
980
+ const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
981
+ slug,
982
+ ...ref ? { ref } : {}
983
+ });
984
+ const world = new WorldSet(
985
+ slugRef,
986
+ slug,
987
+ ref,
988
+ {
989
+ versionId: version.versionId,
990
+ semanticVersion: version.semanticVersion ?? null,
991
+ contentHash: version.contentHash,
992
+ state: version.state
993
+ },
994
+ [],
995
+ options
996
+ );
997
+ for (const name of version.taskNames ?? []) {
998
+ const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
999
+ world.tasks.push(new WorldTask(world, name, variant));
1000
+ }
1001
+ return world;
1002
+ }
1003
+ var WorldTask, WorldSet;
1004
+ var init_worlds = __esm({
1005
+ "src/worlds/index.ts"() {
1006
+ "use strict";
1007
+ init_session();
1008
+ init_hub();
1009
+ init_validate();
1010
+ WorldTask = class {
1011
+ constructor(world, name, variant) {
1012
+ this.world = world;
1013
+ this.name = name;
1014
+ this.variant = variant;
1015
+ }
1016
+ };
1017
+ WorldSet = class {
1018
+ constructor(slugRef, slug, ref, version, tasks, options) {
1019
+ this.slugRef = slugRef;
1020
+ this.slug = slug;
1021
+ this.ref = ref;
1022
+ this.version = version;
1023
+ this.tasks = tasks;
1024
+ this.options = options;
1025
+ }
1026
+ task(variant) {
1027
+ const hit = this.tasks.find(
1028
+ (t) => t.variant === variant || t.name.includes(variant)
1029
+ );
1030
+ if (!hit) {
1031
+ const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
1032
+ throw new Error(`no task matches ${variant}; have: ${known}`);
1033
+ }
1034
+ return hit;
1035
+ }
1036
+ /**
1037
+ * Open a LIVE session against one of this world's tasks: the platform boots
1038
+ * (or reuses) a warm container, and your agent calls its tools through the
1039
+ * platform. See worlds/session.ts.
1040
+ */
1041
+ async open(task, opts = {}) {
1042
+ const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
1043
+ return openSession2(this.slug, task, {
1044
+ versionId: this.version.versionId,
1045
+ reuseWarm: opts.reuseWarm,
1046
+ options: this.options
1047
+ });
1048
+ }
1049
+ /**
1050
+ * Start a HOSTED run of a run config against this world — the platform
1051
+ * executes the (Python) world in parallel containers per the config's
1052
+ * `parallelism` and files one graded run per model. Returns those runs.
1053
+ */
1054
+ async dispatch(runConfigId, opts = {}) {
1055
+ const provenance = opts.provenance ?? detectProvenance();
1056
+ return apiRequest(
1057
+ this.options,
1058
+ "POST",
1059
+ "/api/public/benchmark-containers/run-configs/dispatch",
1060
+ {
1061
+ runConfigId,
1062
+ ...opts.containerId ? { containerId: opts.containerId } : {},
1063
+ versionId: opts.versionId ?? this.version.versionId,
1064
+ ...provenance
1065
+ }
1066
+ );
1067
+ }
1068
+ };
1069
+ }
1070
+ });
1071
+
1072
+ // src/cli.ts
1073
+ init_worlds();
1074
+ init_hub();
1075
+ init_validate();
1076
+ import * as fs2 from "fs";
1077
+ function parseArgv(argv) {
1078
+ const positional = [];
1079
+ const flags = {};
1080
+ for (let i = 0; i < argv.length; i++) {
1081
+ const arg = argv[i];
1082
+ if (arg.startsWith("--")) {
1083
+ const key = arg.slice(2);
1084
+ const next = argv[i + 1];
1085
+ if (next !== void 0 && !next.startsWith("--")) {
1086
+ flags[key] = next;
1087
+ i++;
1088
+ } else {
1089
+ flags[key] = true;
1090
+ }
1091
+ } else {
1092
+ positional.push(arg);
1093
+ }
1094
+ }
1095
+ return { positional, flags };
1096
+ }
1097
+ function str(flags, key) {
1098
+ const value = flags[key];
1099
+ return typeof value === "string" ? value : void 0;
1100
+ }
1101
+ function fail(message) {
1102
+ console.error(message);
1103
+ process.exit(2);
1104
+ }
1105
+ var PYTHON_ONLY = /* @__PURE__ */ new Set(["eval"]);
1106
+ var HELP = `gateway bench <command> \u2014 worlds from TypeScript
1107
+
1108
+ resolve <slug[@ref]> pinned version + task count
1109
+ tasks <slug[@ref]> the task list
1110
+ dispatch <runConfigId> [--container-id X] [--version-id Y]
1111
+ run <runId> [--wait] [--timeout-min N] status, score, per-task rewards
1112
+ run-configs list
1113
+ run-configs create --name N --models a,b [--parallelism K] [--tasks t1,t2] [--agent A] [--auto-run-on-push] [--id existing]
1114
+ run-configs delete --id X
1115
+ secrets set --env <slug> --key K [--value V] (omit --value to read stdin)
1116
+ secrets list --env <slug>
1117
+ secrets unset --env <slug> --key K
1118
+ refs <slug> branches and tags
1119
+ log <slug> [--branch B] [--limit N] version history
1120
+ proposals <slug> [--status open|applied|dismissed]
1121
+
1122
+ pull <slug[@ref]> [--dir D] materialize a working dir
1123
+ push [dir] [--slug S] [--branch B] [--force] [--message M]
1124
+ validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X]
1125
+ [--summary-file F] [--timeout-min N]
1126
+ push this checkout, run it, gate \u2014 the PR-validation entry
1127
+ checkout <slug@ref> --dir D re-aim an existing working dir
1128
+ status [dir] clean/dirty vs base + hash check
1129
+ branch <slug> <name> [--version-id V] create a branch
1130
+ tag <slug> <name> [--version-id V] create a tag
1131
+ propose [dir] --title T --reason R [--target-branch B]
1132
+
1133
+ eval (local engine runs) is Python-only \u2014 run hosted via 'dispatch'.
1134
+
1135
+ Auth: GATEWAY_HOST, GATEWAY_PUBLIC_KEY, GATEWAY_SECRET_KEY.`;
1136
+ async function readStdin() {
1137
+ const chunks = [];
1138
+ for await (const chunk of process.stdin) chunks.push(chunk);
1139
+ return Buffer.concat(chunks).toString("utf-8").trim();
1140
+ }
1141
+ async function main() {
1142
+ const [group, command, ...rest] = process.argv.slice(2);
1143
+ if (!group || group === "help" || group === "--help") {
1144
+ console.log(HELP);
1145
+ return 0;
1146
+ }
1147
+ if (group !== "bench") {
1148
+ fail(
1149
+ `Unknown command group '${group}' \u2014 this CLI covers 'gateway bench'. Run 'gateway help'.`
1150
+ );
1151
+ }
1152
+ if (!command) fail(HELP);
1153
+ if (PYTHON_ONLY.has(command)) {
1154
+ fail(
1155
+ `'gateway bench eval' runs the world locally on the PYTHON engines (verifiers/harbor) \u2014 it cannot exist in a JS process.
1156
+ Run hosted instead: gateway bench dispatch <runConfigId> \u2014 same worlds, graded on the platform.`
1157
+ );
1158
+ }
1159
+ const { positional, flags } = parseArgv(rest);
1160
+ switch (command) {
1161
+ case "resolve": {
1162
+ const slugRef = positional[0] ?? fail("usage: gateway bench resolve <slug[@ref]>");
1163
+ const world = await openWorld(slugRef);
1164
+ console.log(
1165
+ `${world.slug}@${world.ref ?? "latest"}: ${world.version.semanticVersion ?? "-"} (${world.version.contentHash.slice(0, 12)}) \xB7 ${world.version.state ?? "?"} \xB7 ${world.tasks.length} tasks`
1166
+ );
1167
+ console.log(`versionId: ${world.version.versionId}`);
1168
+ return 0;
1169
+ }
1170
+ case "tasks": {
1171
+ const slugRef = positional[0] ?? fail("usage: gateway bench tasks <slug[@ref]>");
1172
+ const world = await openWorld(slugRef);
1173
+ if (world.tasks.length === 0) {
1174
+ console.log("No task manifest served for this version.");
1175
+ return 0;
1176
+ }
1177
+ for (const task of world.tasks) {
1178
+ console.log(`${task.name}${task.variant ? ` (${task.variant})` : ""}`);
1179
+ }
1180
+ return 0;
1181
+ }
1182
+ case "dispatch": {
1183
+ const runConfigId = positional[0] ?? fail("usage: gateway bench dispatch <runConfigId>");
1184
+ const body = { runConfigId };
1185
+ const containerId = str(flags, "container-id");
1186
+ const versionId = str(flags, "version-id");
1187
+ if (containerId) body.containerId = containerId;
1188
+ if (versionId) body.versionId = versionId;
1189
+ const result = await apiRequest(
1190
+ {},
1191
+ "POST",
1192
+ "/api/public/benchmark-containers/run-configs/dispatch",
1193
+ body
1194
+ );
1195
+ for (const run of result.runs) {
1196
+ console.log(
1197
+ `${run.model}: ${run.ok ? "dispatched" : `FAILED: ${run.error}`} run=${run.runId}`
1198
+ );
1199
+ }
1200
+ console.log(`-> ${result.versionLabel}`);
1201
+ return result.runs.every((r) => r.ok) ? 0 : 1;
1202
+ }
1203
+ case "run": {
1204
+ const runId = positional[0] ?? fail("usage: gateway bench run <runId> [--wait]");
1205
+ const report = flags.wait ? await waitForRun(runId, {
1206
+ timeoutMinutes: Number(str(flags, "timeout-min") ?? 45),
1207
+ onTick: (r) => console.error(
1208
+ `\u2026${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
1209
+ )
1210
+ }) : await getRun(runId);
1211
+ console.log(
1212
+ `${report.id} ${report.status}${report.score !== null ? ` score=${report.score.toFixed(3)}` : ""} (${report.model ?? "?"} \xB7 ${report.versionLabel ?? "?"} \xB7 x${report.parallelism})`
1213
+ );
1214
+ for (const sample of report.samples) {
1215
+ const subs = Object.entries(sample.rewards).filter(([, v]) => typeof v === "number").map(([k, v]) => `${k}=${v.toFixed(2)}`).join(" ");
1216
+ console.log(
1217
+ ` ${(sample.reward ?? 0).toFixed(2)} ${sample.task ?? "?"}${subs ? ` [${subs}]` : ""}`
1218
+ );
1219
+ }
1220
+ if (report.summary) console.log(` ${report.summary}`);
1221
+ return report.terminal && report.status.toUpperCase() === "COMPLETED" ? 0 : 1;
1222
+ }
1223
+ case "run-configs": {
1224
+ const action = positional[0] ?? "list";
1225
+ if (action === "list") {
1226
+ const rows = await listRunConfigs();
1227
+ if (rows.length === 0) {
1228
+ console.log("No run configs in this project.");
1229
+ return 0;
1230
+ }
1231
+ for (const row2 of rows) {
1232
+ const extras = [
1233
+ row2.parallelism > 1 ? `x${row2.parallelism} containers` : null,
1234
+ row2.autoRunOnPush ? "auto-on-push" : null
1235
+ ].filter(Boolean);
1236
+ console.log(
1237
+ `${row2.id} ${row2.name} (${row2.models.join(", ") || "no models"})${extras.length ? ` [${extras.join(", ")}]` : ""}`
1238
+ );
1239
+ }
1240
+ return 0;
1241
+ }
1242
+ if (action === "delete") {
1243
+ const id2 = str(flags, "id") ?? fail("--id is required for delete");
1244
+ await apiRequest(
1245
+ {},
1246
+ "DELETE",
1247
+ "/api/public/benchmark-containers/run-configs",
1248
+ { id: id2 }
1249
+ );
1250
+ console.log(`Deleted ${id2}.`);
1251
+ return 0;
1252
+ }
1253
+ if (action !== "create")
1254
+ fail("run-configs actions: list | create | delete");
1255
+ const name = str(flags, "name") ?? fail("--name is required for create");
1256
+ const models = (str(flags, "models") ?? fail("--models is required for create")).split(",").map((m) => m.trim()).filter(Boolean);
1257
+ const body = { name, models };
1258
+ const parallelism = str(flags, "parallelism");
1259
+ if (parallelism) body.parallelism = Number(parallelism);
1260
+ const agent = str(flags, "agent");
1261
+ if (agent) body.agent = agent;
1262
+ const tasks = str(flags, "tasks");
1263
+ if (tasks)
1264
+ body.taskNames = tasks.split(",").map((t) => t.trim()).filter(Boolean);
1265
+ if (flags["auto-run-on-push"]) body.autoRunOnPush = true;
1266
+ const id = str(flags, "id");
1267
+ if (id) body.id = id;
1268
+ const row = await apiRequest({}, "POST", "/api/public/benchmark-containers/run-configs", body);
1269
+ console.log(`${row.id} ${row.name} (${row.models.join(", ")})`);
1270
+ return 0;
1271
+ }
1272
+ case "secrets": {
1273
+ const action = positional[0] ?? fail("secrets actions: set | list | unset");
1274
+ const env = str(flags, "env") ?? fail("--env <world slug> is required");
1275
+ if (action === "list") {
1276
+ const rows = await listEnvSecretKeys(env);
1277
+ if (rows.length === 0) {
1278
+ console.log(`No secrets on ${env}.`);
1279
+ return 0;
1280
+ }
1281
+ for (const row of rows)
1282
+ console.log(`${row.key} (rotated ${row.updatedAt})`);
1283
+ return 0;
1284
+ }
1285
+ const key = str(flags, "key") ?? fail(`--key is required for ${action}`);
1286
+ if (action === "unset") {
1287
+ await deleteEnvSecret(env, key);
1288
+ console.log(`Deleted ${key} from ${env}.`);
1289
+ return 0;
1290
+ }
1291
+ if (action !== "set") fail("secrets actions: set | list | unset");
1292
+ const value = str(flags, "value") ?? await readStdin();
1293
+ if (!value)
1294
+ fail("empty value \u2014 nothing stored (pass --value or pipe stdin)");
1295
+ const res = await setEnvSecret(env, key, value);
1296
+ console.log(
1297
+ `${res.created ? "Added" : "Rotated"} ${key} on ${env} \u2014 next hosted run receives it.`
1298
+ );
1299
+ return 0;
1300
+ }
1301
+ case "refs": {
1302
+ const slug = positional[0] ?? fail("usage: gateway bench refs <slug>");
1303
+ const res = await apiRequest(
1304
+ {},
1305
+ "GET",
1306
+ `/api/public/benchmark-containers/refs?slug=${encodeURIComponent(slug)}`
1307
+ );
1308
+ for (const ref of res.refs) {
1309
+ console.log(
1310
+ `${ref.kind === "tag" ? "tag " : "branch"} ${ref.name} ${ref.semanticVersion ?? "-"} (${ref.contentHash?.slice(0, 12) ?? "?"}) ${ref.state ?? ""}`
1311
+ );
1312
+ }
1313
+ return 0;
1314
+ }
1315
+ case "log": {
1316
+ const slug = positional[0] ?? fail("usage: gateway bench log <slug>");
1317
+ const params = new URLSearchParams({ slug });
1318
+ const branch = str(flags, "branch");
1319
+ if (branch) params.set("branch", branch);
1320
+ const limit = str(flags, "limit");
1321
+ if (limit) params.set("limit", limit);
1322
+ const res = await apiRequest(
1323
+ {},
1324
+ "GET",
1325
+ `/api/public/benchmark-containers/log?${params.toString()}`
1326
+ );
1327
+ console.log(`branch: ${res.branch}`);
1328
+ for (const entry of res.entries) {
1329
+ console.log(
1330
+ `${entry.contentHash.slice(0, 12)} ${entry.semanticVersion ?? "-"} ${entry.state}${entry.author ? ` by ${entry.author}` : ""}${entry.changeReason ? ` \u2014 ${entry.changeReason}` : ""}`
1331
+ );
1332
+ }
1333
+ return 0;
1334
+ }
1335
+ case "proposals": {
1336
+ const slug = positional[0] ?? fail("usage: gateway bench proposals <slug>");
1337
+ const params = new URLSearchParams({ slug });
1338
+ const status = str(flags, "status");
1339
+ if (status) params.set("status", status);
1340
+ const res = await apiRequest(
1341
+ {},
1342
+ "GET",
1343
+ `/api/public/benchmark-containers/proposals?${params.toString()}`
1344
+ );
1345
+ if (res.proposals.length === 0) {
1346
+ console.log("No proposals.");
1347
+ return 0;
1348
+ }
1349
+ for (const proposal of res.proposals) {
1350
+ console.log(
1351
+ `${proposal.id} [${proposal.status}] ${proposal.title ?? ""}`
1352
+ );
1353
+ }
1354
+ return 0;
1355
+ }
1356
+ case "pull": {
1357
+ const slugRef = positional[0] ?? fail("usage: gateway bench pull <slug[@ref]> [--dir D]");
1358
+ const res = await pull(slugRef, str(flags, "dir"));
1359
+ console.log(
1360
+ `Pulled ${res.slug} -> ${res.dir} (${res.files} files, version ${res.versionId})`
1361
+ );
1362
+ return 0;
1363
+ }
1364
+ case "checkout": {
1365
+ const slugRef = positional[0] ?? fail("usage: gateway bench checkout <slug@ref> --dir D");
1366
+ const dir = str(flags, "dir") ?? fail("--dir is required for checkout (the working dir to re-aim)");
1367
+ const res = await pull(slugRef, dir);
1368
+ console.log(
1369
+ `Checked out ${slugRef} into ${res.dir} (${res.files} files)`
1370
+ );
1371
+ return 0;
1372
+ }
1373
+ case "push": {
1374
+ const dir = positional[0] ?? ".";
1375
+ const res = await push(dir, {
1376
+ slug: str(flags, "slug"),
1377
+ branch: str(flags, "branch"),
1378
+ force: Boolean(flags.force),
1379
+ message: str(flags, "message"),
1380
+ author: str(flags, "author")
1381
+ });
1382
+ console.log(
1383
+ `Pushed ${res.slug}@${res.branch}: version ${res.versionId} (${res.contentHash.slice(0, 12)})`
1384
+ );
1385
+ return 0;
1386
+ }
1387
+ case "validate": {
1388
+ const dir = positional[0] ?? ".";
1389
+ const runConfig = str(flags, "run-config") ?? fail(
1390
+ "usage: gateway bench validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X] [--summary-file F] [--timeout-min N]"
1391
+ );
1392
+ const minScoreRaw = str(flags, "min-score");
1393
+ const report = await validateWorld(dir, {
1394
+ slug: str(flags, "slug"),
1395
+ runConfig,
1396
+ branch: str(flags, "branch"),
1397
+ minScore: minScoreRaw === void 0 ? null : Number(minScoreRaw),
1398
+ timeoutMinutes: Number(str(flags, "timeout-min") ?? 45),
1399
+ onProgress: (line) => console.error(line)
1400
+ });
1401
+ for (const run of report.runs) {
1402
+ console.log(
1403
+ `${run.model}: ${run.failure ? `FAIL \u2014 ${run.failure}` : "pass"}${run.report?.score != null ? ` score=${run.report.score.toFixed(3)}` : ""}${run.report?.url ? ` ${run.report.url}` : ""}`
1404
+ );
1405
+ }
1406
+ const summaryFile = str(flags, "summary-file");
1407
+ if (summaryFile) fs2.writeFileSync(summaryFile, report.markdown + "\n");
1408
+ const stepSummary = process.env.GITHUB_STEP_SUMMARY;
1409
+ if (stepSummary) fs2.appendFileSync(stepSummary, report.markdown + "\n");
1410
+ console.log(report.ok ? "validation passed" : "validation FAILED");
1411
+ return report.ok ? 0 : 1;
1412
+ }
1413
+ case "status": {
1414
+ const dir = positional[0] ?? ".";
1415
+ const state = localTreeState(dir);
1416
+ if (!state.meta) {
1417
+ console.log(
1418
+ "Ad-hoc dir: no pull/push base recorded (.gateway missing)."
1419
+ );
1420
+ return 0;
1421
+ }
1422
+ console.log(
1423
+ `base: ${state.meta.slug}@${state.meta.branch ?? "?"} version ${state.meta.versionId ?? "?"}`
1424
+ );
1425
+ const localHash = computeContentHash(dir);
1426
+ const baseHash = state.meta.contentHash;
1427
+ console.log(
1428
+ `content hash: ${localHash.slice(0, 12)} ${localHash === baseHash ? "== base (hash-verified)" : `!= base ${String(baseHash).slice(0, 12)}`}`
1429
+ );
1430
+ if (state.clean) {
1431
+ console.log("clean \u2014 tree matches its base.");
1432
+ return 0;
1433
+ }
1434
+ console.log(`dirty (${state.dirty.length}):`);
1435
+ for (const p of state.dirty.slice(0, 40)) console.log(` ${p}`);
1436
+ return 0;
1437
+ }
1438
+ case "branch":
1439
+ case "tag": {
1440
+ const slug = positional[0] ?? fail(`usage: gateway bench ${command} <slug> <name>`);
1441
+ const name = positional[1] ?? fail(`usage: gateway bench ${command} <slug> <name>`);
1442
+ const ref = await createRef(
1443
+ slug,
1444
+ name,
1445
+ command,
1446
+ str(flags, "version-id")
1447
+ );
1448
+ console.log(`${ref.kind} ${ref.name} -> ${ref.versionId}`);
1449
+ return 0;
1450
+ }
1451
+ case "propose": {
1452
+ const dir = positional[0] ?? ".";
1453
+ const title = str(flags, "title") ?? fail("--title is required");
1454
+ const reason = str(flags, "reason") ?? fail("--reason is required");
1455
+ const res = await propose(dir, {
1456
+ title,
1457
+ reason,
1458
+ targetBranch: str(flags, "target-branch")
1459
+ });
1460
+ console.log(`Proposal ${res.proposalId} opened.`);
1461
+ return 0;
1462
+ }
1463
+ default:
1464
+ fail(`Unknown command 'bench ${command}'.
1465
+
1466
+ ${HELP}`);
1467
+ }
1468
+ }
1469
+ main().then(
1470
+ (code) => process.exit(code),
1471
+ (error) => {
1472
+ console.error(error instanceof Error ? error.message : String(error));
1473
+ process.exit(1);
1474
+ }
1475
+ );
1476
+ //# sourceMappingURL=cli.mjs.map