@withgateway/sdk 0.1.14 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +475 -271
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +471 -267
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +627 -59
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +602 -58
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +132 -1
- package/dist/worlds/index.d.ts +132 -1
- package/dist/worlds/index.js +560 -6
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +550 -6
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/cli.mjs
CHANGED
|
@@ -312,273 +312,11 @@ var init_session = __esm({
|
|
|
312
312
|
}
|
|
313
313
|
});
|
|
314
314
|
|
|
315
|
-
// src/worlds/index.ts
|
|
316
|
-
var worlds_exports = {};
|
|
317
|
-
__export(worlds_exports, {
|
|
318
|
-
WorldSession: () => WorldSession,
|
|
319
|
-
WorldSessionCallError: () => WorldSessionCallError,
|
|
320
|
-
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
321
|
-
WorldSet: () => WorldSet,
|
|
322
|
-
WorldTask: () => WorldTask,
|
|
323
|
-
apiRequest: () => apiRequest,
|
|
324
|
-
attachSession: () => attachSession,
|
|
325
|
-
deleteEnvSecret: () => deleteEnvSecret,
|
|
326
|
-
detectProvenance: () => detectProvenance,
|
|
327
|
-
getRun: () => getRun,
|
|
328
|
-
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
329
|
-
listRunConfigs: () => listRunConfigs,
|
|
330
|
-
openSession: () => openSession,
|
|
331
|
-
openWorld: () => openWorld,
|
|
332
|
-
runSessions: () => runSessions,
|
|
333
|
-
schemaFromSignature: () => schemaFromSignature,
|
|
334
|
-
setEnvSecret: () => setEnvSecret,
|
|
335
|
-
waitForRun: () => waitForRun
|
|
336
|
-
});
|
|
337
|
-
function resolveEnv(options) {
|
|
338
|
-
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
339
|
-
const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
340
|
-
const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
341
|
-
if (!publicKey || !secretKey) {
|
|
342
|
-
throw new Error(
|
|
343
|
-
"Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
|
|
344
|
-
);
|
|
345
|
-
}
|
|
346
|
-
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
347
|
-
}
|
|
348
|
-
async function apiRequest(options, method, path2, body) {
|
|
349
|
-
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
350
|
-
const res = await fetch(`${host}${path2}`, {
|
|
351
|
-
method,
|
|
352
|
-
headers: {
|
|
353
|
-
"content-type": "application/json",
|
|
354
|
-
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
355
|
-
},
|
|
356
|
-
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
357
|
-
});
|
|
358
|
-
const text = await res.text();
|
|
359
|
-
if (!res.ok) {
|
|
360
|
-
throw new Error(
|
|
361
|
-
`${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
|
|
362
|
-
);
|
|
363
|
-
}
|
|
364
|
-
return JSON.parse(text);
|
|
365
|
-
}
|
|
366
|
-
async function setEnvSecret(slug, key, value, options = {}) {
|
|
367
|
-
return apiRequest(
|
|
368
|
-
options,
|
|
369
|
-
"PUT",
|
|
370
|
-
"/api/public/benchmark-containers/secrets",
|
|
371
|
-
{
|
|
372
|
-
slug,
|
|
373
|
-
key,
|
|
374
|
-
value
|
|
375
|
-
}
|
|
376
|
-
);
|
|
377
|
-
}
|
|
378
|
-
async function listEnvSecretKeys(slug, options = {}) {
|
|
379
|
-
const res = await apiRequest(
|
|
380
|
-
options,
|
|
381
|
-
"GET",
|
|
382
|
-
`/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
|
|
383
|
-
);
|
|
384
|
-
return res.secrets;
|
|
385
|
-
}
|
|
386
|
-
async function deleteEnvSecret(slug, key, options = {}) {
|
|
387
|
-
await apiRequest(
|
|
388
|
-
options,
|
|
389
|
-
"DELETE",
|
|
390
|
-
"/api/public/benchmark-containers/secrets",
|
|
391
|
-
{
|
|
392
|
-
slug,
|
|
393
|
-
key
|
|
394
|
-
}
|
|
395
|
-
);
|
|
396
|
-
}
|
|
397
|
-
async function listRunConfigs(options = {}) {
|
|
398
|
-
const res = await apiRequest(
|
|
399
|
-
options,
|
|
400
|
-
"GET",
|
|
401
|
-
"/api/public/benchmark-containers/run-configs"
|
|
402
|
-
);
|
|
403
|
-
return res.runConfigs;
|
|
404
|
-
}
|
|
405
|
-
async function getRun(runId, options = {}) {
|
|
406
|
-
return apiRequest(
|
|
407
|
-
options,
|
|
408
|
-
"GET",
|
|
409
|
-
`/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
|
|
410
|
-
);
|
|
411
|
-
}
|
|
412
|
-
async function waitForRun(runId, opts = {}) {
|
|
413
|
-
const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
|
|
414
|
-
for (; ; ) {
|
|
415
|
-
const run = await getRun(runId, opts.options ?? {});
|
|
416
|
-
if (run.terminal) return run;
|
|
417
|
-
if (Date.now() > deadline)
|
|
418
|
-
throw new Error(
|
|
419
|
-
`Run ${runId} still ${run.status} after the wait timeout.`
|
|
420
|
-
);
|
|
421
|
-
opts.onTick?.(run);
|
|
422
|
-
await new Promise(
|
|
423
|
-
(resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
|
|
424
|
-
);
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
function detectProvenance() {
|
|
428
|
-
const env = globalThis.process?.env ?? {};
|
|
429
|
-
const out = {};
|
|
430
|
-
const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
431
|
-
if (ref) out.branch = ref;
|
|
432
|
-
if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
|
|
433
|
-
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
434
|
-
if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
|
|
435
|
-
if (env.GITHUB_ACTOR) {
|
|
436
|
-
out.actor = {
|
|
437
|
-
kind: "ci",
|
|
438
|
-
id: env.GITHUB_ACTOR,
|
|
439
|
-
name: env.GITHUB_ACTOR,
|
|
440
|
-
// GitHub serves every account's avatar at /<login>.png.
|
|
441
|
-
imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
|
|
442
|
-
};
|
|
443
|
-
}
|
|
444
|
-
if (!out.branch || !out.commit) {
|
|
445
|
-
try {
|
|
446
|
-
const { execFileSync } = __require("child_process");
|
|
447
|
-
const git = (args) => execFileSync("git", args, {
|
|
448
|
-
encoding: "utf-8",
|
|
449
|
-
stdio: ["ignore", "pipe", "ignore"]
|
|
450
|
-
}).toString().trim();
|
|
451
|
-
out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
452
|
-
out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
|
|
453
|
-
if (!out.repoUrl) {
|
|
454
|
-
const remote = git(["config", "--get", "remote.origin.url"]);
|
|
455
|
-
if (remote) out.repoUrl = remote.replace(/\.git$/, "");
|
|
456
|
-
}
|
|
457
|
-
if (!out.actor) {
|
|
458
|
-
const name = git(["config", "--get", "user.name"]);
|
|
459
|
-
if (name) out.actor = { kind: "user", name };
|
|
460
|
-
}
|
|
461
|
-
} catch {
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
return out;
|
|
465
|
-
}
|
|
466
|
-
async function openWorld(slugRef, options = {}) {
|
|
467
|
-
const at = slugRef.indexOf("@");
|
|
468
|
-
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
469
|
-
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
470
|
-
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
471
|
-
slug,
|
|
472
|
-
...ref ? { ref } : {}
|
|
473
|
-
});
|
|
474
|
-
const world = new WorldSet(
|
|
475
|
-
slugRef,
|
|
476
|
-
slug,
|
|
477
|
-
ref,
|
|
478
|
-
{
|
|
479
|
-
versionId: version.versionId,
|
|
480
|
-
semanticVersion: version.semanticVersion ?? null,
|
|
481
|
-
contentHash: version.contentHash,
|
|
482
|
-
state: version.state
|
|
483
|
-
},
|
|
484
|
-
[],
|
|
485
|
-
options
|
|
486
|
-
);
|
|
487
|
-
for (const name of version.taskNames ?? []) {
|
|
488
|
-
const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
|
|
489
|
-
world.tasks.push(new WorldTask(world, name, variant));
|
|
490
|
-
}
|
|
491
|
-
return world;
|
|
492
|
-
}
|
|
493
|
-
var WorldTask, WorldSet;
|
|
494
|
-
var init_worlds = __esm({
|
|
495
|
-
"src/worlds/index.ts"() {
|
|
496
|
-
"use strict";
|
|
497
|
-
init_session();
|
|
498
|
-
WorldTask = class {
|
|
499
|
-
constructor(world, name, variant) {
|
|
500
|
-
this.world = world;
|
|
501
|
-
this.name = name;
|
|
502
|
-
this.variant = variant;
|
|
503
|
-
}
|
|
504
|
-
};
|
|
505
|
-
WorldSet = class {
|
|
506
|
-
constructor(slugRef, slug, ref, version, tasks, options) {
|
|
507
|
-
this.slugRef = slugRef;
|
|
508
|
-
this.slug = slug;
|
|
509
|
-
this.ref = ref;
|
|
510
|
-
this.version = version;
|
|
511
|
-
this.tasks = tasks;
|
|
512
|
-
this.options = options;
|
|
513
|
-
}
|
|
514
|
-
task(variant) {
|
|
515
|
-
const hit = this.tasks.find(
|
|
516
|
-
(t) => t.variant === variant || t.name.includes(variant)
|
|
517
|
-
);
|
|
518
|
-
if (!hit) {
|
|
519
|
-
const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
|
|
520
|
-
throw new Error(`no task matches ${variant}; have: ${known}`);
|
|
521
|
-
}
|
|
522
|
-
return hit;
|
|
523
|
-
}
|
|
524
|
-
/**
|
|
525
|
-
* Open a LIVE session against one of this world's tasks: the platform boots
|
|
526
|
-
* (or reuses) a warm container, and your agent calls its tools through the
|
|
527
|
-
* platform. See worlds/session.ts.
|
|
528
|
-
*/
|
|
529
|
-
async open(task, opts = {}) {
|
|
530
|
-
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
531
|
-
return openSession2(this.slug, task, {
|
|
532
|
-
versionId: this.version.versionId,
|
|
533
|
-
reuseWarm: opts.reuseWarm,
|
|
534
|
-
options: this.options
|
|
535
|
-
});
|
|
536
|
-
}
|
|
537
|
-
/**
|
|
538
|
-
* Start a HOSTED run of a run config against this world — the platform
|
|
539
|
-
* executes the (Python) world in parallel containers per the config's
|
|
540
|
-
* `parallelism` and files one graded run per model. Returns those runs.
|
|
541
|
-
*/
|
|
542
|
-
async dispatch(runConfigId, opts = {}) {
|
|
543
|
-
const provenance = opts.provenance ?? detectProvenance();
|
|
544
|
-
return apiRequest(
|
|
545
|
-
this.options,
|
|
546
|
-
"POST",
|
|
547
|
-
"/api/public/benchmark-containers/run-configs/dispatch",
|
|
548
|
-
{
|
|
549
|
-
runConfigId,
|
|
550
|
-
...opts.containerId ? { containerId: opts.containerId } : {},
|
|
551
|
-
versionId: opts.versionId ?? this.version.versionId,
|
|
552
|
-
...provenance
|
|
553
|
-
}
|
|
554
|
-
);
|
|
555
|
-
}
|
|
556
|
-
};
|
|
557
|
-
}
|
|
558
|
-
});
|
|
559
|
-
|
|
560
|
-
// src/cli.ts
|
|
561
|
-
init_worlds();
|
|
562
|
-
|
|
563
315
|
// src/worlds/hub.ts
|
|
564
|
-
init_worlds();
|
|
565
316
|
import { createHash } from "crypto";
|
|
566
317
|
import { gzipSync, gunzipSync } from "zlib";
|
|
567
318
|
import * as fs from "fs";
|
|
568
319
|
import * as path from "path";
|
|
569
|
-
var DEFAULT_BRANCH = "main";
|
|
570
|
-
var META_RELPATH = path.join(".gateway", ".env-metadata.json");
|
|
571
|
-
var SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
572
|
-
".git",
|
|
573
|
-
"__pycache__",
|
|
574
|
-
"node_modules",
|
|
575
|
-
".venv",
|
|
576
|
-
".mypy_cache",
|
|
577
|
-
"dist",
|
|
578
|
-
"build",
|
|
579
|
-
"outputs"
|
|
580
|
-
]);
|
|
581
|
-
var ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
582
320
|
function isExcludedDirComponent(component) {
|
|
583
321
|
return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
|
|
584
322
|
}
|
|
@@ -708,7 +446,6 @@ function localTreeState(dir) {
|
|
|
708
446
|
for (const p of Object.keys(base)) if (!(p in now)) dirty.add(p);
|
|
709
447
|
return { meta, dirty: [...dirty].sort(), clean: dirty.size === 0 };
|
|
710
448
|
}
|
|
711
|
-
var sha256b64 = (data) => createHash("sha256").update(data).digest("base64");
|
|
712
449
|
function readPyproject(dir) {
|
|
713
450
|
try {
|
|
714
451
|
const text = fs.readFileSync(path.join(dir, "pyproject.toml"), "utf-8");
|
|
@@ -915,11 +652,449 @@ async function createRef(slug, name, kind, versionId, options = {}) {
|
|
|
915
652
|
...versionId ? { versionId } : {}
|
|
916
653
|
});
|
|
917
654
|
}
|
|
655
|
+
var DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, sha256b64;
|
|
656
|
+
var init_hub = __esm({
|
|
657
|
+
"src/worlds/hub.ts"() {
|
|
658
|
+
"use strict";
|
|
659
|
+
init_worlds();
|
|
660
|
+
DEFAULT_BRANCH = "main";
|
|
661
|
+
META_RELPATH = path.join(".gateway", ".env-metadata.json");
|
|
662
|
+
SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
663
|
+
".git",
|
|
664
|
+
"__pycache__",
|
|
665
|
+
"node_modules",
|
|
666
|
+
".venv",
|
|
667
|
+
".mypy_cache",
|
|
668
|
+
"dist",
|
|
669
|
+
"build",
|
|
670
|
+
"outputs"
|
|
671
|
+
]);
|
|
672
|
+
ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
673
|
+
sha256b64 = (data) => createHash("sha256").update(data).digest("base64");
|
|
674
|
+
}
|
|
675
|
+
});
|
|
918
676
|
|
|
919
|
-
// src/
|
|
920
|
-
function
|
|
921
|
-
|
|
922
|
-
|
|
677
|
+
// src/worlds/validate.ts
|
|
678
|
+
function fmtScore(score) {
|
|
679
|
+
return score === null ? "no score" : score.toFixed(3);
|
|
680
|
+
}
|
|
681
|
+
function buildMarkdown(report, minScore) {
|
|
682
|
+
const lines = [];
|
|
683
|
+
lines.push(
|
|
684
|
+
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
685
|
+
);
|
|
686
|
+
lines.push("");
|
|
687
|
+
lines.push(
|
|
688
|
+
`Ran the pull request's own world source (version \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
689
|
+
);
|
|
690
|
+
lines.push("");
|
|
691
|
+
lines.push("| Model | Status | Score | Run |");
|
|
692
|
+
lines.push("| --- | --- | ---: | --- |");
|
|
693
|
+
for (const run of report.runs) {
|
|
694
|
+
const r = run.report;
|
|
695
|
+
const link = r?.url ? `[open](${r.url})` : "\u2014";
|
|
696
|
+
const status = run.failure ? `\u274C ${run.failure}` : `\u2705 ${r?.status ?? "COMPLETED"}`;
|
|
697
|
+
lines.push(
|
|
698
|
+
`| \`${run.model}\` | ${status} | ${fmtScore(r?.score ?? null)} | ${link} |`
|
|
699
|
+
);
|
|
700
|
+
}
|
|
701
|
+
const taskRows = report.runs.flatMap(
|
|
702
|
+
(run) => (run.report?.samples ?? []).map((sample) => ({
|
|
703
|
+
model: run.model,
|
|
704
|
+
task: sample.task ?? "?",
|
|
705
|
+
reward: sample.reward
|
|
706
|
+
}))
|
|
707
|
+
);
|
|
708
|
+
if (taskRows.length > 0 && taskRows.length <= 40) {
|
|
709
|
+
lines.push("");
|
|
710
|
+
lines.push("<details><summary>Per-task rewards</summary>");
|
|
711
|
+
lines.push("");
|
|
712
|
+
lines.push("| Task | Model | Reward |");
|
|
713
|
+
lines.push("| --- | --- | ---: |");
|
|
714
|
+
for (const row of taskRows) {
|
|
715
|
+
lines.push(
|
|
716
|
+
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
717
|
+
);
|
|
718
|
+
}
|
|
719
|
+
lines.push("");
|
|
720
|
+
lines.push("</details>");
|
|
721
|
+
}
|
|
722
|
+
return lines.join("\n");
|
|
723
|
+
}
|
|
724
|
+
async function validateWorld(dir, opts) {
|
|
725
|
+
const options = opts.options ?? {};
|
|
726
|
+
const progress = opts.onProgress ?? (() => {
|
|
727
|
+
});
|
|
728
|
+
const minScore = opts.minScore ?? null;
|
|
729
|
+
const provenance = detectProvenance();
|
|
730
|
+
const branch = opts.branch ?? provenance.branch;
|
|
731
|
+
if (!branch)
|
|
732
|
+
throw new Error(
|
|
733
|
+
"No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
|
|
734
|
+
);
|
|
735
|
+
progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
|
|
736
|
+
const pushed = await push(dir, {
|
|
737
|
+
slug: opts.slug,
|
|
738
|
+
branch,
|
|
739
|
+
message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
|
|
740
|
+
...provenance.actor?.name ? { author: provenance.actor.name } : {},
|
|
741
|
+
options
|
|
742
|
+
});
|
|
743
|
+
progress(`pushed version ${pushed.versionId} (${pushed.contentHash.slice(0, 12)})`);
|
|
744
|
+
const configs = await listRunConfigs(options);
|
|
745
|
+
const runConfig = configs.find((c) => c.id === opts.runConfig) ?? configs.find((c) => c.name === opts.runConfig);
|
|
746
|
+
if (!runConfig) {
|
|
747
|
+
const known = configs.map((c) => c.name).join(", ") || "(none)";
|
|
748
|
+
throw new Error(
|
|
749
|
+
`Run config "${opts.runConfig}" not found in this project. Have: ${known}`
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
progress(`dispatching run config "${runConfig.name}" (${runConfig.models.join(", ")})\u2026`);
|
|
753
|
+
const dispatched = await apiRequest(options, "POST", "/api/public/benchmark-containers/run-configs/dispatch", {
|
|
754
|
+
runConfigId: runConfig.id,
|
|
755
|
+
versionId: pushed.versionId,
|
|
756
|
+
...provenance
|
|
757
|
+
});
|
|
758
|
+
const host = (options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai").replace(/\/+$/, "");
|
|
759
|
+
let projectId = null;
|
|
760
|
+
try {
|
|
761
|
+
const projects = await apiRequest(
|
|
762
|
+
options,
|
|
763
|
+
"GET",
|
|
764
|
+
"/api/public/projects"
|
|
765
|
+
);
|
|
766
|
+
projectId = projects.data?.[0]?.id ?? null;
|
|
767
|
+
} catch {
|
|
768
|
+
projectId = null;
|
|
769
|
+
}
|
|
770
|
+
const fallbackUrl = (report) => {
|
|
771
|
+
if (report.url) return report.url;
|
|
772
|
+
const evaluationId = report.evaluations[0]?.id;
|
|
773
|
+
return projectId && evaluationId ? `${host}/project/${projectId}/environment/runs/${encodeURIComponent(evaluationId)}` : null;
|
|
774
|
+
};
|
|
775
|
+
const runs = [];
|
|
776
|
+
for (const run of dispatched.runs) {
|
|
777
|
+
if (!run.ok) {
|
|
778
|
+
runs.push({
|
|
779
|
+
model: run.model,
|
|
780
|
+
runId: run.runId,
|
|
781
|
+
report: null,
|
|
782
|
+
failure: `dispatch failed: ${run.error ?? "unknown error"}`
|
|
783
|
+
});
|
|
784
|
+
continue;
|
|
785
|
+
}
|
|
786
|
+
progress(`waiting on ${run.model} (run ${run.runId})\u2026`);
|
|
787
|
+
let report = null;
|
|
788
|
+
let failure = null;
|
|
789
|
+
try {
|
|
790
|
+
report = await waitForRun(run.runId, {
|
|
791
|
+
timeoutMinutes: opts.timeoutMinutes ?? 45,
|
|
792
|
+
options,
|
|
793
|
+
onTick: (r) => progress(
|
|
794
|
+
`\u2026${run.model}: ${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
|
|
795
|
+
)
|
|
796
|
+
});
|
|
797
|
+
report = { ...report, url: fallbackUrl(report) };
|
|
798
|
+
if (report.status.toUpperCase() !== "COMPLETED") {
|
|
799
|
+
failure = report.status;
|
|
800
|
+
} else if (minScore !== null) {
|
|
801
|
+
if (report.score === null) failure = "completed without a score";
|
|
802
|
+
else if (report.score < minScore)
|
|
803
|
+
failure = `score ${fmtScore(report.score)} < ${minScore}`;
|
|
804
|
+
}
|
|
805
|
+
} catch (error) {
|
|
806
|
+
failure = error instanceof Error ? error.message : String(error);
|
|
807
|
+
}
|
|
808
|
+
runs.push({ model: run.model, runId: run.runId, report, failure });
|
|
809
|
+
}
|
|
810
|
+
const base = {
|
|
811
|
+
ok: runs.length > 0 && runs.every((r) => r.failure === null),
|
|
812
|
+
slug: pushed.slug,
|
|
813
|
+
branch,
|
|
814
|
+
versionId: pushed.versionId,
|
|
815
|
+
contentHash: pushed.contentHash,
|
|
816
|
+
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
817
|
+
runs
|
|
818
|
+
};
|
|
819
|
+
return { ...base, markdown: buildMarkdown(base, minScore) };
|
|
820
|
+
}
|
|
821
|
+
var init_validate = __esm({
|
|
822
|
+
"src/worlds/validate.ts"() {
|
|
823
|
+
"use strict";
|
|
824
|
+
init_worlds();
|
|
825
|
+
init_hub();
|
|
826
|
+
}
|
|
827
|
+
});
|
|
828
|
+
|
|
829
|
+
// src/worlds/index.ts
|
|
830
|
+
var worlds_exports = {};
|
|
831
|
+
__export(worlds_exports, {
|
|
832
|
+
DEFAULT_BRANCH: () => DEFAULT_BRANCH,
|
|
833
|
+
WorldSession: () => WorldSession,
|
|
834
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
835
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
836
|
+
WorldSet: () => WorldSet,
|
|
837
|
+
WorldTask: () => WorldTask,
|
|
838
|
+
apiRequest: () => apiRequest,
|
|
839
|
+
attachSession: () => attachSession,
|
|
840
|
+
buildSourceTarball: () => buildSourceTarball,
|
|
841
|
+
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
842
|
+
computeContentHash: () => computeContentHash,
|
|
843
|
+
createRef: () => createRef,
|
|
844
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
845
|
+
detectProvenance: () => detectProvenance,
|
|
846
|
+
getRun: () => getRun,
|
|
847
|
+
isIncludedInContentHash: () => isIncludedInContentHash,
|
|
848
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
849
|
+
listRunConfigs: () => listRunConfigs,
|
|
850
|
+
localTreeState: () => localTreeState,
|
|
851
|
+
openSession: () => openSession,
|
|
852
|
+
openWorld: () => openWorld,
|
|
853
|
+
propose: () => propose,
|
|
854
|
+
pull: () => pull,
|
|
855
|
+
push: () => push,
|
|
856
|
+
readLocalMeta: () => readLocalMeta,
|
|
857
|
+
runSessions: () => runSessions,
|
|
858
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
859
|
+
setEnvSecret: () => setEnvSecret,
|
|
860
|
+
snapshotTreeHashes: () => snapshotTreeHashes,
|
|
861
|
+
validateWorld: () => validateWorld,
|
|
862
|
+
waitForRun: () => waitForRun,
|
|
863
|
+
writeLocalMeta: () => writeLocalMeta
|
|
864
|
+
});
|
|
865
|
+
function resolveEnv(options) {
|
|
866
|
+
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
867
|
+
const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
868
|
+
const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
869
|
+
if (!publicKey || !secretKey) {
|
|
870
|
+
throw new Error(
|
|
871
|
+
"Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
|
|
872
|
+
);
|
|
873
|
+
}
|
|
874
|
+
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
875
|
+
}
|
|
876
|
+
async function apiRequest(options, method, path2, body) {
|
|
877
|
+
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
878
|
+
const res = await fetch(`${host}${path2}`, {
|
|
879
|
+
method,
|
|
880
|
+
headers: {
|
|
881
|
+
"content-type": "application/json",
|
|
882
|
+
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
883
|
+
},
|
|
884
|
+
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
885
|
+
});
|
|
886
|
+
const text = await res.text();
|
|
887
|
+
if (!res.ok) {
|
|
888
|
+
throw new Error(
|
|
889
|
+
`${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
|
|
890
|
+
);
|
|
891
|
+
}
|
|
892
|
+
return JSON.parse(text);
|
|
893
|
+
}
|
|
894
|
+
async function setEnvSecret(slug, key, value, options = {}) {
|
|
895
|
+
return apiRequest(
|
|
896
|
+
options,
|
|
897
|
+
"PUT",
|
|
898
|
+
"/api/public/benchmark-containers/secrets",
|
|
899
|
+
{
|
|
900
|
+
slug,
|
|
901
|
+
key,
|
|
902
|
+
value
|
|
903
|
+
}
|
|
904
|
+
);
|
|
905
|
+
}
|
|
906
|
+
async function listEnvSecretKeys(slug, options = {}) {
|
|
907
|
+
const res = await apiRequest(
|
|
908
|
+
options,
|
|
909
|
+
"GET",
|
|
910
|
+
`/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
|
|
911
|
+
);
|
|
912
|
+
return res.secrets;
|
|
913
|
+
}
|
|
914
|
+
async function deleteEnvSecret(slug, key, options = {}) {
|
|
915
|
+
await apiRequest(
|
|
916
|
+
options,
|
|
917
|
+
"DELETE",
|
|
918
|
+
"/api/public/benchmark-containers/secrets",
|
|
919
|
+
{
|
|
920
|
+
slug,
|
|
921
|
+
key
|
|
922
|
+
}
|
|
923
|
+
);
|
|
924
|
+
}
|
|
925
|
+
async function listRunConfigs(options = {}) {
|
|
926
|
+
const res = await apiRequest(
|
|
927
|
+
options,
|
|
928
|
+
"GET",
|
|
929
|
+
"/api/public/benchmark-containers/run-configs"
|
|
930
|
+
);
|
|
931
|
+
return res.runConfigs;
|
|
932
|
+
}
|
|
933
|
+
async function getRun(runId, options = {}) {
|
|
934
|
+
return apiRequest(
|
|
935
|
+
options,
|
|
936
|
+
"GET",
|
|
937
|
+
`/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
|
|
938
|
+
);
|
|
939
|
+
}
|
|
940
|
+
async function waitForRun(runId, opts = {}) {
|
|
941
|
+
const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
|
|
942
|
+
for (; ; ) {
|
|
943
|
+
const run = await getRun(runId, opts.options ?? {});
|
|
944
|
+
if (run.terminal) return run;
|
|
945
|
+
if (Date.now() > deadline)
|
|
946
|
+
throw new Error(
|
|
947
|
+
`Run ${runId} still ${run.status} after the wait timeout.`
|
|
948
|
+
);
|
|
949
|
+
opts.onTick?.(run);
|
|
950
|
+
await new Promise(
|
|
951
|
+
(resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
function detectProvenance() {
|
|
956
|
+
const env = globalThis.process?.env ?? {};
|
|
957
|
+
const out = {};
|
|
958
|
+
const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
959
|
+
if (ref) out.branch = ref;
|
|
960
|
+
if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
|
|
961
|
+
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
962
|
+
if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
|
|
963
|
+
if (env.GITHUB_ACTOR) {
|
|
964
|
+
out.actor = {
|
|
965
|
+
kind: "ci",
|
|
966
|
+
id: env.GITHUB_ACTOR,
|
|
967
|
+
name: env.GITHUB_ACTOR,
|
|
968
|
+
// GitHub serves every account's avatar at /<login>.png.
|
|
969
|
+
imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
|
|
970
|
+
};
|
|
971
|
+
}
|
|
972
|
+
if (!out.branch || !out.commit) {
|
|
973
|
+
try {
|
|
974
|
+
const { execFileSync } = __require("child_process");
|
|
975
|
+
const git = (args) => execFileSync("git", args, {
|
|
976
|
+
encoding: "utf-8",
|
|
977
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
978
|
+
}).toString().trim();
|
|
979
|
+
out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
980
|
+
out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
|
|
981
|
+
if (!out.repoUrl) {
|
|
982
|
+
const remote = git(["config", "--get", "remote.origin.url"]);
|
|
983
|
+
if (remote) out.repoUrl = remote.replace(/\.git$/, "");
|
|
984
|
+
}
|
|
985
|
+
if (!out.actor) {
|
|
986
|
+
const name = git(["config", "--get", "user.name"]);
|
|
987
|
+
if (name) out.actor = { kind: "user", name };
|
|
988
|
+
}
|
|
989
|
+
} catch {
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
return out;
|
|
993
|
+
}
|
|
994
|
+
async function openWorld(slugRef, options = {}) {
|
|
995
|
+
const at = slugRef.indexOf("@");
|
|
996
|
+
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
997
|
+
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
998
|
+
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
999
|
+
slug,
|
|
1000
|
+
...ref ? { ref } : {}
|
|
1001
|
+
});
|
|
1002
|
+
const world = new WorldSet(
|
|
1003
|
+
slugRef,
|
|
1004
|
+
slug,
|
|
1005
|
+
ref,
|
|
1006
|
+
{
|
|
1007
|
+
versionId: version.versionId,
|
|
1008
|
+
semanticVersion: version.semanticVersion ?? null,
|
|
1009
|
+
contentHash: version.contentHash,
|
|
1010
|
+
state: version.state
|
|
1011
|
+
},
|
|
1012
|
+
[],
|
|
1013
|
+
options
|
|
1014
|
+
);
|
|
1015
|
+
for (const name of version.taskNames ?? []) {
|
|
1016
|
+
const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
|
|
1017
|
+
world.tasks.push(new WorldTask(world, name, variant));
|
|
1018
|
+
}
|
|
1019
|
+
return world;
|
|
1020
|
+
}
|
|
1021
|
+
var WorldTask, WorldSet;
|
|
1022
|
+
var init_worlds = __esm({
|
|
1023
|
+
"src/worlds/index.ts"() {
|
|
1024
|
+
"use strict";
|
|
1025
|
+
init_session();
|
|
1026
|
+
init_hub();
|
|
1027
|
+
init_validate();
|
|
1028
|
+
WorldTask = class {
|
|
1029
|
+
constructor(world, name, variant) {
|
|
1030
|
+
this.world = world;
|
|
1031
|
+
this.name = name;
|
|
1032
|
+
this.variant = variant;
|
|
1033
|
+
}
|
|
1034
|
+
};
|
|
1035
|
+
WorldSet = class {
|
|
1036
|
+
constructor(slugRef, slug, ref, version, tasks, options) {
|
|
1037
|
+
this.slugRef = slugRef;
|
|
1038
|
+
this.slug = slug;
|
|
1039
|
+
this.ref = ref;
|
|
1040
|
+
this.version = version;
|
|
1041
|
+
this.tasks = tasks;
|
|
1042
|
+
this.options = options;
|
|
1043
|
+
}
|
|
1044
|
+
task(variant) {
|
|
1045
|
+
const hit = this.tasks.find(
|
|
1046
|
+
(t) => t.variant === variant || t.name.includes(variant)
|
|
1047
|
+
);
|
|
1048
|
+
if (!hit) {
|
|
1049
|
+
const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
|
|
1050
|
+
throw new Error(`no task matches ${variant}; have: ${known}`);
|
|
1051
|
+
}
|
|
1052
|
+
return hit;
|
|
1053
|
+
}
|
|
1054
|
+
/**
|
|
1055
|
+
* Open a LIVE session against one of this world's tasks: the platform boots
|
|
1056
|
+
* (or reuses) a warm container, and your agent calls its tools through the
|
|
1057
|
+
* platform. See worlds/session.ts.
|
|
1058
|
+
*/
|
|
1059
|
+
async open(task, opts = {}) {
|
|
1060
|
+
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
1061
|
+
return openSession2(this.slug, task, {
|
|
1062
|
+
versionId: this.version.versionId,
|
|
1063
|
+
reuseWarm: opts.reuseWarm,
|
|
1064
|
+
options: this.options
|
|
1065
|
+
});
|
|
1066
|
+
}
|
|
1067
|
+
/**
|
|
1068
|
+
* Start a HOSTED run of a run config against this world — the platform
|
|
1069
|
+
* executes the (Python) world in parallel containers per the config's
|
|
1070
|
+
* `parallelism` and files one graded run per model. Returns those runs.
|
|
1071
|
+
*/
|
|
1072
|
+
async dispatch(runConfigId, opts = {}) {
|
|
1073
|
+
const provenance = opts.provenance ?? detectProvenance();
|
|
1074
|
+
return apiRequest(
|
|
1075
|
+
this.options,
|
|
1076
|
+
"POST",
|
|
1077
|
+
"/api/public/benchmark-containers/run-configs/dispatch",
|
|
1078
|
+
{
|
|
1079
|
+
runConfigId,
|
|
1080
|
+
...opts.containerId ? { containerId: opts.containerId } : {},
|
|
1081
|
+
versionId: opts.versionId ?? this.version.versionId,
|
|
1082
|
+
...provenance
|
|
1083
|
+
}
|
|
1084
|
+
);
|
|
1085
|
+
}
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
1088
|
+
});
|
|
1089
|
+
|
|
1090
|
+
// src/cli.ts
|
|
1091
|
+
init_worlds();
|
|
1092
|
+
init_hub();
|
|
1093
|
+
init_validate();
|
|
1094
|
+
import * as fs2 from "fs";
|
|
1095
|
+
function parseArgv(argv) {
|
|
1096
|
+
const positional = [];
|
|
1097
|
+
const flags = {};
|
|
923
1098
|
for (let i = 0; i < argv.length; i++) {
|
|
924
1099
|
const arg = argv[i];
|
|
925
1100
|
if (arg.startsWith("--")) {
|
|
@@ -964,6 +1139,9 @@ var HELP = `gateway bench <command> \u2014 worlds from TypeScript
|
|
|
964
1139
|
|
|
965
1140
|
pull <slug[@ref]> [--dir D] materialize a working dir
|
|
966
1141
|
push [dir] [--slug S] [--branch B] [--force] [--message M]
|
|
1142
|
+
validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X]
|
|
1143
|
+
[--summary-file F] [--timeout-min N]
|
|
1144
|
+
push this checkout, run it, gate \u2014 the PR-validation entry
|
|
967
1145
|
checkout <slug@ref> --dir D re-aim an existing working dir
|
|
968
1146
|
status [dir] clean/dirty vs base + hash check
|
|
969
1147
|
branch <slug> <name> [--version-id V] create a branch
|
|
@@ -1224,6 +1402,32 @@ Run hosted instead: gateway bench dispatch <runConfigId> \u2014 same worlds, gra
|
|
|
1224
1402
|
);
|
|
1225
1403
|
return 0;
|
|
1226
1404
|
}
|
|
1405
|
+
case "validate": {
|
|
1406
|
+
const dir = positional[0] ?? ".";
|
|
1407
|
+
const runConfig = str(flags, "run-config") ?? fail(
|
|
1408
|
+
"usage: gateway bench validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X] [--summary-file F] [--timeout-min N]"
|
|
1409
|
+
);
|
|
1410
|
+
const minScoreRaw = str(flags, "min-score");
|
|
1411
|
+
const report = await validateWorld(dir, {
|
|
1412
|
+
slug: str(flags, "slug"),
|
|
1413
|
+
runConfig,
|
|
1414
|
+
branch: str(flags, "branch"),
|
|
1415
|
+
minScore: minScoreRaw === void 0 ? null : Number(minScoreRaw),
|
|
1416
|
+
timeoutMinutes: Number(str(flags, "timeout-min") ?? 45),
|
|
1417
|
+
onProgress: (line) => console.error(line)
|
|
1418
|
+
});
|
|
1419
|
+
for (const run of report.runs) {
|
|
1420
|
+
console.log(
|
|
1421
|
+
`${run.model}: ${run.failure ? `FAIL \u2014 ${run.failure}` : "pass"}${run.report?.score != null ? ` score=${run.report.score.toFixed(3)}` : ""}${run.report?.url ? ` ${run.report.url}` : ""}`
|
|
1422
|
+
);
|
|
1423
|
+
}
|
|
1424
|
+
const summaryFile = str(flags, "summary-file");
|
|
1425
|
+
if (summaryFile) fs2.writeFileSync(summaryFile, report.markdown + "\n");
|
|
1426
|
+
const stepSummary = process.env.GITHUB_STEP_SUMMARY;
|
|
1427
|
+
if (stepSummary) fs2.appendFileSync(stepSummary, report.markdown + "\n");
|
|
1428
|
+
console.log(report.ok ? "validation passed" : "validation FAILED");
|
|
1429
|
+
return report.ok ? 0 : 1;
|
|
1430
|
+
}
|
|
1227
1431
|
case "status": {
|
|
1228
1432
|
const dir = positional[0] ?? ".";
|
|
1229
1433
|
const state = localTreeState(dir);
|