@withgateway/sdk 0.1.14 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +475 -271
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +471 -267
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +627 -59
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +602 -58
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +132 -1
- package/dist/worlds/index.d.ts +132 -1
- package/dist/worlds/index.js +560 -6
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +550 -6
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -325,273 +325,7 @@ var init_session = __esm({
|
|
|
325
325
|
}
|
|
326
326
|
});
|
|
327
327
|
|
|
328
|
-
// src/worlds/index.ts
|
|
329
|
-
var worlds_exports = {};
|
|
330
|
-
__export(worlds_exports, {
|
|
331
|
-
WorldSession: () => WorldSession,
|
|
332
|
-
WorldSessionCallError: () => WorldSessionCallError,
|
|
333
|
-
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
334
|
-
WorldSet: () => WorldSet,
|
|
335
|
-
WorldTask: () => WorldTask,
|
|
336
|
-
apiRequest: () => apiRequest,
|
|
337
|
-
attachSession: () => attachSession,
|
|
338
|
-
deleteEnvSecret: () => deleteEnvSecret,
|
|
339
|
-
detectProvenance: () => detectProvenance,
|
|
340
|
-
getRun: () => getRun,
|
|
341
|
-
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
342
|
-
listRunConfigs: () => listRunConfigs,
|
|
343
|
-
openSession: () => openSession,
|
|
344
|
-
openWorld: () => openWorld,
|
|
345
|
-
runSessions: () => runSessions,
|
|
346
|
-
schemaFromSignature: () => schemaFromSignature,
|
|
347
|
-
setEnvSecret: () => setEnvSecret,
|
|
348
|
-
waitForRun: () => waitForRun
|
|
349
|
-
});
|
|
350
|
-
function resolveEnv(options) {
|
|
351
|
-
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
352
|
-
const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
353
|
-
const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
354
|
-
if (!publicKey || !secretKey) {
|
|
355
|
-
throw new Error(
|
|
356
|
-
"Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
|
|
357
|
-
);
|
|
358
|
-
}
|
|
359
|
-
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
360
|
-
}
|
|
361
|
-
async function apiRequest(options, method, path2, body) {
|
|
362
|
-
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
363
|
-
const res = await fetch(`${host}${path2}`, {
|
|
364
|
-
method,
|
|
365
|
-
headers: {
|
|
366
|
-
"content-type": "application/json",
|
|
367
|
-
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
368
|
-
},
|
|
369
|
-
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
370
|
-
});
|
|
371
|
-
const text = await res.text();
|
|
372
|
-
if (!res.ok) {
|
|
373
|
-
throw new Error(
|
|
374
|
-
`${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
|
|
375
|
-
);
|
|
376
|
-
}
|
|
377
|
-
return JSON.parse(text);
|
|
378
|
-
}
|
|
379
|
-
async function setEnvSecret(slug, key, value, options = {}) {
|
|
380
|
-
return apiRequest(
|
|
381
|
-
options,
|
|
382
|
-
"PUT",
|
|
383
|
-
"/api/public/benchmark-containers/secrets",
|
|
384
|
-
{
|
|
385
|
-
slug,
|
|
386
|
-
key,
|
|
387
|
-
value
|
|
388
|
-
}
|
|
389
|
-
);
|
|
390
|
-
}
|
|
391
|
-
async function listEnvSecretKeys(slug, options = {}) {
|
|
392
|
-
const res = await apiRequest(
|
|
393
|
-
options,
|
|
394
|
-
"GET",
|
|
395
|
-
`/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
|
|
396
|
-
);
|
|
397
|
-
return res.secrets;
|
|
398
|
-
}
|
|
399
|
-
async function deleteEnvSecret(slug, key, options = {}) {
|
|
400
|
-
await apiRequest(
|
|
401
|
-
options,
|
|
402
|
-
"DELETE",
|
|
403
|
-
"/api/public/benchmark-containers/secrets",
|
|
404
|
-
{
|
|
405
|
-
slug,
|
|
406
|
-
key
|
|
407
|
-
}
|
|
408
|
-
);
|
|
409
|
-
}
|
|
410
|
-
async function listRunConfigs(options = {}) {
|
|
411
|
-
const res = await apiRequest(
|
|
412
|
-
options,
|
|
413
|
-
"GET",
|
|
414
|
-
"/api/public/benchmark-containers/run-configs"
|
|
415
|
-
);
|
|
416
|
-
return res.runConfigs;
|
|
417
|
-
}
|
|
418
|
-
async function getRun(runId, options = {}) {
|
|
419
|
-
return apiRequest(
|
|
420
|
-
options,
|
|
421
|
-
"GET",
|
|
422
|
-
`/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
|
|
423
|
-
);
|
|
424
|
-
}
|
|
425
|
-
async function waitForRun(runId, opts = {}) {
|
|
426
|
-
const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
|
|
427
|
-
for (; ; ) {
|
|
428
|
-
const run = await getRun(runId, opts.options ?? {});
|
|
429
|
-
if (run.terminal) return run;
|
|
430
|
-
if (Date.now() > deadline)
|
|
431
|
-
throw new Error(
|
|
432
|
-
`Run ${runId} still ${run.status} after the wait timeout.`
|
|
433
|
-
);
|
|
434
|
-
opts.onTick?.(run);
|
|
435
|
-
await new Promise(
|
|
436
|
-
(resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
|
|
437
|
-
);
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
function detectProvenance() {
|
|
441
|
-
const env = globalThis.process?.env ?? {};
|
|
442
|
-
const out = {};
|
|
443
|
-
const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
444
|
-
if (ref) out.branch = ref;
|
|
445
|
-
if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
|
|
446
|
-
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
447
|
-
if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
|
|
448
|
-
if (env.GITHUB_ACTOR) {
|
|
449
|
-
out.actor = {
|
|
450
|
-
kind: "ci",
|
|
451
|
-
id: env.GITHUB_ACTOR,
|
|
452
|
-
name: env.GITHUB_ACTOR,
|
|
453
|
-
// GitHub serves every account's avatar at /<login>.png.
|
|
454
|
-
imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
|
|
455
|
-
};
|
|
456
|
-
}
|
|
457
|
-
if (!out.branch || !out.commit) {
|
|
458
|
-
try {
|
|
459
|
-
const { execFileSync } = require("child_process");
|
|
460
|
-
const git = (args) => execFileSync("git", args, {
|
|
461
|
-
encoding: "utf-8",
|
|
462
|
-
stdio: ["ignore", "pipe", "ignore"]
|
|
463
|
-
}).toString().trim();
|
|
464
|
-
out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
465
|
-
out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
|
|
466
|
-
if (!out.repoUrl) {
|
|
467
|
-
const remote = git(["config", "--get", "remote.origin.url"]);
|
|
468
|
-
if (remote) out.repoUrl = remote.replace(/\.git$/, "");
|
|
469
|
-
}
|
|
470
|
-
if (!out.actor) {
|
|
471
|
-
const name = git(["config", "--get", "user.name"]);
|
|
472
|
-
if (name) out.actor = { kind: "user", name };
|
|
473
|
-
}
|
|
474
|
-
} catch {
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
return out;
|
|
478
|
-
}
|
|
479
|
-
async function openWorld(slugRef, options = {}) {
|
|
480
|
-
const at = slugRef.indexOf("@");
|
|
481
|
-
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
482
|
-
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
483
|
-
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
484
|
-
slug,
|
|
485
|
-
...ref ? { ref } : {}
|
|
486
|
-
});
|
|
487
|
-
const world = new WorldSet(
|
|
488
|
-
slugRef,
|
|
489
|
-
slug,
|
|
490
|
-
ref,
|
|
491
|
-
{
|
|
492
|
-
versionId: version.versionId,
|
|
493
|
-
semanticVersion: version.semanticVersion ?? null,
|
|
494
|
-
contentHash: version.contentHash,
|
|
495
|
-
state: version.state
|
|
496
|
-
},
|
|
497
|
-
[],
|
|
498
|
-
options
|
|
499
|
-
);
|
|
500
|
-
for (const name of version.taskNames ?? []) {
|
|
501
|
-
const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
|
|
502
|
-
world.tasks.push(new WorldTask(world, name, variant));
|
|
503
|
-
}
|
|
504
|
-
return world;
|
|
505
|
-
}
|
|
506
|
-
var WorldTask, WorldSet;
|
|
507
|
-
var init_worlds = __esm({
|
|
508
|
-
"src/worlds/index.ts"() {
|
|
509
|
-
"use strict";
|
|
510
|
-
init_session();
|
|
511
|
-
WorldTask = class {
|
|
512
|
-
constructor(world, name, variant) {
|
|
513
|
-
this.world = world;
|
|
514
|
-
this.name = name;
|
|
515
|
-
this.variant = variant;
|
|
516
|
-
}
|
|
517
|
-
};
|
|
518
|
-
WorldSet = class {
|
|
519
|
-
constructor(slugRef, slug, ref, version, tasks, options) {
|
|
520
|
-
this.slugRef = slugRef;
|
|
521
|
-
this.slug = slug;
|
|
522
|
-
this.ref = ref;
|
|
523
|
-
this.version = version;
|
|
524
|
-
this.tasks = tasks;
|
|
525
|
-
this.options = options;
|
|
526
|
-
}
|
|
527
|
-
task(variant) {
|
|
528
|
-
const hit = this.tasks.find(
|
|
529
|
-
(t) => t.variant === variant || t.name.includes(variant)
|
|
530
|
-
);
|
|
531
|
-
if (!hit) {
|
|
532
|
-
const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
|
|
533
|
-
throw new Error(`no task matches ${variant}; have: ${known}`);
|
|
534
|
-
}
|
|
535
|
-
return hit;
|
|
536
|
-
}
|
|
537
|
-
/**
|
|
538
|
-
* Open a LIVE session against one of this world's tasks: the platform boots
|
|
539
|
-
* (or reuses) a warm container, and your agent calls its tools through the
|
|
540
|
-
* platform. See worlds/session.ts.
|
|
541
|
-
*/
|
|
542
|
-
async open(task, opts = {}) {
|
|
543
|
-
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
544
|
-
return openSession2(this.slug, task, {
|
|
545
|
-
versionId: this.version.versionId,
|
|
546
|
-
reuseWarm: opts.reuseWarm,
|
|
547
|
-
options: this.options
|
|
548
|
-
});
|
|
549
|
-
}
|
|
550
|
-
/**
|
|
551
|
-
* Start a HOSTED run of a run config against this world — the platform
|
|
552
|
-
* executes the (Python) world in parallel containers per the config's
|
|
553
|
-
* `parallelism` and files one graded run per model. Returns those runs.
|
|
554
|
-
*/
|
|
555
|
-
async dispatch(runConfigId, opts = {}) {
|
|
556
|
-
const provenance = opts.provenance ?? detectProvenance();
|
|
557
|
-
return apiRequest(
|
|
558
|
-
this.options,
|
|
559
|
-
"POST",
|
|
560
|
-
"/api/public/benchmark-containers/run-configs/dispatch",
|
|
561
|
-
{
|
|
562
|
-
runConfigId,
|
|
563
|
-
...opts.containerId ? { containerId: opts.containerId } : {},
|
|
564
|
-
versionId: opts.versionId ?? this.version.versionId,
|
|
565
|
-
...provenance
|
|
566
|
-
}
|
|
567
|
-
);
|
|
568
|
-
}
|
|
569
|
-
};
|
|
570
|
-
}
|
|
571
|
-
});
|
|
572
|
-
|
|
573
|
-
// src/cli.ts
|
|
574
|
-
init_worlds();
|
|
575
|
-
|
|
576
328
|
// src/worlds/hub.ts
|
|
577
|
-
var import_node_crypto = require("crypto");
|
|
578
|
-
var import_node_zlib = require("zlib");
|
|
579
|
-
var fs = __toESM(require("fs"));
|
|
580
|
-
var path = __toESM(require("path"));
|
|
581
|
-
init_worlds();
|
|
582
|
-
var DEFAULT_BRANCH = "main";
|
|
583
|
-
var META_RELPATH = path.join(".gateway", ".env-metadata.json");
|
|
584
|
-
var SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
585
|
-
".git",
|
|
586
|
-
"__pycache__",
|
|
587
|
-
"node_modules",
|
|
588
|
-
".venv",
|
|
589
|
-
".mypy_cache",
|
|
590
|
-
"dist",
|
|
591
|
-
"build",
|
|
592
|
-
"outputs"
|
|
593
|
-
]);
|
|
594
|
-
var ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
595
329
|
function isExcludedDirComponent(component) {
|
|
596
330
|
return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
|
|
597
331
|
}
|
|
@@ -721,7 +455,6 @@ function localTreeState(dir) {
|
|
|
721
455
|
for (const p of Object.keys(base)) if (!(p in now)) dirty.add(p);
|
|
722
456
|
return { meta, dirty: [...dirty].sort(), clean: dirty.size === 0 };
|
|
723
457
|
}
|
|
724
|
-
var sha256b64 = (data) => (0, import_node_crypto.createHash)("sha256").update(data).digest("base64");
|
|
725
458
|
function readPyproject(dir) {
|
|
726
459
|
try {
|
|
727
460
|
const text = fs.readFileSync(path.join(dir, "pyproject.toml"), "utf-8");
|
|
@@ -928,11 +661,453 @@ async function createRef(slug, name, kind, versionId, options = {}) {
|
|
|
928
661
|
...versionId ? { versionId } : {}
|
|
929
662
|
});
|
|
930
663
|
}
|
|
664
|
+
var import_node_crypto, import_node_zlib, fs, path, DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, sha256b64;
|
|
665
|
+
var init_hub = __esm({
|
|
666
|
+
"src/worlds/hub.ts"() {
|
|
667
|
+
"use strict";
|
|
668
|
+
import_node_crypto = require("crypto");
|
|
669
|
+
import_node_zlib = require("zlib");
|
|
670
|
+
fs = __toESM(require("fs"));
|
|
671
|
+
path = __toESM(require("path"));
|
|
672
|
+
init_worlds();
|
|
673
|
+
DEFAULT_BRANCH = "main";
|
|
674
|
+
META_RELPATH = path.join(".gateway", ".env-metadata.json");
|
|
675
|
+
SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
676
|
+
".git",
|
|
677
|
+
"__pycache__",
|
|
678
|
+
"node_modules",
|
|
679
|
+
".venv",
|
|
680
|
+
".mypy_cache",
|
|
681
|
+
"dist",
|
|
682
|
+
"build",
|
|
683
|
+
"outputs"
|
|
684
|
+
]);
|
|
685
|
+
ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
686
|
+
sha256b64 = (data) => (0, import_node_crypto.createHash)("sha256").update(data).digest("base64");
|
|
687
|
+
}
|
|
688
|
+
});
|
|
931
689
|
|
|
932
|
-
// src/
|
|
933
|
-
function
|
|
934
|
-
|
|
935
|
-
|
|
690
|
+
// src/worlds/validate.ts
|
|
691
|
+
function fmtScore(score) {
|
|
692
|
+
return score === null ? "no score" : score.toFixed(3);
|
|
693
|
+
}
|
|
694
|
+
function buildMarkdown(report, minScore) {
|
|
695
|
+
const lines = [];
|
|
696
|
+
lines.push(
|
|
697
|
+
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
698
|
+
);
|
|
699
|
+
lines.push("");
|
|
700
|
+
lines.push(
|
|
701
|
+
`Ran the pull request's own world source (version \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
702
|
+
);
|
|
703
|
+
lines.push("");
|
|
704
|
+
lines.push("| Model | Status | Score | Run |");
|
|
705
|
+
lines.push("| --- | --- | ---: | --- |");
|
|
706
|
+
for (const run of report.runs) {
|
|
707
|
+
const r = run.report;
|
|
708
|
+
const link = r?.url ? `[open](${r.url})` : "\u2014";
|
|
709
|
+
const status = run.failure ? `\u274C ${run.failure}` : `\u2705 ${r?.status ?? "COMPLETED"}`;
|
|
710
|
+
lines.push(
|
|
711
|
+
`| \`${run.model}\` | ${status} | ${fmtScore(r?.score ?? null)} | ${link} |`
|
|
712
|
+
);
|
|
713
|
+
}
|
|
714
|
+
const taskRows = report.runs.flatMap(
|
|
715
|
+
(run) => (run.report?.samples ?? []).map((sample) => ({
|
|
716
|
+
model: run.model,
|
|
717
|
+
task: sample.task ?? "?",
|
|
718
|
+
reward: sample.reward
|
|
719
|
+
}))
|
|
720
|
+
);
|
|
721
|
+
if (taskRows.length > 0 && taskRows.length <= 40) {
|
|
722
|
+
lines.push("");
|
|
723
|
+
lines.push("<details><summary>Per-task rewards</summary>");
|
|
724
|
+
lines.push("");
|
|
725
|
+
lines.push("| Task | Model | Reward |");
|
|
726
|
+
lines.push("| --- | --- | ---: |");
|
|
727
|
+
for (const row of taskRows) {
|
|
728
|
+
lines.push(
|
|
729
|
+
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
730
|
+
);
|
|
731
|
+
}
|
|
732
|
+
lines.push("");
|
|
733
|
+
lines.push("</details>");
|
|
734
|
+
}
|
|
735
|
+
return lines.join("\n");
|
|
736
|
+
}
|
|
737
|
+
async function validateWorld(dir, opts) {
|
|
738
|
+
const options = opts.options ?? {};
|
|
739
|
+
const progress = opts.onProgress ?? (() => {
|
|
740
|
+
});
|
|
741
|
+
const minScore = opts.minScore ?? null;
|
|
742
|
+
const provenance = detectProvenance();
|
|
743
|
+
const branch = opts.branch ?? provenance.branch;
|
|
744
|
+
if (!branch)
|
|
745
|
+
throw new Error(
|
|
746
|
+
"No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
|
|
747
|
+
);
|
|
748
|
+
progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
|
|
749
|
+
const pushed = await push(dir, {
|
|
750
|
+
slug: opts.slug,
|
|
751
|
+
branch,
|
|
752
|
+
message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
|
|
753
|
+
...provenance.actor?.name ? { author: provenance.actor.name } : {},
|
|
754
|
+
options
|
|
755
|
+
});
|
|
756
|
+
progress(`pushed version ${pushed.versionId} (${pushed.contentHash.slice(0, 12)})`);
|
|
757
|
+
const configs = await listRunConfigs(options);
|
|
758
|
+
const runConfig = configs.find((c) => c.id === opts.runConfig) ?? configs.find((c) => c.name === opts.runConfig);
|
|
759
|
+
if (!runConfig) {
|
|
760
|
+
const known = configs.map((c) => c.name).join(", ") || "(none)";
|
|
761
|
+
throw new Error(
|
|
762
|
+
`Run config "${opts.runConfig}" not found in this project. Have: ${known}`
|
|
763
|
+
);
|
|
764
|
+
}
|
|
765
|
+
progress(`dispatching run config "${runConfig.name}" (${runConfig.models.join(", ")})\u2026`);
|
|
766
|
+
const dispatched = await apiRequest(options, "POST", "/api/public/benchmark-containers/run-configs/dispatch", {
|
|
767
|
+
runConfigId: runConfig.id,
|
|
768
|
+
versionId: pushed.versionId,
|
|
769
|
+
...provenance
|
|
770
|
+
});
|
|
771
|
+
const host = (options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai").replace(/\/+$/, "");
|
|
772
|
+
let projectId = null;
|
|
773
|
+
try {
|
|
774
|
+
const projects = await apiRequest(
|
|
775
|
+
options,
|
|
776
|
+
"GET",
|
|
777
|
+
"/api/public/projects"
|
|
778
|
+
);
|
|
779
|
+
projectId = projects.data?.[0]?.id ?? null;
|
|
780
|
+
} catch {
|
|
781
|
+
projectId = null;
|
|
782
|
+
}
|
|
783
|
+
const fallbackUrl = (report) => {
|
|
784
|
+
if (report.url) return report.url;
|
|
785
|
+
const evaluationId = report.evaluations[0]?.id;
|
|
786
|
+
return projectId && evaluationId ? `${host}/project/${projectId}/environment/runs/${encodeURIComponent(evaluationId)}` : null;
|
|
787
|
+
};
|
|
788
|
+
const runs = [];
|
|
789
|
+
for (const run of dispatched.runs) {
|
|
790
|
+
if (!run.ok) {
|
|
791
|
+
runs.push({
|
|
792
|
+
model: run.model,
|
|
793
|
+
runId: run.runId,
|
|
794
|
+
report: null,
|
|
795
|
+
failure: `dispatch failed: ${run.error ?? "unknown error"}`
|
|
796
|
+
});
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
799
|
+
progress(`waiting on ${run.model} (run ${run.runId})\u2026`);
|
|
800
|
+
let report = null;
|
|
801
|
+
let failure = null;
|
|
802
|
+
try {
|
|
803
|
+
report = await waitForRun(run.runId, {
|
|
804
|
+
timeoutMinutes: opts.timeoutMinutes ?? 45,
|
|
805
|
+
options,
|
|
806
|
+
onTick: (r) => progress(
|
|
807
|
+
`\u2026${run.model}: ${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
|
|
808
|
+
)
|
|
809
|
+
});
|
|
810
|
+
report = { ...report, url: fallbackUrl(report) };
|
|
811
|
+
if (report.status.toUpperCase() !== "COMPLETED") {
|
|
812
|
+
failure = report.status;
|
|
813
|
+
} else if (minScore !== null) {
|
|
814
|
+
if (report.score === null) failure = "completed without a score";
|
|
815
|
+
else if (report.score < minScore)
|
|
816
|
+
failure = `score ${fmtScore(report.score)} < ${minScore}`;
|
|
817
|
+
}
|
|
818
|
+
} catch (error) {
|
|
819
|
+
failure = error instanceof Error ? error.message : String(error);
|
|
820
|
+
}
|
|
821
|
+
runs.push({ model: run.model, runId: run.runId, report, failure });
|
|
822
|
+
}
|
|
823
|
+
const base = {
|
|
824
|
+
ok: runs.length > 0 && runs.every((r) => r.failure === null),
|
|
825
|
+
slug: pushed.slug,
|
|
826
|
+
branch,
|
|
827
|
+
versionId: pushed.versionId,
|
|
828
|
+
contentHash: pushed.contentHash,
|
|
829
|
+
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
830
|
+
runs
|
|
831
|
+
};
|
|
832
|
+
return { ...base, markdown: buildMarkdown(base, minScore) };
|
|
833
|
+
}
|
|
834
|
+
var init_validate = __esm({
|
|
835
|
+
"src/worlds/validate.ts"() {
|
|
836
|
+
"use strict";
|
|
837
|
+
init_worlds();
|
|
838
|
+
init_hub();
|
|
839
|
+
}
|
|
840
|
+
});
|
|
841
|
+
|
|
842
|
+
// src/worlds/index.ts
|
|
843
|
+
var worlds_exports = {};
|
|
844
|
+
__export(worlds_exports, {
|
|
845
|
+
DEFAULT_BRANCH: () => DEFAULT_BRANCH,
|
|
846
|
+
WorldSession: () => WorldSession,
|
|
847
|
+
WorldSessionCallError: () => WorldSessionCallError,
|
|
848
|
+
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
849
|
+
WorldSet: () => WorldSet,
|
|
850
|
+
WorldTask: () => WorldTask,
|
|
851
|
+
apiRequest: () => apiRequest,
|
|
852
|
+
attachSession: () => attachSession,
|
|
853
|
+
buildSourceTarball: () => buildSourceTarball,
|
|
854
|
+
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
855
|
+
computeContentHash: () => computeContentHash,
|
|
856
|
+
createRef: () => createRef,
|
|
857
|
+
deleteEnvSecret: () => deleteEnvSecret,
|
|
858
|
+
detectProvenance: () => detectProvenance,
|
|
859
|
+
getRun: () => getRun,
|
|
860
|
+
isIncludedInContentHash: () => isIncludedInContentHash,
|
|
861
|
+
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
862
|
+
listRunConfigs: () => listRunConfigs,
|
|
863
|
+
localTreeState: () => localTreeState,
|
|
864
|
+
openSession: () => openSession,
|
|
865
|
+
openWorld: () => openWorld,
|
|
866
|
+
propose: () => propose,
|
|
867
|
+
pull: () => pull,
|
|
868
|
+
push: () => push,
|
|
869
|
+
readLocalMeta: () => readLocalMeta,
|
|
870
|
+
runSessions: () => runSessions,
|
|
871
|
+
schemaFromSignature: () => schemaFromSignature,
|
|
872
|
+
setEnvSecret: () => setEnvSecret,
|
|
873
|
+
snapshotTreeHashes: () => snapshotTreeHashes,
|
|
874
|
+
validateWorld: () => validateWorld,
|
|
875
|
+
waitForRun: () => waitForRun,
|
|
876
|
+
writeLocalMeta: () => writeLocalMeta
|
|
877
|
+
});
|
|
878
|
+
function resolveEnv(options) {
|
|
879
|
+
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
880
|
+
const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
|
|
881
|
+
const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
|
|
882
|
+
if (!publicKey || !secretKey) {
|
|
883
|
+
throw new Error(
|
|
884
|
+
"Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
|
|
885
|
+
);
|
|
886
|
+
}
|
|
887
|
+
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
888
|
+
}
|
|
889
|
+
async function apiRequest(options, method, path2, body) {
|
|
890
|
+
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
891
|
+
const res = await fetch(`${host}${path2}`, {
|
|
892
|
+
method,
|
|
893
|
+
headers: {
|
|
894
|
+
"content-type": "application/json",
|
|
895
|
+
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
896
|
+
},
|
|
897
|
+
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
898
|
+
});
|
|
899
|
+
const text = await res.text();
|
|
900
|
+
if (!res.ok) {
|
|
901
|
+
throw new Error(
|
|
902
|
+
`${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
|
|
903
|
+
);
|
|
904
|
+
}
|
|
905
|
+
return JSON.parse(text);
|
|
906
|
+
}
|
|
907
|
+
async function setEnvSecret(slug, key, value, options = {}) {
|
|
908
|
+
return apiRequest(
|
|
909
|
+
options,
|
|
910
|
+
"PUT",
|
|
911
|
+
"/api/public/benchmark-containers/secrets",
|
|
912
|
+
{
|
|
913
|
+
slug,
|
|
914
|
+
key,
|
|
915
|
+
value
|
|
916
|
+
}
|
|
917
|
+
);
|
|
918
|
+
}
|
|
919
|
+
async function listEnvSecretKeys(slug, options = {}) {
|
|
920
|
+
const res = await apiRequest(
|
|
921
|
+
options,
|
|
922
|
+
"GET",
|
|
923
|
+
`/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
|
|
924
|
+
);
|
|
925
|
+
return res.secrets;
|
|
926
|
+
}
|
|
927
|
+
async function deleteEnvSecret(slug, key, options = {}) {
|
|
928
|
+
await apiRequest(
|
|
929
|
+
options,
|
|
930
|
+
"DELETE",
|
|
931
|
+
"/api/public/benchmark-containers/secrets",
|
|
932
|
+
{
|
|
933
|
+
slug,
|
|
934
|
+
key
|
|
935
|
+
}
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
async function listRunConfigs(options = {}) {
|
|
939
|
+
const res = await apiRequest(
|
|
940
|
+
options,
|
|
941
|
+
"GET",
|
|
942
|
+
"/api/public/benchmark-containers/run-configs"
|
|
943
|
+
);
|
|
944
|
+
return res.runConfigs;
|
|
945
|
+
}
|
|
946
|
+
async function getRun(runId, options = {}) {
|
|
947
|
+
return apiRequest(
|
|
948
|
+
options,
|
|
949
|
+
"GET",
|
|
950
|
+
`/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
|
|
951
|
+
);
|
|
952
|
+
}
|
|
953
|
+
async function waitForRun(runId, opts = {}) {
|
|
954
|
+
const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
|
|
955
|
+
for (; ; ) {
|
|
956
|
+
const run = await getRun(runId, opts.options ?? {});
|
|
957
|
+
if (run.terminal) return run;
|
|
958
|
+
if (Date.now() > deadline)
|
|
959
|
+
throw new Error(
|
|
960
|
+
`Run ${runId} still ${run.status} after the wait timeout.`
|
|
961
|
+
);
|
|
962
|
+
opts.onTick?.(run);
|
|
963
|
+
await new Promise(
|
|
964
|
+
(resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
|
|
965
|
+
);
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
function detectProvenance() {
|
|
969
|
+
const env = globalThis.process?.env ?? {};
|
|
970
|
+
const out = {};
|
|
971
|
+
const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
|
|
972
|
+
if (ref) out.branch = ref;
|
|
973
|
+
if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
|
|
974
|
+
const server = env.GITHUB_SERVER_URL ?? "https://github.com";
|
|
975
|
+
if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
|
|
976
|
+
if (env.GITHUB_ACTOR) {
|
|
977
|
+
out.actor = {
|
|
978
|
+
kind: "ci",
|
|
979
|
+
id: env.GITHUB_ACTOR,
|
|
980
|
+
name: env.GITHUB_ACTOR,
|
|
981
|
+
// GitHub serves every account's avatar at /<login>.png.
|
|
982
|
+
imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
|
|
983
|
+
};
|
|
984
|
+
}
|
|
985
|
+
if (!out.branch || !out.commit) {
|
|
986
|
+
try {
|
|
987
|
+
const { execFileSync } = require("child_process");
|
|
988
|
+
const git = (args) => execFileSync("git", args, {
|
|
989
|
+
encoding: "utf-8",
|
|
990
|
+
stdio: ["ignore", "pipe", "ignore"]
|
|
991
|
+
}).toString().trim();
|
|
992
|
+
out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
|
|
993
|
+
out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
|
|
994
|
+
if (!out.repoUrl) {
|
|
995
|
+
const remote = git(["config", "--get", "remote.origin.url"]);
|
|
996
|
+
if (remote) out.repoUrl = remote.replace(/\.git$/, "");
|
|
997
|
+
}
|
|
998
|
+
if (!out.actor) {
|
|
999
|
+
const name = git(["config", "--get", "user.name"]);
|
|
1000
|
+
if (name) out.actor = { kind: "user", name };
|
|
1001
|
+
}
|
|
1002
|
+
} catch {
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
return out;
|
|
1006
|
+
}
|
|
1007
|
+
async function openWorld(slugRef, options = {}) {
|
|
1008
|
+
const at = slugRef.indexOf("@");
|
|
1009
|
+
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
1010
|
+
const ref = at === -1 ? null : slugRef.slice(at + 1);
|
|
1011
|
+
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
1012
|
+
slug,
|
|
1013
|
+
...ref ? { ref } : {}
|
|
1014
|
+
});
|
|
1015
|
+
const world = new WorldSet(
|
|
1016
|
+
slugRef,
|
|
1017
|
+
slug,
|
|
1018
|
+
ref,
|
|
1019
|
+
{
|
|
1020
|
+
versionId: version.versionId,
|
|
1021
|
+
semanticVersion: version.semanticVersion ?? null,
|
|
1022
|
+
contentHash: version.contentHash,
|
|
1023
|
+
state: version.state
|
|
1024
|
+
},
|
|
1025
|
+
[],
|
|
1026
|
+
options
|
|
1027
|
+
);
|
|
1028
|
+
for (const name of version.taskNames ?? []) {
|
|
1029
|
+
const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
|
|
1030
|
+
world.tasks.push(new WorldTask(world, name, variant));
|
|
1031
|
+
}
|
|
1032
|
+
return world;
|
|
1033
|
+
}
|
|
1034
|
+
var WorldTask, WorldSet;
|
|
1035
|
+
var init_worlds = __esm({
|
|
1036
|
+
"src/worlds/index.ts"() {
|
|
1037
|
+
"use strict";
|
|
1038
|
+
init_session();
|
|
1039
|
+
init_hub();
|
|
1040
|
+
init_validate();
|
|
1041
|
+
WorldTask = class {
|
|
1042
|
+
constructor(world, name, variant) {
|
|
1043
|
+
this.world = world;
|
|
1044
|
+
this.name = name;
|
|
1045
|
+
this.variant = variant;
|
|
1046
|
+
}
|
|
1047
|
+
};
|
|
1048
|
+
WorldSet = class {
|
|
1049
|
+
constructor(slugRef, slug, ref, version, tasks, options) {
|
|
1050
|
+
this.slugRef = slugRef;
|
|
1051
|
+
this.slug = slug;
|
|
1052
|
+
this.ref = ref;
|
|
1053
|
+
this.version = version;
|
|
1054
|
+
this.tasks = tasks;
|
|
1055
|
+
this.options = options;
|
|
1056
|
+
}
|
|
1057
|
+
task(variant) {
|
|
1058
|
+
const hit = this.tasks.find(
|
|
1059
|
+
(t) => t.variant === variant || t.name.includes(variant)
|
|
1060
|
+
);
|
|
1061
|
+
if (!hit) {
|
|
1062
|
+
const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
|
|
1063
|
+
throw new Error(`no task matches ${variant}; have: ${known}`);
|
|
1064
|
+
}
|
|
1065
|
+
return hit;
|
|
1066
|
+
}
|
|
1067
|
+
/**
|
|
1068
|
+
* Open a LIVE session against one of this world's tasks: the platform boots
|
|
1069
|
+
* (or reuses) a warm container, and your agent calls its tools through the
|
|
1070
|
+
* platform. See worlds/session.ts.
|
|
1071
|
+
*/
|
|
1072
|
+
async open(task, opts = {}) {
|
|
1073
|
+
const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
|
|
1074
|
+
return openSession2(this.slug, task, {
|
|
1075
|
+
versionId: this.version.versionId,
|
|
1076
|
+
reuseWarm: opts.reuseWarm,
|
|
1077
|
+
options: this.options
|
|
1078
|
+
});
|
|
1079
|
+
}
|
|
1080
|
+
/**
|
|
1081
|
+
* Start a HOSTED run of a run config against this world — the platform
|
|
1082
|
+
* executes the (Python) world in parallel containers per the config's
|
|
1083
|
+
* `parallelism` and files one graded run per model. Returns those runs.
|
|
1084
|
+
*/
|
|
1085
|
+
async dispatch(runConfigId, opts = {}) {
|
|
1086
|
+
const provenance = opts.provenance ?? detectProvenance();
|
|
1087
|
+
return apiRequest(
|
|
1088
|
+
this.options,
|
|
1089
|
+
"POST",
|
|
1090
|
+
"/api/public/benchmark-containers/run-configs/dispatch",
|
|
1091
|
+
{
|
|
1092
|
+
runConfigId,
|
|
1093
|
+
...opts.containerId ? { containerId: opts.containerId } : {},
|
|
1094
|
+
versionId: opts.versionId ?? this.version.versionId,
|
|
1095
|
+
...provenance
|
|
1096
|
+
}
|
|
1097
|
+
);
|
|
1098
|
+
}
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1101
|
+
});
|
|
1102
|
+
|
|
1103
|
+
// src/cli.ts
|
|
1104
|
+
var fs2 = __toESM(require("fs"));
|
|
1105
|
+
init_worlds();
|
|
1106
|
+
init_hub();
|
|
1107
|
+
init_validate();
|
|
1108
|
+
function parseArgv(argv) {
|
|
1109
|
+
const positional = [];
|
|
1110
|
+
const flags = {};
|
|
936
1111
|
for (let i = 0; i < argv.length; i++) {
|
|
937
1112
|
const arg = argv[i];
|
|
938
1113
|
if (arg.startsWith("--")) {
|
|
@@ -977,6 +1152,9 @@ var HELP = `gateway bench <command> \u2014 worlds from TypeScript
|
|
|
977
1152
|
|
|
978
1153
|
pull <slug[@ref]> [--dir D] materialize a working dir
|
|
979
1154
|
push [dir] [--slug S] [--branch B] [--force] [--message M]
|
|
1155
|
+
validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X]
|
|
1156
|
+
[--summary-file F] [--timeout-min N]
|
|
1157
|
+
push this checkout, run it, gate \u2014 the PR-validation entry
|
|
980
1158
|
checkout <slug@ref> --dir D re-aim an existing working dir
|
|
981
1159
|
status [dir] clean/dirty vs base + hash check
|
|
982
1160
|
branch <slug> <name> [--version-id V] create a branch
|
|
@@ -1237,6 +1415,32 @@ Run hosted instead: gateway bench dispatch <runConfigId> \u2014 same worlds, gra
|
|
|
1237
1415
|
);
|
|
1238
1416
|
return 0;
|
|
1239
1417
|
}
|
|
1418
|
+
case "validate": {
|
|
1419
|
+
const dir = positional[0] ?? ".";
|
|
1420
|
+
const runConfig = str(flags, "run-config") ?? fail(
|
|
1421
|
+
"usage: gateway bench validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X] [--summary-file F] [--timeout-min N]"
|
|
1422
|
+
);
|
|
1423
|
+
const minScoreRaw = str(flags, "min-score");
|
|
1424
|
+
const report = await validateWorld(dir, {
|
|
1425
|
+
slug: str(flags, "slug"),
|
|
1426
|
+
runConfig,
|
|
1427
|
+
branch: str(flags, "branch"),
|
|
1428
|
+
minScore: minScoreRaw === void 0 ? null : Number(minScoreRaw),
|
|
1429
|
+
timeoutMinutes: Number(str(flags, "timeout-min") ?? 45),
|
|
1430
|
+
onProgress: (line) => console.error(line)
|
|
1431
|
+
});
|
|
1432
|
+
for (const run of report.runs) {
|
|
1433
|
+
console.log(
|
|
1434
|
+
`${run.model}: ${run.failure ? `FAIL \u2014 ${run.failure}` : "pass"}${run.report?.score != null ? ` score=${run.report.score.toFixed(3)}` : ""}${run.report?.url ? ` ${run.report.url}` : ""}`
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
const summaryFile = str(flags, "summary-file");
|
|
1438
|
+
if (summaryFile) fs2.writeFileSync(summaryFile, report.markdown + "\n");
|
|
1439
|
+
const stepSummary = process.env.GITHUB_STEP_SUMMARY;
|
|
1440
|
+
if (stepSummary) fs2.appendFileSync(stepSummary, report.markdown + "\n");
|
|
1441
|
+
console.log(report.ok ? "validation passed" : "validation FAILED");
|
|
1442
|
+
return report.ok ? 0 : 1;
|
|
1443
|
+
}
|
|
1240
1444
|
case "status": {
|
|
1241
1445
|
const dir = positional[0] ?? ".";
|
|
1242
1446
|
const state = localTreeState(dir);
|