@withgateway/sdk 0.1.14 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -325,273 +325,7 @@ var init_session = __esm({
325
325
  }
326
326
  });
327
327
 
328
- // src/worlds/index.ts
329
- var worlds_exports = {};
330
- __export(worlds_exports, {
331
- WorldSession: () => WorldSession,
332
- WorldSessionCallError: () => WorldSessionCallError,
333
- WorldSessionTimeout: () => WorldSessionTimeout,
334
- WorldSet: () => WorldSet,
335
- WorldTask: () => WorldTask,
336
- apiRequest: () => apiRequest,
337
- attachSession: () => attachSession,
338
- deleteEnvSecret: () => deleteEnvSecret,
339
- detectProvenance: () => detectProvenance,
340
- getRun: () => getRun,
341
- listEnvSecretKeys: () => listEnvSecretKeys,
342
- listRunConfigs: () => listRunConfigs,
343
- openSession: () => openSession,
344
- openWorld: () => openWorld,
345
- runSessions: () => runSessions,
346
- schemaFromSignature: () => schemaFromSignature,
347
- setEnvSecret: () => setEnvSecret,
348
- waitForRun: () => waitForRun
349
- });
350
- function resolveEnv(options) {
351
- const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
352
- const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
353
- const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
354
- if (!publicKey || !secretKey) {
355
- throw new Error(
356
- "Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
357
- );
358
- }
359
- return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
360
- }
361
- async function apiRequest(options, method, path2, body) {
362
- const { host, publicKey, secretKey } = resolveEnv(options);
363
- const res = await fetch(`${host}${path2}`, {
364
- method,
365
- headers: {
366
- "content-type": "application/json",
367
- authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
368
- },
369
- ...body === void 0 ? {} : { body: JSON.stringify(body) }
370
- });
371
- const text = await res.text();
372
- if (!res.ok) {
373
- throw new Error(
374
- `${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
375
- );
376
- }
377
- return JSON.parse(text);
378
- }
379
- async function setEnvSecret(slug, key, value, options = {}) {
380
- return apiRequest(
381
- options,
382
- "PUT",
383
- "/api/public/benchmark-containers/secrets",
384
- {
385
- slug,
386
- key,
387
- value
388
- }
389
- );
390
- }
391
- async function listEnvSecretKeys(slug, options = {}) {
392
- const res = await apiRequest(
393
- options,
394
- "GET",
395
- `/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
396
- );
397
- return res.secrets;
398
- }
399
- async function deleteEnvSecret(slug, key, options = {}) {
400
- await apiRequest(
401
- options,
402
- "DELETE",
403
- "/api/public/benchmark-containers/secrets",
404
- {
405
- slug,
406
- key
407
- }
408
- );
409
- }
410
- async function listRunConfigs(options = {}) {
411
- const res = await apiRequest(
412
- options,
413
- "GET",
414
- "/api/public/benchmark-containers/run-configs"
415
- );
416
- return res.runConfigs;
417
- }
418
- async function getRun(runId, options = {}) {
419
- return apiRequest(
420
- options,
421
- "GET",
422
- `/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
423
- );
424
- }
425
- async function waitForRun(runId, opts = {}) {
426
- const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
427
- for (; ; ) {
428
- const run = await getRun(runId, opts.options ?? {});
429
- if (run.terminal) return run;
430
- if (Date.now() > deadline)
431
- throw new Error(
432
- `Run ${runId} still ${run.status} after the wait timeout.`
433
- );
434
- opts.onTick?.(run);
435
- await new Promise(
436
- (resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
437
- );
438
- }
439
- }
440
- function detectProvenance() {
441
- const env = globalThis.process?.env ?? {};
442
- const out = {};
443
- const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
444
- if (ref) out.branch = ref;
445
- if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
446
- const server = env.GITHUB_SERVER_URL ?? "https://github.com";
447
- if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
448
- if (env.GITHUB_ACTOR) {
449
- out.actor = {
450
- kind: "ci",
451
- id: env.GITHUB_ACTOR,
452
- name: env.GITHUB_ACTOR,
453
- // GitHub serves every account's avatar at /<login>.png.
454
- imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
455
- };
456
- }
457
- if (!out.branch || !out.commit) {
458
- try {
459
- const { execFileSync } = require("child_process");
460
- const git = (args) => execFileSync("git", args, {
461
- encoding: "utf-8",
462
- stdio: ["ignore", "pipe", "ignore"]
463
- }).toString().trim();
464
- out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
465
- out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
466
- if (!out.repoUrl) {
467
- const remote = git(["config", "--get", "remote.origin.url"]);
468
- if (remote) out.repoUrl = remote.replace(/\.git$/, "");
469
- }
470
- if (!out.actor) {
471
- const name = git(["config", "--get", "user.name"]);
472
- if (name) out.actor = { kind: "user", name };
473
- }
474
- } catch {
475
- }
476
- }
477
- return out;
478
- }
479
- async function openWorld(slugRef, options = {}) {
480
- const at = slugRef.indexOf("@");
481
- const slug = at === -1 ? slugRef : slugRef.slice(0, at);
482
- const ref = at === -1 ? null : slugRef.slice(at + 1);
483
- const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
484
- slug,
485
- ...ref ? { ref } : {}
486
- });
487
- const world = new WorldSet(
488
- slugRef,
489
- slug,
490
- ref,
491
- {
492
- versionId: version.versionId,
493
- semanticVersion: version.semanticVersion ?? null,
494
- contentHash: version.contentHash,
495
- state: version.state
496
- },
497
- [],
498
- options
499
- );
500
- for (const name of version.taskNames ?? []) {
501
- const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
502
- world.tasks.push(new WorldTask(world, name, variant));
503
- }
504
- return world;
505
- }
506
- var WorldTask, WorldSet;
507
- var init_worlds = __esm({
508
- "src/worlds/index.ts"() {
509
- "use strict";
510
- init_session();
511
- WorldTask = class {
512
- constructor(world, name, variant) {
513
- this.world = world;
514
- this.name = name;
515
- this.variant = variant;
516
- }
517
- };
518
- WorldSet = class {
519
- constructor(slugRef, slug, ref, version, tasks, options) {
520
- this.slugRef = slugRef;
521
- this.slug = slug;
522
- this.ref = ref;
523
- this.version = version;
524
- this.tasks = tasks;
525
- this.options = options;
526
- }
527
- task(variant) {
528
- const hit = this.tasks.find(
529
- (t) => t.variant === variant || t.name.includes(variant)
530
- );
531
- if (!hit) {
532
- const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
533
- throw new Error(`no task matches ${variant}; have: ${known}`);
534
- }
535
- return hit;
536
- }
537
- /**
538
- * Open a LIVE session against one of this world's tasks: the platform boots
539
- * (or reuses) a warm container, and your agent calls its tools through the
540
- * platform. See worlds/session.ts.
541
- */
542
- async open(task, opts = {}) {
543
- const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
544
- return openSession2(this.slug, task, {
545
- versionId: this.version.versionId,
546
- reuseWarm: opts.reuseWarm,
547
- options: this.options
548
- });
549
- }
550
- /**
551
- * Start a HOSTED run of a run config against this world — the platform
552
- * executes the (Python) world in parallel containers per the config's
553
- * `parallelism` and files one graded run per model. Returns those runs.
554
- */
555
- async dispatch(runConfigId, opts = {}) {
556
- const provenance = opts.provenance ?? detectProvenance();
557
- return apiRequest(
558
- this.options,
559
- "POST",
560
- "/api/public/benchmark-containers/run-configs/dispatch",
561
- {
562
- runConfigId,
563
- ...opts.containerId ? { containerId: opts.containerId } : {},
564
- versionId: opts.versionId ?? this.version.versionId,
565
- ...provenance
566
- }
567
- );
568
- }
569
- };
570
- }
571
- });
572
-
573
- // src/cli.ts
574
- init_worlds();
575
-
576
328
  // src/worlds/hub.ts
577
- var import_node_crypto = require("crypto");
578
- var import_node_zlib = require("zlib");
579
- var fs = __toESM(require("fs"));
580
- var path = __toESM(require("path"));
581
- init_worlds();
582
- var DEFAULT_BRANCH = "main";
583
- var META_RELPATH = path.join(".gateway", ".env-metadata.json");
584
- var SKIP_DIRS = /* @__PURE__ */ new Set([
585
- ".git",
586
- "__pycache__",
587
- "node_modules",
588
- ".venv",
589
- ".mypy_cache",
590
- "dist",
591
- "build",
592
- "outputs"
593
- ]);
594
- var ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
595
329
  function isExcludedDirComponent(component) {
596
330
  return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
597
331
  }
@@ -721,7 +455,6 @@ function localTreeState(dir) {
721
455
  for (const p of Object.keys(base)) if (!(p in now)) dirty.add(p);
722
456
  return { meta, dirty: [...dirty].sort(), clean: dirty.size === 0 };
723
457
  }
724
- var sha256b64 = (data) => (0, import_node_crypto.createHash)("sha256").update(data).digest("base64");
725
458
  function readPyproject(dir) {
726
459
  try {
727
460
  const text = fs.readFileSync(path.join(dir, "pyproject.toml"), "utf-8");
@@ -928,11 +661,435 @@ async function createRef(slug, name, kind, versionId, options = {}) {
928
661
  ...versionId ? { versionId } : {}
929
662
  });
930
663
  }
664
+ var import_node_crypto, import_node_zlib, fs, path, DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, sha256b64;
665
+ var init_hub = __esm({
666
+ "src/worlds/hub.ts"() {
667
+ "use strict";
668
+ import_node_crypto = require("crypto");
669
+ import_node_zlib = require("zlib");
670
+ fs = __toESM(require("fs"));
671
+ path = __toESM(require("path"));
672
+ init_worlds();
673
+ DEFAULT_BRANCH = "main";
674
+ META_RELPATH = path.join(".gateway", ".env-metadata.json");
675
+ SKIP_DIRS = /* @__PURE__ */ new Set([
676
+ ".git",
677
+ "__pycache__",
678
+ "node_modules",
679
+ ".venv",
680
+ ".mypy_cache",
681
+ "dist",
682
+ "build",
683
+ "outputs"
684
+ ]);
685
+ ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
686
+ sha256b64 = (data) => (0, import_node_crypto.createHash)("sha256").update(data).digest("base64");
687
+ }
688
+ });
931
689
 
932
- // src/cli.ts
933
- function parseArgv(argv) {
934
- const positional = [];
935
- const flags = {};
690
+ // src/worlds/validate.ts
691
+ function fmtScore(score) {
692
+ return score === null ? "no score" : score.toFixed(3);
693
+ }
694
+ function buildMarkdown(report, minScore) {
695
+ const lines = [];
696
+ lines.push(
697
+ `### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
698
+ );
699
+ lines.push("");
700
+ lines.push(
701
+ `Ran the pull request's own world source (version \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
702
+ );
703
+ lines.push("");
704
+ lines.push("| Model | Status | Score | Run |");
705
+ lines.push("| --- | --- | ---: | --- |");
706
+ for (const run of report.runs) {
707
+ const r = run.report;
708
+ const link = r?.url ? `[open](${r.url})` : "\u2014";
709
+ const status = run.failure ? `\u274C ${run.failure}` : `\u2705 ${r?.status ?? "COMPLETED"}`;
710
+ lines.push(
711
+ `| \`${run.model}\` | ${status} | ${fmtScore(r?.score ?? null)} | ${link} |`
712
+ );
713
+ }
714
+ const taskRows = report.runs.flatMap(
715
+ (run) => (run.report?.samples ?? []).map((sample) => ({
716
+ model: run.model,
717
+ task: sample.task ?? "?",
718
+ reward: sample.reward
719
+ }))
720
+ );
721
+ if (taskRows.length > 0 && taskRows.length <= 40) {
722
+ lines.push("");
723
+ lines.push("<details><summary>Per-task rewards</summary>");
724
+ lines.push("");
725
+ lines.push("| Task | Model | Reward |");
726
+ lines.push("| --- | --- | ---: |");
727
+ for (const row of taskRows) {
728
+ lines.push(
729
+ `| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
730
+ );
731
+ }
732
+ lines.push("");
733
+ lines.push("</details>");
734
+ }
735
+ return lines.join("\n");
736
+ }
737
+ async function validateWorld(dir, opts) {
738
+ const options = opts.options ?? {};
739
+ const progress = opts.onProgress ?? (() => {
740
+ });
741
+ const minScore = opts.minScore ?? null;
742
+ const provenance = detectProvenance();
743
+ const branch = opts.branch ?? provenance.branch;
744
+ if (!branch)
745
+ throw new Error(
746
+ "No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
747
+ );
748
+ progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
749
+ const pushed = await push(dir, {
750
+ slug: opts.slug,
751
+ branch,
752
+ message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
753
+ ...provenance.actor?.name ? { author: provenance.actor.name } : {},
754
+ options
755
+ });
756
+ progress(`pushed version ${pushed.versionId} (${pushed.contentHash.slice(0, 12)})`);
757
+ const configs = await listRunConfigs(options);
758
+ const runConfig = configs.find((c) => c.id === opts.runConfig) ?? configs.find((c) => c.name === opts.runConfig);
759
+ if (!runConfig) {
760
+ const known = configs.map((c) => c.name).join(", ") || "(none)";
761
+ throw new Error(
762
+ `Run config "${opts.runConfig}" not found in this project. Have: ${known}`
763
+ );
764
+ }
765
+ progress(`dispatching run config "${runConfig.name}" (${runConfig.models.join(", ")})\u2026`);
766
+ const dispatched = await apiRequest(options, "POST", "/api/public/benchmark-containers/run-configs/dispatch", {
767
+ runConfigId: runConfig.id,
768
+ versionId: pushed.versionId,
769
+ ...provenance
770
+ });
771
+ const runs = [];
772
+ for (const run of dispatched.runs) {
773
+ if (!run.ok) {
774
+ runs.push({
775
+ model: run.model,
776
+ runId: run.runId,
777
+ report: null,
778
+ failure: `dispatch failed: ${run.error ?? "unknown error"}`
779
+ });
780
+ continue;
781
+ }
782
+ progress(`waiting on ${run.model} (run ${run.runId})\u2026`);
783
+ let report = null;
784
+ let failure = null;
785
+ try {
786
+ report = await waitForRun(run.runId, {
787
+ timeoutMinutes: opts.timeoutMinutes ?? 45,
788
+ options,
789
+ onTick: (r) => progress(
790
+ `\u2026${run.model}: ${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
791
+ )
792
+ });
793
+ if (report.status.toUpperCase() !== "COMPLETED") {
794
+ failure = report.status;
795
+ } else if (minScore !== null) {
796
+ if (report.score === null) failure = "completed without a score";
797
+ else if (report.score < minScore)
798
+ failure = `score ${fmtScore(report.score)} < ${minScore}`;
799
+ }
800
+ } catch (error) {
801
+ failure = error instanceof Error ? error.message : String(error);
802
+ }
803
+ runs.push({ model: run.model, runId: run.runId, report, failure });
804
+ }
805
+ const base = {
806
+ ok: runs.length > 0 && runs.every((r) => r.failure === null),
807
+ slug: pushed.slug,
808
+ branch,
809
+ versionId: pushed.versionId,
810
+ contentHash: pushed.contentHash,
811
+ runConfig: { id: runConfig.id, name: runConfig.name },
812
+ runs
813
+ };
814
+ return { ...base, markdown: buildMarkdown(base, minScore) };
815
+ }
816
+ var init_validate = __esm({
817
+ "src/worlds/validate.ts"() {
818
+ "use strict";
819
+ init_worlds();
820
+ init_hub();
821
+ }
822
+ });
823
+
824
+ // src/worlds/index.ts
825
+ var worlds_exports = {};
826
+ __export(worlds_exports, {
827
+ DEFAULT_BRANCH: () => DEFAULT_BRANCH,
828
+ WorldSession: () => WorldSession,
829
+ WorldSessionCallError: () => WorldSessionCallError,
830
+ WorldSessionTimeout: () => WorldSessionTimeout,
831
+ WorldSet: () => WorldSet,
832
+ WorldTask: () => WorldTask,
833
+ apiRequest: () => apiRequest,
834
+ attachSession: () => attachSession,
835
+ buildSourceTarball: () => buildSourceTarball,
836
+ collectArchiveRelPaths: () => collectArchiveRelPaths,
837
+ computeContentHash: () => computeContentHash,
838
+ createRef: () => createRef,
839
+ deleteEnvSecret: () => deleteEnvSecret,
840
+ detectProvenance: () => detectProvenance,
841
+ getRun: () => getRun,
842
+ isIncludedInContentHash: () => isIncludedInContentHash,
843
+ listEnvSecretKeys: () => listEnvSecretKeys,
844
+ listRunConfigs: () => listRunConfigs,
845
+ localTreeState: () => localTreeState,
846
+ openSession: () => openSession,
847
+ openWorld: () => openWorld,
848
+ propose: () => propose,
849
+ pull: () => pull,
850
+ push: () => push,
851
+ readLocalMeta: () => readLocalMeta,
852
+ runSessions: () => runSessions,
853
+ schemaFromSignature: () => schemaFromSignature,
854
+ setEnvSecret: () => setEnvSecret,
855
+ snapshotTreeHashes: () => snapshotTreeHashes,
856
+ validateWorld: () => validateWorld,
857
+ waitForRun: () => waitForRun,
858
+ writeLocalMeta: () => writeLocalMeta
859
+ });
860
+ function resolveEnv(options) {
861
+ const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
862
+ const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
863
+ const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
864
+ if (!publicKey || !secretKey) {
865
+ throw new Error(
866
+ "Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
867
+ );
868
+ }
869
+ return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
870
+ }
871
+ async function apiRequest(options, method, path2, body) {
872
+ const { host, publicKey, secretKey } = resolveEnv(options);
873
+ const res = await fetch(`${host}${path2}`, {
874
+ method,
875
+ headers: {
876
+ "content-type": "application/json",
877
+ authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
878
+ },
879
+ ...body === void 0 ? {} : { body: JSON.stringify(body) }
880
+ });
881
+ const text = await res.text();
882
+ if (!res.ok) {
883
+ throw new Error(
884
+ `${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
885
+ );
886
+ }
887
+ return JSON.parse(text);
888
+ }
889
+ async function setEnvSecret(slug, key, value, options = {}) {
890
+ return apiRequest(
891
+ options,
892
+ "PUT",
893
+ "/api/public/benchmark-containers/secrets",
894
+ {
895
+ slug,
896
+ key,
897
+ value
898
+ }
899
+ );
900
+ }
901
+ async function listEnvSecretKeys(slug, options = {}) {
902
+ const res = await apiRequest(
903
+ options,
904
+ "GET",
905
+ `/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
906
+ );
907
+ return res.secrets;
908
+ }
909
+ async function deleteEnvSecret(slug, key, options = {}) {
910
+ await apiRequest(
911
+ options,
912
+ "DELETE",
913
+ "/api/public/benchmark-containers/secrets",
914
+ {
915
+ slug,
916
+ key
917
+ }
918
+ );
919
+ }
920
+ async function listRunConfigs(options = {}) {
921
+ const res = await apiRequest(
922
+ options,
923
+ "GET",
924
+ "/api/public/benchmark-containers/run-configs"
925
+ );
926
+ return res.runConfigs;
927
+ }
928
+ async function getRun(runId, options = {}) {
929
+ return apiRequest(
930
+ options,
931
+ "GET",
932
+ `/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
933
+ );
934
+ }
935
+ async function waitForRun(runId, opts = {}) {
936
+ const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
937
+ for (; ; ) {
938
+ const run = await getRun(runId, opts.options ?? {});
939
+ if (run.terminal) return run;
940
+ if (Date.now() > deadline)
941
+ throw new Error(
942
+ `Run ${runId} still ${run.status} after the wait timeout.`
943
+ );
944
+ opts.onTick?.(run);
945
+ await new Promise(
946
+ (resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
947
+ );
948
+ }
949
+ }
950
+ function detectProvenance() {
951
+ const env = globalThis.process?.env ?? {};
952
+ const out = {};
953
+ const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
954
+ if (ref) out.branch = ref;
955
+ if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
956
+ const server = env.GITHUB_SERVER_URL ?? "https://github.com";
957
+ if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
958
+ if (env.GITHUB_ACTOR) {
959
+ out.actor = {
960
+ kind: "ci",
961
+ id: env.GITHUB_ACTOR,
962
+ name: env.GITHUB_ACTOR,
963
+ // GitHub serves every account's avatar at /<login>.png.
964
+ imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
965
+ };
966
+ }
967
+ if (!out.branch || !out.commit) {
968
+ try {
969
+ const { execFileSync } = require("child_process");
970
+ const git = (args) => execFileSync("git", args, {
971
+ encoding: "utf-8",
972
+ stdio: ["ignore", "pipe", "ignore"]
973
+ }).toString().trim();
974
+ out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
975
+ out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
976
+ if (!out.repoUrl) {
977
+ const remote = git(["config", "--get", "remote.origin.url"]);
978
+ if (remote) out.repoUrl = remote.replace(/\.git$/, "");
979
+ }
980
+ if (!out.actor) {
981
+ const name = git(["config", "--get", "user.name"]);
982
+ if (name) out.actor = { kind: "user", name };
983
+ }
984
+ } catch {
985
+ }
986
+ }
987
+ return out;
988
+ }
989
+ async function openWorld(slugRef, options = {}) {
990
+ const at = slugRef.indexOf("@");
991
+ const slug = at === -1 ? slugRef : slugRef.slice(0, at);
992
+ const ref = at === -1 ? null : slugRef.slice(at + 1);
993
+ const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
994
+ slug,
995
+ ...ref ? { ref } : {}
996
+ });
997
+ const world = new WorldSet(
998
+ slugRef,
999
+ slug,
1000
+ ref,
1001
+ {
1002
+ versionId: version.versionId,
1003
+ semanticVersion: version.semanticVersion ?? null,
1004
+ contentHash: version.contentHash,
1005
+ state: version.state
1006
+ },
1007
+ [],
1008
+ options
1009
+ );
1010
+ for (const name of version.taskNames ?? []) {
1011
+ const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
1012
+ world.tasks.push(new WorldTask(world, name, variant));
1013
+ }
1014
+ return world;
1015
+ }
1016
+ var WorldTask, WorldSet;
1017
+ var init_worlds = __esm({
1018
+ "src/worlds/index.ts"() {
1019
+ "use strict";
1020
+ init_session();
1021
+ init_hub();
1022
+ init_validate();
1023
+ WorldTask = class {
1024
+ constructor(world, name, variant) {
1025
+ this.world = world;
1026
+ this.name = name;
1027
+ this.variant = variant;
1028
+ }
1029
+ };
1030
+ WorldSet = class {
1031
+ constructor(slugRef, slug, ref, version, tasks, options) {
1032
+ this.slugRef = slugRef;
1033
+ this.slug = slug;
1034
+ this.ref = ref;
1035
+ this.version = version;
1036
+ this.tasks = tasks;
1037
+ this.options = options;
1038
+ }
1039
+ task(variant) {
1040
+ const hit = this.tasks.find(
1041
+ (t) => t.variant === variant || t.name.includes(variant)
1042
+ );
1043
+ if (!hit) {
1044
+ const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
1045
+ throw new Error(`no task matches ${variant}; have: ${known}`);
1046
+ }
1047
+ return hit;
1048
+ }
1049
+ /**
1050
+ * Open a LIVE session against one of this world's tasks: the platform boots
1051
+ * (or reuses) a warm container, and your agent calls its tools through the
1052
+ * platform. See worlds/session.ts.
1053
+ */
1054
+ async open(task, opts = {}) {
1055
+ const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
1056
+ return openSession2(this.slug, task, {
1057
+ versionId: this.version.versionId,
1058
+ reuseWarm: opts.reuseWarm,
1059
+ options: this.options
1060
+ });
1061
+ }
1062
+ /**
1063
+ * Start a HOSTED run of a run config against this world — the platform
1064
+ * executes the (Python) world in parallel containers per the config's
1065
+ * `parallelism` and files one graded run per model. Returns those runs.
1066
+ */
1067
+ async dispatch(runConfigId, opts = {}) {
1068
+ const provenance = opts.provenance ?? detectProvenance();
1069
+ return apiRequest(
1070
+ this.options,
1071
+ "POST",
1072
+ "/api/public/benchmark-containers/run-configs/dispatch",
1073
+ {
1074
+ runConfigId,
1075
+ ...opts.containerId ? { containerId: opts.containerId } : {},
1076
+ versionId: opts.versionId ?? this.version.versionId,
1077
+ ...provenance
1078
+ }
1079
+ );
1080
+ }
1081
+ };
1082
+ }
1083
+ });
1084
+
1085
+ // src/cli.ts
1086
+ var fs2 = __toESM(require("fs"));
1087
+ init_worlds();
1088
+ init_hub();
1089
+ init_validate();
1090
+ function parseArgv(argv) {
1091
+ const positional = [];
1092
+ const flags = {};
936
1093
  for (let i = 0; i < argv.length; i++) {
937
1094
  const arg = argv[i];
938
1095
  if (arg.startsWith("--")) {
@@ -977,6 +1134,9 @@ var HELP = `gateway bench <command> \u2014 worlds from TypeScript
977
1134
 
978
1135
  pull <slug[@ref]> [--dir D] materialize a working dir
979
1136
  push [dir] [--slug S] [--branch B] [--force] [--message M]
1137
+ validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X]
1138
+ [--summary-file F] [--timeout-min N]
1139
+ push this checkout, run it, gate \u2014 the PR-validation entry
980
1140
  checkout <slug@ref> --dir D re-aim an existing working dir
981
1141
  status [dir] clean/dirty vs base + hash check
982
1142
  branch <slug> <name> [--version-id V] create a branch
@@ -1237,6 +1397,32 @@ Run hosted instead: gateway bench dispatch <runConfigId> \u2014 same worlds, gra
1237
1397
  );
1238
1398
  return 0;
1239
1399
  }
1400
+ case "validate": {
1401
+ const dir = positional[0] ?? ".";
1402
+ const runConfig = str(flags, "run-config") ?? fail(
1403
+ "usage: gateway bench validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X] [--summary-file F] [--timeout-min N]"
1404
+ );
1405
+ const minScoreRaw = str(flags, "min-score");
1406
+ const report = await validateWorld(dir, {
1407
+ slug: str(flags, "slug"),
1408
+ runConfig,
1409
+ branch: str(flags, "branch"),
1410
+ minScore: minScoreRaw === void 0 ? null : Number(minScoreRaw),
1411
+ timeoutMinutes: Number(str(flags, "timeout-min") ?? 45),
1412
+ onProgress: (line) => console.error(line)
1413
+ });
1414
+ for (const run of report.runs) {
1415
+ console.log(
1416
+ `${run.model}: ${run.failure ? `FAIL \u2014 ${run.failure}` : "pass"}${run.report?.score != null ? ` score=${run.report.score.toFixed(3)}` : ""}${run.report?.url ? ` ${run.report.url}` : ""}`
1417
+ );
1418
+ }
1419
+ const summaryFile = str(flags, "summary-file");
1420
+ if (summaryFile) fs2.writeFileSync(summaryFile, report.markdown + "\n");
1421
+ const stepSummary = process.env.GITHUB_STEP_SUMMARY;
1422
+ if (stepSummary) fs2.appendFileSync(stepSummary, report.markdown + "\n");
1423
+ console.log(report.ok ? "validation passed" : "validation FAILED");
1424
+ return report.ok ? 0 : 1;
1425
+ }
1240
1426
  case "status": {
1241
1427
  const dir = positional[0] ?? ".";
1242
1428
  const state = localTreeState(dir);