@withgateway/sdk 0.1.14 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs CHANGED
@@ -312,273 +312,11 @@ var init_session = __esm({
312
312
  }
313
313
  });
314
314
 
315
- // src/worlds/index.ts
316
- var worlds_exports = {};
317
- __export(worlds_exports, {
318
- WorldSession: () => WorldSession,
319
- WorldSessionCallError: () => WorldSessionCallError,
320
- WorldSessionTimeout: () => WorldSessionTimeout,
321
- WorldSet: () => WorldSet,
322
- WorldTask: () => WorldTask,
323
- apiRequest: () => apiRequest,
324
- attachSession: () => attachSession,
325
- deleteEnvSecret: () => deleteEnvSecret,
326
- detectProvenance: () => detectProvenance,
327
- getRun: () => getRun,
328
- listEnvSecretKeys: () => listEnvSecretKeys,
329
- listRunConfigs: () => listRunConfigs,
330
- openSession: () => openSession,
331
- openWorld: () => openWorld,
332
- runSessions: () => runSessions,
333
- schemaFromSignature: () => schemaFromSignature,
334
- setEnvSecret: () => setEnvSecret,
335
- waitForRun: () => waitForRun
336
- });
337
- function resolveEnv(options) {
338
- const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
339
- const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
340
- const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
341
- if (!publicKey || !secretKey) {
342
- throw new Error(
343
- "Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
344
- );
345
- }
346
- return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
347
- }
348
- async function apiRequest(options, method, path2, body) {
349
- const { host, publicKey, secretKey } = resolveEnv(options);
350
- const res = await fetch(`${host}${path2}`, {
351
- method,
352
- headers: {
353
- "content-type": "application/json",
354
- authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
355
- },
356
- ...body === void 0 ? {} : { body: JSON.stringify(body) }
357
- });
358
- const text = await res.text();
359
- if (!res.ok) {
360
- throw new Error(
361
- `${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
362
- );
363
- }
364
- return JSON.parse(text);
365
- }
366
- async function setEnvSecret(slug, key, value, options = {}) {
367
- return apiRequest(
368
- options,
369
- "PUT",
370
- "/api/public/benchmark-containers/secrets",
371
- {
372
- slug,
373
- key,
374
- value
375
- }
376
- );
377
- }
378
- async function listEnvSecretKeys(slug, options = {}) {
379
- const res = await apiRequest(
380
- options,
381
- "GET",
382
- `/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
383
- );
384
- return res.secrets;
385
- }
386
- async function deleteEnvSecret(slug, key, options = {}) {
387
- await apiRequest(
388
- options,
389
- "DELETE",
390
- "/api/public/benchmark-containers/secrets",
391
- {
392
- slug,
393
- key
394
- }
395
- );
396
- }
397
- async function listRunConfigs(options = {}) {
398
- const res = await apiRequest(
399
- options,
400
- "GET",
401
- "/api/public/benchmark-containers/run-configs"
402
- );
403
- return res.runConfigs;
404
- }
405
- async function getRun(runId, options = {}) {
406
- return apiRequest(
407
- options,
408
- "GET",
409
- `/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
410
- );
411
- }
412
- async function waitForRun(runId, opts = {}) {
413
- const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
414
- for (; ; ) {
415
- const run = await getRun(runId, opts.options ?? {});
416
- if (run.terminal) return run;
417
- if (Date.now() > deadline)
418
- throw new Error(
419
- `Run ${runId} still ${run.status} after the wait timeout.`
420
- );
421
- opts.onTick?.(run);
422
- await new Promise(
423
- (resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
424
- );
425
- }
426
- }
427
- function detectProvenance() {
428
- const env = globalThis.process?.env ?? {};
429
- const out = {};
430
- const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
431
- if (ref) out.branch = ref;
432
- if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
433
- const server = env.GITHUB_SERVER_URL ?? "https://github.com";
434
- if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
435
- if (env.GITHUB_ACTOR) {
436
- out.actor = {
437
- kind: "ci",
438
- id: env.GITHUB_ACTOR,
439
- name: env.GITHUB_ACTOR,
440
- // GitHub serves every account's avatar at /<login>.png.
441
- imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
442
- };
443
- }
444
- if (!out.branch || !out.commit) {
445
- try {
446
- const { execFileSync } = __require("child_process");
447
- const git = (args) => execFileSync("git", args, {
448
- encoding: "utf-8",
449
- stdio: ["ignore", "pipe", "ignore"]
450
- }).toString().trim();
451
- out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
452
- out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
453
- if (!out.repoUrl) {
454
- const remote = git(["config", "--get", "remote.origin.url"]);
455
- if (remote) out.repoUrl = remote.replace(/\.git$/, "");
456
- }
457
- if (!out.actor) {
458
- const name = git(["config", "--get", "user.name"]);
459
- if (name) out.actor = { kind: "user", name };
460
- }
461
- } catch {
462
- }
463
- }
464
- return out;
465
- }
466
- async function openWorld(slugRef, options = {}) {
467
- const at = slugRef.indexOf("@");
468
- const slug = at === -1 ? slugRef : slugRef.slice(0, at);
469
- const ref = at === -1 ? null : slugRef.slice(at + 1);
470
- const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
471
- slug,
472
- ...ref ? { ref } : {}
473
- });
474
- const world = new WorldSet(
475
- slugRef,
476
- slug,
477
- ref,
478
- {
479
- versionId: version.versionId,
480
- semanticVersion: version.semanticVersion ?? null,
481
- contentHash: version.contentHash,
482
- state: version.state
483
- },
484
- [],
485
- options
486
- );
487
- for (const name of version.taskNames ?? []) {
488
- const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
489
- world.tasks.push(new WorldTask(world, name, variant));
490
- }
491
- return world;
492
- }
493
- var WorldTask, WorldSet;
494
- var init_worlds = __esm({
495
- "src/worlds/index.ts"() {
496
- "use strict";
497
- init_session();
498
- WorldTask = class {
499
- constructor(world, name, variant) {
500
- this.world = world;
501
- this.name = name;
502
- this.variant = variant;
503
- }
504
- };
505
- WorldSet = class {
506
- constructor(slugRef, slug, ref, version, tasks, options) {
507
- this.slugRef = slugRef;
508
- this.slug = slug;
509
- this.ref = ref;
510
- this.version = version;
511
- this.tasks = tasks;
512
- this.options = options;
513
- }
514
- task(variant) {
515
- const hit = this.tasks.find(
516
- (t) => t.variant === variant || t.name.includes(variant)
517
- );
518
- if (!hit) {
519
- const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
520
- throw new Error(`no task matches ${variant}; have: ${known}`);
521
- }
522
- return hit;
523
- }
524
- /**
525
- * Open a LIVE session against one of this world's tasks: the platform boots
526
- * (or reuses) a warm container, and your agent calls its tools through the
527
- * platform. See worlds/session.ts.
528
- */
529
- async open(task, opts = {}) {
530
- const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
531
- return openSession2(this.slug, task, {
532
- versionId: this.version.versionId,
533
- reuseWarm: opts.reuseWarm,
534
- options: this.options
535
- });
536
- }
537
- /**
538
- * Start a HOSTED run of a run config against this world — the platform
539
- * executes the (Python) world in parallel containers per the config's
540
- * `parallelism` and files one graded run per model. Returns those runs.
541
- */
542
- async dispatch(runConfigId, opts = {}) {
543
- const provenance = opts.provenance ?? detectProvenance();
544
- return apiRequest(
545
- this.options,
546
- "POST",
547
- "/api/public/benchmark-containers/run-configs/dispatch",
548
- {
549
- runConfigId,
550
- ...opts.containerId ? { containerId: opts.containerId } : {},
551
- versionId: opts.versionId ?? this.version.versionId,
552
- ...provenance
553
- }
554
- );
555
- }
556
- };
557
- }
558
- });
559
-
560
- // src/cli.ts
561
- init_worlds();
562
-
563
315
  // src/worlds/hub.ts
564
- init_worlds();
565
316
  import { createHash } from "crypto";
566
317
  import { gzipSync, gunzipSync } from "zlib";
567
318
  import * as fs from "fs";
568
319
  import * as path from "path";
569
- var DEFAULT_BRANCH = "main";
570
- var META_RELPATH = path.join(".gateway", ".env-metadata.json");
571
- var SKIP_DIRS = /* @__PURE__ */ new Set([
572
- ".git",
573
- "__pycache__",
574
- "node_modules",
575
- ".venv",
576
- ".mypy_cache",
577
- "dist",
578
- "build",
579
- "outputs"
580
- ]);
581
- var ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
582
320
  function isExcludedDirComponent(component) {
583
321
  return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
584
322
  }
@@ -708,7 +446,6 @@ function localTreeState(dir) {
708
446
  for (const p of Object.keys(base)) if (!(p in now)) dirty.add(p);
709
447
  return { meta, dirty: [...dirty].sort(), clean: dirty.size === 0 };
710
448
  }
711
- var sha256b64 = (data) => createHash("sha256").update(data).digest("base64");
712
449
  function readPyproject(dir) {
713
450
  try {
714
451
  const text = fs.readFileSync(path.join(dir, "pyproject.toml"), "utf-8");
@@ -915,11 +652,431 @@ async function createRef(slug, name, kind, versionId, options = {}) {
915
652
  ...versionId ? { versionId } : {}
916
653
  });
917
654
  }
655
+ var DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, sha256b64;
656
+ var init_hub = __esm({
657
+ "src/worlds/hub.ts"() {
658
+ "use strict";
659
+ init_worlds();
660
+ DEFAULT_BRANCH = "main";
661
+ META_RELPATH = path.join(".gateway", ".env-metadata.json");
662
+ SKIP_DIRS = /* @__PURE__ */ new Set([
663
+ ".git",
664
+ "__pycache__",
665
+ "node_modules",
666
+ ".venv",
667
+ ".mypy_cache",
668
+ "dist",
669
+ "build",
670
+ "outputs"
671
+ ]);
672
+ ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
673
+ sha256b64 = (data) => createHash("sha256").update(data).digest("base64");
674
+ }
675
+ });
918
676
 
919
- // src/cli.ts
920
- function parseArgv(argv) {
921
- const positional = [];
922
- const flags = {};
677
+ // src/worlds/validate.ts
678
+ function fmtScore(score) {
679
+ return score === null ? "no score" : score.toFixed(3);
680
+ }
681
+ function buildMarkdown(report, minScore) {
682
+ const lines = [];
683
+ lines.push(
684
+ `### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
685
+ );
686
+ lines.push("");
687
+ lines.push(
688
+ `Ran the pull request's own world source (version \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
689
+ );
690
+ lines.push("");
691
+ lines.push("| Model | Status | Score | Run |");
692
+ lines.push("| --- | --- | ---: | --- |");
693
+ for (const run of report.runs) {
694
+ const r = run.report;
695
+ const link = r?.url ? `[open](${r.url})` : "\u2014";
696
+ const status = run.failure ? `\u274C ${run.failure}` : `\u2705 ${r?.status ?? "COMPLETED"}`;
697
+ lines.push(
698
+ `| \`${run.model}\` | ${status} | ${fmtScore(r?.score ?? null)} | ${link} |`
699
+ );
700
+ }
701
+ const taskRows = report.runs.flatMap(
702
+ (run) => (run.report?.samples ?? []).map((sample) => ({
703
+ model: run.model,
704
+ task: sample.task ?? "?",
705
+ reward: sample.reward
706
+ }))
707
+ );
708
+ if (taskRows.length > 0 && taskRows.length <= 40) {
709
+ lines.push("");
710
+ lines.push("<details><summary>Per-task rewards</summary>");
711
+ lines.push("");
712
+ lines.push("| Task | Model | Reward |");
713
+ lines.push("| --- | --- | ---: |");
714
+ for (const row of taskRows) {
715
+ lines.push(
716
+ `| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
717
+ );
718
+ }
719
+ lines.push("");
720
+ lines.push("</details>");
721
+ }
722
+ return lines.join("\n");
723
+ }
724
+ async function validateWorld(dir, opts) {
725
+ const options = opts.options ?? {};
726
+ const progress = opts.onProgress ?? (() => {
727
+ });
728
+ const minScore = opts.minScore ?? null;
729
+ const provenance = detectProvenance();
730
+ const branch = opts.branch ?? provenance.branch;
731
+ if (!branch)
732
+ throw new Error(
733
+ "No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
734
+ );
735
+ progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
736
+ const pushed = await push(dir, {
737
+ slug: opts.slug,
738
+ branch,
739
+ message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
740
+ ...provenance.actor?.name ? { author: provenance.actor.name } : {},
741
+ options
742
+ });
743
+ progress(`pushed version ${pushed.versionId} (${pushed.contentHash.slice(0, 12)})`);
744
+ const configs = await listRunConfigs(options);
745
+ const runConfig = configs.find((c) => c.id === opts.runConfig) ?? configs.find((c) => c.name === opts.runConfig);
746
+ if (!runConfig) {
747
+ const known = configs.map((c) => c.name).join(", ") || "(none)";
748
+ throw new Error(
749
+ `Run config "${opts.runConfig}" not found in this project. Have: ${known}`
750
+ );
751
+ }
752
+ progress(`dispatching run config "${runConfig.name}" (${runConfig.models.join(", ")})\u2026`);
753
+ const dispatched = await apiRequest(options, "POST", "/api/public/benchmark-containers/run-configs/dispatch", {
754
+ runConfigId: runConfig.id,
755
+ versionId: pushed.versionId,
756
+ ...provenance
757
+ });
758
+ const runs = [];
759
+ for (const run of dispatched.runs) {
760
+ if (!run.ok) {
761
+ runs.push({
762
+ model: run.model,
763
+ runId: run.runId,
764
+ report: null,
765
+ failure: `dispatch failed: ${run.error ?? "unknown error"}`
766
+ });
767
+ continue;
768
+ }
769
+ progress(`waiting on ${run.model} (run ${run.runId})\u2026`);
770
+ let report = null;
771
+ let failure = null;
772
+ try {
773
+ report = await waitForRun(run.runId, {
774
+ timeoutMinutes: opts.timeoutMinutes ?? 45,
775
+ options,
776
+ onTick: (r) => progress(
777
+ `\u2026${run.model}: ${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
778
+ )
779
+ });
780
+ if (report.status.toUpperCase() !== "COMPLETED") {
781
+ failure = report.status;
782
+ } else if (minScore !== null) {
783
+ if (report.score === null) failure = "completed without a score";
784
+ else if (report.score < minScore)
785
+ failure = `score ${fmtScore(report.score)} < ${minScore}`;
786
+ }
787
+ } catch (error) {
788
+ failure = error instanceof Error ? error.message : String(error);
789
+ }
790
+ runs.push({ model: run.model, runId: run.runId, report, failure });
791
+ }
792
+ const base = {
793
+ ok: runs.length > 0 && runs.every((r) => r.failure === null),
794
+ slug: pushed.slug,
795
+ branch,
796
+ versionId: pushed.versionId,
797
+ contentHash: pushed.contentHash,
798
+ runConfig: { id: runConfig.id, name: runConfig.name },
799
+ runs
800
+ };
801
+ return { ...base, markdown: buildMarkdown(base, minScore) };
802
+ }
803
+ var init_validate = __esm({
804
+ "src/worlds/validate.ts"() {
805
+ "use strict";
806
+ init_worlds();
807
+ init_hub();
808
+ }
809
+ });
810
+
811
+ // src/worlds/index.ts
812
+ var worlds_exports = {};
813
+ __export(worlds_exports, {
814
+ DEFAULT_BRANCH: () => DEFAULT_BRANCH,
815
+ WorldSession: () => WorldSession,
816
+ WorldSessionCallError: () => WorldSessionCallError,
817
+ WorldSessionTimeout: () => WorldSessionTimeout,
818
+ WorldSet: () => WorldSet,
819
+ WorldTask: () => WorldTask,
820
+ apiRequest: () => apiRequest,
821
+ attachSession: () => attachSession,
822
+ buildSourceTarball: () => buildSourceTarball,
823
+ collectArchiveRelPaths: () => collectArchiveRelPaths,
824
+ computeContentHash: () => computeContentHash,
825
+ createRef: () => createRef,
826
+ deleteEnvSecret: () => deleteEnvSecret,
827
+ detectProvenance: () => detectProvenance,
828
+ getRun: () => getRun,
829
+ isIncludedInContentHash: () => isIncludedInContentHash,
830
+ listEnvSecretKeys: () => listEnvSecretKeys,
831
+ listRunConfigs: () => listRunConfigs,
832
+ localTreeState: () => localTreeState,
833
+ openSession: () => openSession,
834
+ openWorld: () => openWorld,
835
+ propose: () => propose,
836
+ pull: () => pull,
837
+ push: () => push,
838
+ readLocalMeta: () => readLocalMeta,
839
+ runSessions: () => runSessions,
840
+ schemaFromSignature: () => schemaFromSignature,
841
+ setEnvSecret: () => setEnvSecret,
842
+ snapshotTreeHashes: () => snapshotTreeHashes,
843
+ validateWorld: () => validateWorld,
844
+ waitForRun: () => waitForRun,
845
+ writeLocalMeta: () => writeLocalMeta
846
+ });
847
+ function resolveEnv(options) {
848
+ const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
849
+ const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
850
+ const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
851
+ if (!publicKey || !secretKey) {
852
+ throw new Error(
853
+ "Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
854
+ );
855
+ }
856
+ return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
857
+ }
858
+ async function apiRequest(options, method, path2, body) {
859
+ const { host, publicKey, secretKey } = resolveEnv(options);
860
+ const res = await fetch(`${host}${path2}`, {
861
+ method,
862
+ headers: {
863
+ "content-type": "application/json",
864
+ authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
865
+ },
866
+ ...body === void 0 ? {} : { body: JSON.stringify(body) }
867
+ });
868
+ const text = await res.text();
869
+ if (!res.ok) {
870
+ throw new Error(
871
+ `${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
872
+ );
873
+ }
874
+ return JSON.parse(text);
875
+ }
876
+ async function setEnvSecret(slug, key, value, options = {}) {
877
+ return apiRequest(
878
+ options,
879
+ "PUT",
880
+ "/api/public/benchmark-containers/secrets",
881
+ {
882
+ slug,
883
+ key,
884
+ value
885
+ }
886
+ );
887
+ }
888
+ async function listEnvSecretKeys(slug, options = {}) {
889
+ const res = await apiRequest(
890
+ options,
891
+ "GET",
892
+ `/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
893
+ );
894
+ return res.secrets;
895
+ }
896
+ async function deleteEnvSecret(slug, key, options = {}) {
897
+ await apiRequest(
898
+ options,
899
+ "DELETE",
900
+ "/api/public/benchmark-containers/secrets",
901
+ {
902
+ slug,
903
+ key
904
+ }
905
+ );
906
+ }
907
+ async function listRunConfigs(options = {}) {
908
+ const res = await apiRequest(
909
+ options,
910
+ "GET",
911
+ "/api/public/benchmark-containers/run-configs"
912
+ );
913
+ return res.runConfigs;
914
+ }
915
+ async function getRun(runId, options = {}) {
916
+ return apiRequest(
917
+ options,
918
+ "GET",
919
+ `/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
920
+ );
921
+ }
922
+ async function waitForRun(runId, opts = {}) {
923
+ const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
924
+ for (; ; ) {
925
+ const run = await getRun(runId, opts.options ?? {});
926
+ if (run.terminal) return run;
927
+ if (Date.now() > deadline)
928
+ throw new Error(
929
+ `Run ${runId} still ${run.status} after the wait timeout.`
930
+ );
931
+ opts.onTick?.(run);
932
+ await new Promise(
933
+ (resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
934
+ );
935
+ }
936
+ }
937
+ function detectProvenance() {
938
+ const env = globalThis.process?.env ?? {};
939
+ const out = {};
940
+ const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
941
+ if (ref) out.branch = ref;
942
+ if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
943
+ const server = env.GITHUB_SERVER_URL ?? "https://github.com";
944
+ if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
945
+ if (env.GITHUB_ACTOR) {
946
+ out.actor = {
947
+ kind: "ci",
948
+ id: env.GITHUB_ACTOR,
949
+ name: env.GITHUB_ACTOR,
950
+ // GitHub serves every account's avatar at /<login>.png.
951
+ imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
952
+ };
953
+ }
954
+ if (!out.branch || !out.commit) {
955
+ try {
956
+ const { execFileSync } = __require("child_process");
957
+ const git = (args) => execFileSync("git", args, {
958
+ encoding: "utf-8",
959
+ stdio: ["ignore", "pipe", "ignore"]
960
+ }).toString().trim();
961
+ out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
962
+ out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
963
+ if (!out.repoUrl) {
964
+ const remote = git(["config", "--get", "remote.origin.url"]);
965
+ if (remote) out.repoUrl = remote.replace(/\.git$/, "");
966
+ }
967
+ if (!out.actor) {
968
+ const name = git(["config", "--get", "user.name"]);
969
+ if (name) out.actor = { kind: "user", name };
970
+ }
971
+ } catch {
972
+ }
973
+ }
974
+ return out;
975
+ }
976
+ async function openWorld(slugRef, options = {}) {
977
+ const at = slugRef.indexOf("@");
978
+ const slug = at === -1 ? slugRef : slugRef.slice(0, at);
979
+ const ref = at === -1 ? null : slugRef.slice(at + 1);
980
+ const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
981
+ slug,
982
+ ...ref ? { ref } : {}
983
+ });
984
+ const world = new WorldSet(
985
+ slugRef,
986
+ slug,
987
+ ref,
988
+ {
989
+ versionId: version.versionId,
990
+ semanticVersion: version.semanticVersion ?? null,
991
+ contentHash: version.contentHash,
992
+ state: version.state
993
+ },
994
+ [],
995
+ options
996
+ );
997
+ for (const name of version.taskNames ?? []) {
998
+ const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
999
+ world.tasks.push(new WorldTask(world, name, variant));
1000
+ }
1001
+ return world;
1002
+ }
1003
+ var WorldTask, WorldSet;
1004
+ var init_worlds = __esm({
1005
+ "src/worlds/index.ts"() {
1006
+ "use strict";
1007
+ init_session();
1008
+ init_hub();
1009
+ init_validate();
1010
+ WorldTask = class {
1011
+ constructor(world, name, variant) {
1012
+ this.world = world;
1013
+ this.name = name;
1014
+ this.variant = variant;
1015
+ }
1016
+ };
1017
+ WorldSet = class {
1018
+ constructor(slugRef, slug, ref, version, tasks, options) {
1019
+ this.slugRef = slugRef;
1020
+ this.slug = slug;
1021
+ this.ref = ref;
1022
+ this.version = version;
1023
+ this.tasks = tasks;
1024
+ this.options = options;
1025
+ }
1026
+ task(variant) {
1027
+ const hit = this.tasks.find(
1028
+ (t) => t.variant === variant || t.name.includes(variant)
1029
+ );
1030
+ if (!hit) {
1031
+ const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
1032
+ throw new Error(`no task matches ${variant}; have: ${known}`);
1033
+ }
1034
+ return hit;
1035
+ }
1036
+ /**
1037
+ * Open a LIVE session against one of this world's tasks: the platform boots
1038
+ * (or reuses) a warm container, and your agent calls its tools through the
1039
+ * platform. See worlds/session.ts.
1040
+ */
1041
+ async open(task, opts = {}) {
1042
+ const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
1043
+ return openSession2(this.slug, task, {
1044
+ versionId: this.version.versionId,
1045
+ reuseWarm: opts.reuseWarm,
1046
+ options: this.options
1047
+ });
1048
+ }
1049
+ /**
1050
+ * Start a HOSTED run of a run config against this world — the platform
1051
+ * executes the (Python) world in parallel containers per the config's
1052
+ * `parallelism` and files one graded run per model. Returns those runs.
1053
+ */
1054
+ async dispatch(runConfigId, opts = {}) {
1055
+ const provenance = opts.provenance ?? detectProvenance();
1056
+ return apiRequest(
1057
+ this.options,
1058
+ "POST",
1059
+ "/api/public/benchmark-containers/run-configs/dispatch",
1060
+ {
1061
+ runConfigId,
1062
+ ...opts.containerId ? { containerId: opts.containerId } : {},
1063
+ versionId: opts.versionId ?? this.version.versionId,
1064
+ ...provenance
1065
+ }
1066
+ );
1067
+ }
1068
+ };
1069
+ }
1070
+ });
1071
+
1072
+ // src/cli.ts
1073
+ init_worlds();
1074
+ init_hub();
1075
+ init_validate();
1076
+ import * as fs2 from "fs";
1077
+ function parseArgv(argv) {
1078
+ const positional = [];
1079
+ const flags = {};
923
1080
  for (let i = 0; i < argv.length; i++) {
924
1081
  const arg = argv[i];
925
1082
  if (arg.startsWith("--")) {
@@ -964,6 +1121,9 @@ var HELP = `gateway bench <command> \u2014 worlds from TypeScript
964
1121
 
965
1122
  pull <slug[@ref]> [--dir D] materialize a working dir
966
1123
  push [dir] [--slug S] [--branch B] [--force] [--message M]
1124
+ validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X]
1125
+ [--summary-file F] [--timeout-min N]
1126
+ push this checkout, run it, gate \u2014 the PR-validation entry
967
1127
  checkout <slug@ref> --dir D re-aim an existing working dir
968
1128
  status [dir] clean/dirty vs base + hash check
969
1129
  branch <slug> <name> [--version-id V] create a branch
@@ -1224,6 +1384,32 @@ Run hosted instead: gateway bench dispatch <runConfigId> \u2014 same worlds, gra
1224
1384
  );
1225
1385
  return 0;
1226
1386
  }
1387
+ case "validate": {
1388
+ const dir = positional[0] ?? ".";
1389
+ const runConfig = str(flags, "run-config") ?? fail(
1390
+ "usage: gateway bench validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X] [--summary-file F] [--timeout-min N]"
1391
+ );
1392
+ const minScoreRaw = str(flags, "min-score");
1393
+ const report = await validateWorld(dir, {
1394
+ slug: str(flags, "slug"),
1395
+ runConfig,
1396
+ branch: str(flags, "branch"),
1397
+ minScore: minScoreRaw === void 0 ? null : Number(minScoreRaw),
1398
+ timeoutMinutes: Number(str(flags, "timeout-min") ?? 45),
1399
+ onProgress: (line) => console.error(line)
1400
+ });
1401
+ for (const run of report.runs) {
1402
+ console.log(
1403
+ `${run.model}: ${run.failure ? `FAIL \u2014 ${run.failure}` : "pass"}${run.report?.score != null ? ` score=${run.report.score.toFixed(3)}` : ""}${run.report?.url ? ` ${run.report.url}` : ""}`
1404
+ );
1405
+ }
1406
+ const summaryFile = str(flags, "summary-file");
1407
+ if (summaryFile) fs2.writeFileSync(summaryFile, report.markdown + "\n");
1408
+ const stepSummary = process.env.GITHUB_STEP_SUMMARY;
1409
+ if (stepSummary) fs2.appendFileSync(stepSummary, report.markdown + "\n");
1410
+ console.log(report.ok ? "validation passed" : "validation FAILED");
1411
+ return report.ok ? 0 : 1;
1412
+ }
1227
1413
  case "status": {
1228
1414
  const dir = positional[0] ?? ".";
1229
1415
  const state = localTreeState(dir);