@withgateway/sdk 0.1.14 → 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs CHANGED
@@ -312,273 +312,11 @@ var init_session = __esm({
312
312
  }
313
313
  });
314
314
 
315
- // src/worlds/index.ts
316
- var worlds_exports = {};
317
- __export(worlds_exports, {
318
- WorldSession: () => WorldSession,
319
- WorldSessionCallError: () => WorldSessionCallError,
320
- WorldSessionTimeout: () => WorldSessionTimeout,
321
- WorldSet: () => WorldSet,
322
- WorldTask: () => WorldTask,
323
- apiRequest: () => apiRequest,
324
- attachSession: () => attachSession,
325
- deleteEnvSecret: () => deleteEnvSecret,
326
- detectProvenance: () => detectProvenance,
327
- getRun: () => getRun,
328
- listEnvSecretKeys: () => listEnvSecretKeys,
329
- listRunConfigs: () => listRunConfigs,
330
- openSession: () => openSession,
331
- openWorld: () => openWorld,
332
- runSessions: () => runSessions,
333
- schemaFromSignature: () => schemaFromSignature,
334
- setEnvSecret: () => setEnvSecret,
335
- waitForRun: () => waitForRun
336
- });
337
- function resolveEnv(options) {
338
- const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
339
- const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
340
- const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
341
- if (!publicKey || !secretKey) {
342
- throw new Error(
343
- "Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
344
- );
345
- }
346
- return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
347
- }
348
- async function apiRequest(options, method, path2, body) {
349
- const { host, publicKey, secretKey } = resolveEnv(options);
350
- const res = await fetch(`${host}${path2}`, {
351
- method,
352
- headers: {
353
- "content-type": "application/json",
354
- authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
355
- },
356
- ...body === void 0 ? {} : { body: JSON.stringify(body) }
357
- });
358
- const text = await res.text();
359
- if (!res.ok) {
360
- throw new Error(
361
- `${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
362
- );
363
- }
364
- return JSON.parse(text);
365
- }
366
- async function setEnvSecret(slug, key, value, options = {}) {
367
- return apiRequest(
368
- options,
369
- "PUT",
370
- "/api/public/benchmark-containers/secrets",
371
- {
372
- slug,
373
- key,
374
- value
375
- }
376
- );
377
- }
378
- async function listEnvSecretKeys(slug, options = {}) {
379
- const res = await apiRequest(
380
- options,
381
- "GET",
382
- `/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
383
- );
384
- return res.secrets;
385
- }
386
- async function deleteEnvSecret(slug, key, options = {}) {
387
- await apiRequest(
388
- options,
389
- "DELETE",
390
- "/api/public/benchmark-containers/secrets",
391
- {
392
- slug,
393
- key
394
- }
395
- );
396
- }
397
- async function listRunConfigs(options = {}) {
398
- const res = await apiRequest(
399
- options,
400
- "GET",
401
- "/api/public/benchmark-containers/run-configs"
402
- );
403
- return res.runConfigs;
404
- }
405
- async function getRun(runId, options = {}) {
406
- return apiRequest(
407
- options,
408
- "GET",
409
- `/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
410
- );
411
- }
412
- async function waitForRun(runId, opts = {}) {
413
- const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
414
- for (; ; ) {
415
- const run = await getRun(runId, opts.options ?? {});
416
- if (run.terminal) return run;
417
- if (Date.now() > deadline)
418
- throw new Error(
419
- `Run ${runId} still ${run.status} after the wait timeout.`
420
- );
421
- opts.onTick?.(run);
422
- await new Promise(
423
- (resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
424
- );
425
- }
426
- }
427
- function detectProvenance() {
428
- const env = globalThis.process?.env ?? {};
429
- const out = {};
430
- const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
431
- if (ref) out.branch = ref;
432
- if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
433
- const server = env.GITHUB_SERVER_URL ?? "https://github.com";
434
- if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
435
- if (env.GITHUB_ACTOR) {
436
- out.actor = {
437
- kind: "ci",
438
- id: env.GITHUB_ACTOR,
439
- name: env.GITHUB_ACTOR,
440
- // GitHub serves every account's avatar at /<login>.png.
441
- imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
442
- };
443
- }
444
- if (!out.branch || !out.commit) {
445
- try {
446
- const { execFileSync } = __require("child_process");
447
- const git = (args) => execFileSync("git", args, {
448
- encoding: "utf-8",
449
- stdio: ["ignore", "pipe", "ignore"]
450
- }).toString().trim();
451
- out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
452
- out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
453
- if (!out.repoUrl) {
454
- const remote = git(["config", "--get", "remote.origin.url"]);
455
- if (remote) out.repoUrl = remote.replace(/\.git$/, "");
456
- }
457
- if (!out.actor) {
458
- const name = git(["config", "--get", "user.name"]);
459
- if (name) out.actor = { kind: "user", name };
460
- }
461
- } catch {
462
- }
463
- }
464
- return out;
465
- }
466
- async function openWorld(slugRef, options = {}) {
467
- const at = slugRef.indexOf("@");
468
- const slug = at === -1 ? slugRef : slugRef.slice(0, at);
469
- const ref = at === -1 ? null : slugRef.slice(at + 1);
470
- const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
471
- slug,
472
- ...ref ? { ref } : {}
473
- });
474
- const world = new WorldSet(
475
- slugRef,
476
- slug,
477
- ref,
478
- {
479
- versionId: version.versionId,
480
- semanticVersion: version.semanticVersion ?? null,
481
- contentHash: version.contentHash,
482
- state: version.state
483
- },
484
- [],
485
- options
486
- );
487
- for (const name of version.taskNames ?? []) {
488
- const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
489
- world.tasks.push(new WorldTask(world, name, variant));
490
- }
491
- return world;
492
- }
493
- var WorldTask, WorldSet;
494
- var init_worlds = __esm({
495
- "src/worlds/index.ts"() {
496
- "use strict";
497
- init_session();
498
- WorldTask = class {
499
- constructor(world, name, variant) {
500
- this.world = world;
501
- this.name = name;
502
- this.variant = variant;
503
- }
504
- };
505
- WorldSet = class {
506
- constructor(slugRef, slug, ref, version, tasks, options) {
507
- this.slugRef = slugRef;
508
- this.slug = slug;
509
- this.ref = ref;
510
- this.version = version;
511
- this.tasks = tasks;
512
- this.options = options;
513
- }
514
- task(variant) {
515
- const hit = this.tasks.find(
516
- (t) => t.variant === variant || t.name.includes(variant)
517
- );
518
- if (!hit) {
519
- const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
520
- throw new Error(`no task matches ${variant}; have: ${known}`);
521
- }
522
- return hit;
523
- }
524
- /**
525
- * Open a LIVE session against one of this world's tasks: the platform boots
526
- * (or reuses) a warm container, and your agent calls its tools through the
527
- * platform. See worlds/session.ts.
528
- */
529
- async open(task, opts = {}) {
530
- const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
531
- return openSession2(this.slug, task, {
532
- versionId: this.version.versionId,
533
- reuseWarm: opts.reuseWarm,
534
- options: this.options
535
- });
536
- }
537
- /**
538
- * Start a HOSTED run of a run config against this world — the platform
539
- * executes the (Python) world in parallel containers per the config's
540
- * `parallelism` and files one graded run per model. Returns those runs.
541
- */
542
- async dispatch(runConfigId, opts = {}) {
543
- const provenance = opts.provenance ?? detectProvenance();
544
- return apiRequest(
545
- this.options,
546
- "POST",
547
- "/api/public/benchmark-containers/run-configs/dispatch",
548
- {
549
- runConfigId,
550
- ...opts.containerId ? { containerId: opts.containerId } : {},
551
- versionId: opts.versionId ?? this.version.versionId,
552
- ...provenance
553
- }
554
- );
555
- }
556
- };
557
- }
558
- });
559
-
560
- // src/cli.ts
561
- init_worlds();
562
-
563
315
  // src/worlds/hub.ts
564
- init_worlds();
565
316
  import { createHash } from "crypto";
566
317
  import { gzipSync, gunzipSync } from "zlib";
567
318
  import * as fs from "fs";
568
319
  import * as path from "path";
569
- var DEFAULT_BRANCH = "main";
570
- var META_RELPATH = path.join(".gateway", ".env-metadata.json");
571
- var SKIP_DIRS = /* @__PURE__ */ new Set([
572
- ".git",
573
- "__pycache__",
574
- "node_modules",
575
- ".venv",
576
- ".mypy_cache",
577
- "dist",
578
- "build",
579
- "outputs"
580
- ]);
581
- var ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
582
320
  function isExcludedDirComponent(component) {
583
321
  return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
584
322
  }
@@ -708,7 +446,6 @@ function localTreeState(dir) {
708
446
  for (const p of Object.keys(base)) if (!(p in now)) dirty.add(p);
709
447
  return { meta, dirty: [...dirty].sort(), clean: dirty.size === 0 };
710
448
  }
711
- var sha256b64 = (data) => createHash("sha256").update(data).digest("base64");
712
449
  function readPyproject(dir) {
713
450
  try {
714
451
  const text = fs.readFileSync(path.join(dir, "pyproject.toml"), "utf-8");
@@ -915,11 +652,449 @@ async function createRef(slug, name, kind, versionId, options = {}) {
915
652
  ...versionId ? { versionId } : {}
916
653
  });
917
654
  }
655
+ var DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, sha256b64;
656
+ var init_hub = __esm({
657
+ "src/worlds/hub.ts"() {
658
+ "use strict";
659
+ init_worlds();
660
+ DEFAULT_BRANCH = "main";
661
+ META_RELPATH = path.join(".gateway", ".env-metadata.json");
662
+ SKIP_DIRS = /* @__PURE__ */ new Set([
663
+ ".git",
664
+ "__pycache__",
665
+ "node_modules",
666
+ ".venv",
667
+ ".mypy_cache",
668
+ "dist",
669
+ "build",
670
+ "outputs"
671
+ ]);
672
+ ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
673
+ sha256b64 = (data) => createHash("sha256").update(data).digest("base64");
674
+ }
675
+ });
918
676
 
919
- // src/cli.ts
920
- function parseArgv(argv) {
921
- const positional = [];
922
- const flags = {};
677
+ // src/worlds/validate.ts
678
+ function fmtScore(score) {
679
+ return score === null ? "no score" : score.toFixed(3);
680
+ }
681
+ function buildMarkdown(report, minScore) {
682
+ const lines = [];
683
+ lines.push(
684
+ `### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
685
+ );
686
+ lines.push("");
687
+ lines.push(
688
+ `Ran the pull request's own world source (version \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
689
+ );
690
+ lines.push("");
691
+ lines.push("| Model | Status | Score | Run |");
692
+ lines.push("| --- | --- | ---: | --- |");
693
+ for (const run of report.runs) {
694
+ const r = run.report;
695
+ const link = r?.url ? `[open](${r.url})` : "\u2014";
696
+ const status = run.failure ? `\u274C ${run.failure}` : `\u2705 ${r?.status ?? "COMPLETED"}`;
697
+ lines.push(
698
+ `| \`${run.model}\` | ${status} | ${fmtScore(r?.score ?? null)} | ${link} |`
699
+ );
700
+ }
701
+ const taskRows = report.runs.flatMap(
702
+ (run) => (run.report?.samples ?? []).map((sample) => ({
703
+ model: run.model,
704
+ task: sample.task ?? "?",
705
+ reward: sample.reward
706
+ }))
707
+ );
708
+ if (taskRows.length > 0 && taskRows.length <= 40) {
709
+ lines.push("");
710
+ lines.push("<details><summary>Per-task rewards</summary>");
711
+ lines.push("");
712
+ lines.push("| Task | Model | Reward |");
713
+ lines.push("| --- | --- | ---: |");
714
+ for (const row of taskRows) {
715
+ lines.push(
716
+ `| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
717
+ );
718
+ }
719
+ lines.push("");
720
+ lines.push("</details>");
721
+ }
722
+ return lines.join("\n");
723
+ }
724
+ async function validateWorld(dir, opts) {
725
+ const options = opts.options ?? {};
726
+ const progress = opts.onProgress ?? (() => {
727
+ });
728
+ const minScore = opts.minScore ?? null;
729
+ const provenance = detectProvenance();
730
+ const branch = opts.branch ?? provenance.branch;
731
+ if (!branch)
732
+ throw new Error(
733
+ "No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
734
+ );
735
+ progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
736
+ const pushed = await push(dir, {
737
+ slug: opts.slug,
738
+ branch,
739
+ message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
740
+ ...provenance.actor?.name ? { author: provenance.actor.name } : {},
741
+ options
742
+ });
743
+ progress(`pushed version ${pushed.versionId} (${pushed.contentHash.slice(0, 12)})`);
744
+ const configs = await listRunConfigs(options);
745
+ const runConfig = configs.find((c) => c.id === opts.runConfig) ?? configs.find((c) => c.name === opts.runConfig);
746
+ if (!runConfig) {
747
+ const known = configs.map((c) => c.name).join(", ") || "(none)";
748
+ throw new Error(
749
+ `Run config "${opts.runConfig}" not found in this project. Have: ${known}`
750
+ );
751
+ }
752
+ progress(`dispatching run config "${runConfig.name}" (${runConfig.models.join(", ")})\u2026`);
753
+ const dispatched = await apiRequest(options, "POST", "/api/public/benchmark-containers/run-configs/dispatch", {
754
+ runConfigId: runConfig.id,
755
+ versionId: pushed.versionId,
756
+ ...provenance
757
+ });
758
+ const host = (options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai").replace(/\/+$/, "");
759
+ let projectId = null;
760
+ try {
761
+ const projects = await apiRequest(
762
+ options,
763
+ "GET",
764
+ "/api/public/projects"
765
+ );
766
+ projectId = projects.data?.[0]?.id ?? null;
767
+ } catch {
768
+ projectId = null;
769
+ }
770
+ const fallbackUrl = (report) => {
771
+ if (report.url) return report.url;
772
+ const evaluationId = report.evaluations[0]?.id;
773
+ return projectId && evaluationId ? `${host}/project/${projectId}/environment/runs/${encodeURIComponent(evaluationId)}` : null;
774
+ };
775
+ const runs = [];
776
+ for (const run of dispatched.runs) {
777
+ if (!run.ok) {
778
+ runs.push({
779
+ model: run.model,
780
+ runId: run.runId,
781
+ report: null,
782
+ failure: `dispatch failed: ${run.error ?? "unknown error"}`
783
+ });
784
+ continue;
785
+ }
786
+ progress(`waiting on ${run.model} (run ${run.runId})\u2026`);
787
+ let report = null;
788
+ let failure = null;
789
+ try {
790
+ report = await waitForRun(run.runId, {
791
+ timeoutMinutes: opts.timeoutMinutes ?? 45,
792
+ options,
793
+ onTick: (r) => progress(
794
+ `\u2026${run.model}: ${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
795
+ )
796
+ });
797
+ report = { ...report, url: fallbackUrl(report) };
798
+ if (report.status.toUpperCase() !== "COMPLETED") {
799
+ failure = report.status;
800
+ } else if (minScore !== null) {
801
+ if (report.score === null) failure = "completed without a score";
802
+ else if (report.score < minScore)
803
+ failure = `score ${fmtScore(report.score)} < ${minScore}`;
804
+ }
805
+ } catch (error) {
806
+ failure = error instanceof Error ? error.message : String(error);
807
+ }
808
+ runs.push({ model: run.model, runId: run.runId, report, failure });
809
+ }
810
+ const base = {
811
+ ok: runs.length > 0 && runs.every((r) => r.failure === null),
812
+ slug: pushed.slug,
813
+ branch,
814
+ versionId: pushed.versionId,
815
+ contentHash: pushed.contentHash,
816
+ runConfig: { id: runConfig.id, name: runConfig.name },
817
+ runs
818
+ };
819
+ return { ...base, markdown: buildMarkdown(base, minScore) };
820
+ }
821
+ var init_validate = __esm({
822
+ "src/worlds/validate.ts"() {
823
+ "use strict";
824
+ init_worlds();
825
+ init_hub();
826
+ }
827
+ });
828
+
829
+ // src/worlds/index.ts
830
+ var worlds_exports = {};
831
+ __export(worlds_exports, {
832
+ DEFAULT_BRANCH: () => DEFAULT_BRANCH,
833
+ WorldSession: () => WorldSession,
834
+ WorldSessionCallError: () => WorldSessionCallError,
835
+ WorldSessionTimeout: () => WorldSessionTimeout,
836
+ WorldSet: () => WorldSet,
837
+ WorldTask: () => WorldTask,
838
+ apiRequest: () => apiRequest,
839
+ attachSession: () => attachSession,
840
+ buildSourceTarball: () => buildSourceTarball,
841
+ collectArchiveRelPaths: () => collectArchiveRelPaths,
842
+ computeContentHash: () => computeContentHash,
843
+ createRef: () => createRef,
844
+ deleteEnvSecret: () => deleteEnvSecret,
845
+ detectProvenance: () => detectProvenance,
846
+ getRun: () => getRun,
847
+ isIncludedInContentHash: () => isIncludedInContentHash,
848
+ listEnvSecretKeys: () => listEnvSecretKeys,
849
+ listRunConfigs: () => listRunConfigs,
850
+ localTreeState: () => localTreeState,
851
+ openSession: () => openSession,
852
+ openWorld: () => openWorld,
853
+ propose: () => propose,
854
+ pull: () => pull,
855
+ push: () => push,
856
+ readLocalMeta: () => readLocalMeta,
857
+ runSessions: () => runSessions,
858
+ schemaFromSignature: () => schemaFromSignature,
859
+ setEnvSecret: () => setEnvSecret,
860
+ snapshotTreeHashes: () => snapshotTreeHashes,
861
+ validateWorld: () => validateWorld,
862
+ waitForRun: () => waitForRun,
863
+ writeLocalMeta: () => writeLocalMeta
864
+ });
865
+ function resolveEnv(options) {
866
+ const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
867
+ const publicKey = options.publicKey ?? process.env.GATEWAY_PUBLIC_KEY;
868
+ const secretKey = options.secretKey ?? process.env.GATEWAY_SECRET_KEY;
869
+ if (!publicKey || !secretKey) {
870
+ throw new Error(
871
+ "Gateway keys missing: set GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY (or pass publicKey/secretKey)."
872
+ );
873
+ }
874
+ return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
875
+ }
876
+ async function apiRequest(options, method, path2, body) {
877
+ const { host, publicKey, secretKey } = resolveEnv(options);
878
+ const res = await fetch(`${host}${path2}`, {
879
+ method,
880
+ headers: {
881
+ "content-type": "application/json",
882
+ authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
883
+ },
884
+ ...body === void 0 ? {} : { body: JSON.stringify(body) }
885
+ });
886
+ const text = await res.text();
887
+ if (!res.ok) {
888
+ throw new Error(
889
+ `${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
890
+ );
891
+ }
892
+ return JSON.parse(text);
893
+ }
894
+ async function setEnvSecret(slug, key, value, options = {}) {
895
+ return apiRequest(
896
+ options,
897
+ "PUT",
898
+ "/api/public/benchmark-containers/secrets",
899
+ {
900
+ slug,
901
+ key,
902
+ value
903
+ }
904
+ );
905
+ }
906
+ async function listEnvSecretKeys(slug, options = {}) {
907
+ const res = await apiRequest(
908
+ options,
909
+ "GET",
910
+ `/api/public/benchmark-containers/secrets?slug=${encodeURIComponent(slug)}`
911
+ );
912
+ return res.secrets;
913
+ }
914
+ async function deleteEnvSecret(slug, key, options = {}) {
915
+ await apiRequest(
916
+ options,
917
+ "DELETE",
918
+ "/api/public/benchmark-containers/secrets",
919
+ {
920
+ slug,
921
+ key
922
+ }
923
+ );
924
+ }
925
+ async function listRunConfigs(options = {}) {
926
+ const res = await apiRequest(
927
+ options,
928
+ "GET",
929
+ "/api/public/benchmark-containers/run-configs"
930
+ );
931
+ return res.runConfigs;
932
+ }
933
+ async function getRun(runId, options = {}) {
934
+ return apiRequest(
935
+ options,
936
+ "GET",
937
+ `/api/public/benchmark-containers/runs/${encodeURIComponent(runId)}`
938
+ );
939
+ }
940
+ async function waitForRun(runId, opts = {}) {
941
+ const deadline = Date.now() + (opts.timeoutMinutes ?? 45) * 6e4;
942
+ for (; ; ) {
943
+ const run = await getRun(runId, opts.options ?? {});
944
+ if (run.terminal) return run;
945
+ if (Date.now() > deadline)
946
+ throw new Error(
947
+ `Run ${runId} still ${run.status} after the wait timeout.`
948
+ );
949
+ opts.onTick?.(run);
950
+ await new Promise(
951
+ (resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
952
+ );
953
+ }
954
+ }
955
+ function detectProvenance() {
956
+ const env = globalThis.process?.env ?? {};
957
+ const out = {};
958
+ const ref = env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME;
959
+ if (ref) out.branch = ref;
960
+ if (env.GITHUB_SHA) out.commit = env.GITHUB_SHA;
961
+ const server = env.GITHUB_SERVER_URL ?? "https://github.com";
962
+ if (env.GITHUB_REPOSITORY) out.repoUrl = `${server}/${env.GITHUB_REPOSITORY}`;
963
+ if (env.GITHUB_ACTOR) {
964
+ out.actor = {
965
+ kind: "ci",
966
+ id: env.GITHUB_ACTOR,
967
+ name: env.GITHUB_ACTOR,
968
+ // GitHub serves every account's avatar at /<login>.png.
969
+ imageUrl: `${server}/${env.GITHUB_ACTOR}.png`
970
+ };
971
+ }
972
+ if (!out.branch || !out.commit) {
973
+ try {
974
+ const { execFileSync } = __require("child_process");
975
+ const git = (args) => execFileSync("git", args, {
976
+ encoding: "utf-8",
977
+ stdio: ["ignore", "pipe", "ignore"]
978
+ }).toString().trim();
979
+ out.branch = out.branch ?? git(["rev-parse", "--abbrev-ref", "HEAD"]);
980
+ out.commit = out.commit ?? git(["rev-parse", "HEAD"]);
981
+ if (!out.repoUrl) {
982
+ const remote = git(["config", "--get", "remote.origin.url"]);
983
+ if (remote) out.repoUrl = remote.replace(/\.git$/, "");
984
+ }
985
+ if (!out.actor) {
986
+ const name = git(["config", "--get", "user.name"]);
987
+ if (name) out.actor = { kind: "user", name };
988
+ }
989
+ } catch {
990
+ }
991
+ }
992
+ return out;
993
+ }
994
+ async function openWorld(slugRef, options = {}) {
995
+ const at = slugRef.indexOf("@");
996
+ const slug = at === -1 ? slugRef : slugRef.slice(0, at);
997
+ const ref = at === -1 ? null : slugRef.slice(at + 1);
998
+ const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
999
+ slug,
1000
+ ...ref ? { ref } : {}
1001
+ });
1002
+ const world = new WorldSet(
1003
+ slugRef,
1004
+ slug,
1005
+ ref,
1006
+ {
1007
+ versionId: version.versionId,
1008
+ semanticVersion: version.semanticVersion ?? null,
1009
+ contentHash: version.contentHash,
1010
+ state: version.state
1011
+ },
1012
+ [],
1013
+ options
1014
+ );
1015
+ for (const name of version.taskNames ?? []) {
1016
+ const variant = /^[a-z0-9]+-([a-z0-9]+)-/.exec(name)?.[1] ?? null;
1017
+ world.tasks.push(new WorldTask(world, name, variant));
1018
+ }
1019
+ return world;
1020
+ }
1021
+ var WorldTask, WorldSet;
1022
+ var init_worlds = __esm({
1023
+ "src/worlds/index.ts"() {
1024
+ "use strict";
1025
+ init_session();
1026
+ init_hub();
1027
+ init_validate();
1028
+ WorldTask = class {
1029
+ constructor(world, name, variant) {
1030
+ this.world = world;
1031
+ this.name = name;
1032
+ this.variant = variant;
1033
+ }
1034
+ };
1035
+ WorldSet = class {
1036
+ constructor(slugRef, slug, ref, version, tasks, options) {
1037
+ this.slugRef = slugRef;
1038
+ this.slug = slug;
1039
+ this.ref = ref;
1040
+ this.version = version;
1041
+ this.tasks = tasks;
1042
+ this.options = options;
1043
+ }
1044
+ task(variant) {
1045
+ const hit = this.tasks.find(
1046
+ (t) => t.variant === variant || t.name.includes(variant)
1047
+ );
1048
+ if (!hit) {
1049
+ const known = this.tasks.map((t) => t.variant ?? t.name).join(", ");
1050
+ throw new Error(`no task matches ${variant}; have: ${known}`);
1051
+ }
1052
+ return hit;
1053
+ }
1054
+ /**
1055
+ * Open a LIVE session against one of this world's tasks: the platform boots
1056
+ * (or reuses) a warm container, and your agent calls its tools through the
1057
+ * platform. See worlds/session.ts.
1058
+ */
1059
+ async open(task, opts = {}) {
1060
+ const { openSession: openSession2 } = await Promise.resolve().then(() => (init_session(), session_exports));
1061
+ return openSession2(this.slug, task, {
1062
+ versionId: this.version.versionId,
1063
+ reuseWarm: opts.reuseWarm,
1064
+ options: this.options
1065
+ });
1066
+ }
1067
+ /**
1068
+ * Start a HOSTED run of a run config against this world — the platform
1069
+ * executes the (Python) world in parallel containers per the config's
1070
+ * `parallelism` and files one graded run per model. Returns those runs.
1071
+ */
1072
+ async dispatch(runConfigId, opts = {}) {
1073
+ const provenance = opts.provenance ?? detectProvenance();
1074
+ return apiRequest(
1075
+ this.options,
1076
+ "POST",
1077
+ "/api/public/benchmark-containers/run-configs/dispatch",
1078
+ {
1079
+ runConfigId,
1080
+ ...opts.containerId ? { containerId: opts.containerId } : {},
1081
+ versionId: opts.versionId ?? this.version.versionId,
1082
+ ...provenance
1083
+ }
1084
+ );
1085
+ }
1086
+ };
1087
+ }
1088
+ });
1089
+
1090
+ // src/cli.ts
1091
+ init_worlds();
1092
+ init_hub();
1093
+ init_validate();
1094
+ import * as fs2 from "fs";
1095
+ function parseArgv(argv) {
1096
+ const positional = [];
1097
+ const flags = {};
923
1098
  for (let i = 0; i < argv.length; i++) {
924
1099
  const arg = argv[i];
925
1100
  if (arg.startsWith("--")) {
@@ -964,6 +1139,9 @@ var HELP = `gateway bench <command> \u2014 worlds from TypeScript
964
1139
 
965
1140
  pull <slug[@ref]> [--dir D] materialize a working dir
966
1141
  push [dir] [--slug S] [--branch B] [--force] [--message M]
1142
+ validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X]
1143
+ [--summary-file F] [--timeout-min N]
1144
+ push this checkout, run it, gate \u2014 the PR-validation entry
967
1145
  checkout <slug@ref> --dir D re-aim an existing working dir
968
1146
  status [dir] clean/dirty vs base + hash check
969
1147
  branch <slug> <name> [--version-id V] create a branch
@@ -1224,6 +1402,32 @@ Run hosted instead: gateway bench dispatch <runConfigId> \u2014 same worlds, gra
1224
1402
  );
1225
1403
  return 0;
1226
1404
  }
1405
+ case "validate": {
1406
+ const dir = positional[0] ?? ".";
1407
+ const runConfig = str(flags, "run-config") ?? fail(
1408
+ "usage: gateway bench validate [dir] --run-config NAME [--slug S] [--branch B] [--min-score X] [--summary-file F] [--timeout-min N]"
1409
+ );
1410
+ const minScoreRaw = str(flags, "min-score");
1411
+ const report = await validateWorld(dir, {
1412
+ slug: str(flags, "slug"),
1413
+ runConfig,
1414
+ branch: str(flags, "branch"),
1415
+ minScore: minScoreRaw === void 0 ? null : Number(minScoreRaw),
1416
+ timeoutMinutes: Number(str(flags, "timeout-min") ?? 45),
1417
+ onProgress: (line) => console.error(line)
1418
+ });
1419
+ for (const run of report.runs) {
1420
+ console.log(
1421
+ `${run.model}: ${run.failure ? `FAIL \u2014 ${run.failure}` : "pass"}${run.report?.score != null ? ` score=${run.report.score.toFixed(3)}` : ""}${run.report?.url ? ` ${run.report.url}` : ""}`
1422
+ );
1423
+ }
1424
+ const summaryFile = str(flags, "summary-file");
1425
+ if (summaryFile) fs2.writeFileSync(summaryFile, report.markdown + "\n");
1426
+ const stepSummary = process.env.GITHUB_STEP_SUMMARY;
1427
+ if (stepSummary) fs2.appendFileSync(stepSummary, report.markdown + "\n");
1428
+ console.log(report.ok ? "validation passed" : "validation FAILED");
1429
+ return report.ok ? 0 : 1;
1430
+ }
1227
1431
  case "status": {
1228
1432
  const dir = positional[0] ?? ".";
1229
1433
  const state = localTreeState(dir);