@withgateway/sdk 0.1.14 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +475 -271
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +471 -267
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +627 -59
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +602 -58
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +132 -1
- package/dist/worlds/index.d.ts +132 -1
- package/dist/worlds/index.js +560 -6
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +550 -6
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/worlds/index.mjs
CHANGED
|
@@ -311,9 +311,524 @@ var init_session = __esm({
|
|
|
311
311
|
}
|
|
312
312
|
});
|
|
313
313
|
|
|
314
|
+
// src/worlds/hub.ts
|
|
315
|
+
import { createHash } from "crypto";
|
|
316
|
+
import { gzipSync, gunzipSync } from "zlib";
|
|
317
|
+
import * as fs from "fs";
|
|
318
|
+
import * as path from "path";
|
|
319
|
+
function isExcludedDirComponent(component) {
|
|
320
|
+
return component.startsWith(".") || ROOT_EXCLUDED_DIRS.has(component) || component.endsWith(".egg-info");
|
|
321
|
+
}
|
|
322
|
+
function isIncludedInContentHash(relPath, includeAllRootFiles) {
|
|
323
|
+
const parts = relPath.split("/");
|
|
324
|
+
const name = parts[parts.length - 1];
|
|
325
|
+
const dirParts = parts.slice(0, -1);
|
|
326
|
+
if (name.startsWith(".")) return false;
|
|
327
|
+
if (parts.includes("__pycache__")) return false;
|
|
328
|
+
if (dirParts.length === 0) {
|
|
329
|
+
return includeAllRootFiles || name === "README.md" || name === "pyproject.toml" || name === "gateway-env.toml" || name.endsWith(".py");
|
|
330
|
+
}
|
|
331
|
+
return !dirParts.some(isExcludedDirComponent);
|
|
332
|
+
}
|
|
333
|
+
function collectArchiveRelPaths(dir) {
|
|
334
|
+
if (fs.existsSync(path.join(dir, ".gitignore"))) {
|
|
335
|
+
throw new Error(
|
|
336
|
+
"This bundle has a root .gitignore, which the Python CLI honors via gitignore_parser. Reimplementing that imperfectly here would silently produce a different content hash, so the TS CLI refuses. Remove the .gitignore (or bake its effect into the tree) and retry."
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
const out = [];
|
|
340
|
+
const includeAllRootFiles = fs.existsSync(path.join(dir, "gateway-env.toml"));
|
|
341
|
+
const walk = (rel) => {
|
|
342
|
+
const abs = rel ? path.join(dir, rel) : dir;
|
|
343
|
+
for (const entry of fs.readdirSync(abs, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
344
|
+
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
345
|
+
if (entry.isDirectory()) {
|
|
346
|
+
if (!isExcludedDirComponent(entry.name)) walk(childRel);
|
|
347
|
+
} else if (entry.isFile()) {
|
|
348
|
+
if (isIncludedInContentHash(childRel, includeAllRootFiles)) {
|
|
349
|
+
out.push(childRel);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
};
|
|
354
|
+
walk("");
|
|
355
|
+
return out.sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
356
|
+
}
|
|
357
|
+
function computeContentHash(dir) {
|
|
358
|
+
const hasher = createHash("sha256");
|
|
359
|
+
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
360
|
+
hasher.update(Buffer.from(`file:${relPath}`, "utf-8"));
|
|
361
|
+
hasher.update(fs.readFileSync(path.join(dir, relPath)));
|
|
362
|
+
}
|
|
363
|
+
return hasher.digest("hex");
|
|
364
|
+
}
|
|
365
|
+
function octal(value, length) {
|
|
366
|
+
return Buffer.from(value.toString(8).padStart(length - 1, "0") + "\0", "ascii");
|
|
367
|
+
}
|
|
368
|
+
function tarHeader(name, size) {
|
|
369
|
+
const header = Buffer.alloc(512);
|
|
370
|
+
header.write(name, 0, 100, "utf-8");
|
|
371
|
+
octal(420, 8).copy(header, 100);
|
|
372
|
+
octal(0, 8).copy(header, 108);
|
|
373
|
+
octal(0, 8).copy(header, 116);
|
|
374
|
+
octal(size, 12).copy(header, 124);
|
|
375
|
+
octal(0, 12).copy(header, 136);
|
|
376
|
+
header.write(" ", 148, "ascii");
|
|
377
|
+
header.write("0", 156, "ascii");
|
|
378
|
+
header.write("ustar", 257, "ascii");
|
|
379
|
+
header.write("00", 263, "ascii");
|
|
380
|
+
let sum = 0;
|
|
381
|
+
for (const byte of header) sum += byte;
|
|
382
|
+
header.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
|
|
383
|
+
return header;
|
|
384
|
+
}
|
|
385
|
+
function tarMember(name, payload) {
|
|
386
|
+
const parts = [];
|
|
387
|
+
const nameBytes = Buffer.byteLength(name, "utf-8");
|
|
388
|
+
if (nameBytes > 100) {
|
|
389
|
+
const body = Buffer.from(name + "\0", "utf-8");
|
|
390
|
+
const lHeader = tarHeader("././@LongLink", body.length);
|
|
391
|
+
lHeader.write("L", 156, "ascii");
|
|
392
|
+
let sum = 0;
|
|
393
|
+
lHeader.write(" ", 148, "ascii");
|
|
394
|
+
for (const byte of lHeader) sum += byte;
|
|
395
|
+
lHeader.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
|
|
396
|
+
parts.push(lHeader, body);
|
|
397
|
+
const pad2 = body.length % 512 === 0 ? 0 : 512 - body.length % 512;
|
|
398
|
+
if (pad2) parts.push(Buffer.alloc(pad2));
|
|
399
|
+
parts.push(tarHeader(name.slice(0, 100), payload.length));
|
|
400
|
+
} else {
|
|
401
|
+
parts.push(tarHeader(name, payload.length));
|
|
402
|
+
}
|
|
403
|
+
parts.push(payload);
|
|
404
|
+
const pad = payload.length % 512 === 0 ? 0 : 512 - payload.length % 512;
|
|
405
|
+
if (pad) parts.push(Buffer.alloc(pad));
|
|
406
|
+
return parts;
|
|
407
|
+
}
|
|
408
|
+
function buildSourceTarball(dir) {
|
|
409
|
+
const parts = [];
|
|
410
|
+
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
411
|
+
parts.push(...tarMember(relPath, fs.readFileSync(path.join(dir, relPath))));
|
|
412
|
+
}
|
|
413
|
+
parts.push(Buffer.alloc(1024));
|
|
414
|
+
return gzipSync(Buffer.concat(parts), { level: 9 });
|
|
415
|
+
}
|
|
416
|
+
function readLocalMeta(dir) {
|
|
417
|
+
try {
|
|
418
|
+
const raw = fs.readFileSync(path.join(dir, META_RELPATH), "utf-8");
|
|
419
|
+
const parsed = JSON.parse(raw);
|
|
420
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
421
|
+
} catch {
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
function writeLocalMeta(dir, meta) {
|
|
426
|
+
fs.mkdirSync(path.join(dir, ".gateway"), { recursive: true });
|
|
427
|
+
fs.writeFileSync(path.join(dir, META_RELPATH), JSON.stringify(meta, null, 2));
|
|
428
|
+
}
|
|
429
|
+
function snapshotTreeHashes(dir) {
|
|
430
|
+
const out = {};
|
|
431
|
+
for (const relPath of collectArchiveRelPaths(dir)) {
|
|
432
|
+
out[relPath] = createHash("sha256").update(fs.readFileSync(path.join(dir, relPath))).digest("hex");
|
|
433
|
+
}
|
|
434
|
+
return out;
|
|
435
|
+
}
|
|
436
|
+
function localTreeState(dir) {
|
|
437
|
+
const meta = readLocalMeta(dir);
|
|
438
|
+
const base = meta?.baseFiles;
|
|
439
|
+
if (!base || typeof base !== "object") {
|
|
440
|
+
return { meta, dirty: [], clean: false };
|
|
441
|
+
}
|
|
442
|
+
const now = snapshotTreeHashes(dir);
|
|
443
|
+
const dirty = /* @__PURE__ */ new Set();
|
|
444
|
+
for (const [p, h] of Object.entries(now)) if (base[p] !== h) dirty.add(p);
|
|
445
|
+
for (const p of Object.keys(base)) if (!(p in now)) dirty.add(p);
|
|
446
|
+
return { meta, dirty: [...dirty].sort(), clean: dirty.size === 0 };
|
|
447
|
+
}
|
|
448
|
+
function readPyproject(dir) {
|
|
449
|
+
try {
|
|
450
|
+
const text = fs.readFileSync(path.join(dir, "pyproject.toml"), "utf-8");
|
|
451
|
+
const section = /\[project\]([\s\S]*?)(\n\[|$)/.exec(text)?.[1] ?? "";
|
|
452
|
+
const name = /\bname\s*=\s*"([^"]+)"/.exec(section)?.[1];
|
|
453
|
+
const version = /\bversion\s*=\s*"([^"]+)"/.exec(section)?.[1];
|
|
454
|
+
return { name, version };
|
|
455
|
+
} catch {
|
|
456
|
+
return {};
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
function buildFileManifest(dir) {
|
|
460
|
+
const entries = [];
|
|
461
|
+
const walk = (rel) => {
|
|
462
|
+
const abs = rel ? path.join(dir, rel) : dir;
|
|
463
|
+
for (const entry of fs.readdirSync(abs, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
464
|
+
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
|
465
|
+
if (entry.isDirectory()) {
|
|
466
|
+
if (!SKIP_DIRS.has(entry.name) && !entry.name.endsWith(".egg-info")) {
|
|
467
|
+
walk(childRel);
|
|
468
|
+
}
|
|
469
|
+
} else if (entry.isFile()) {
|
|
470
|
+
const payload = fs.readFileSync(path.join(dir, childRel));
|
|
471
|
+
entries.push({
|
|
472
|
+
path: childRel,
|
|
473
|
+
isDirectory: false,
|
|
474
|
+
size: payload.length,
|
|
475
|
+
sha256: createHash("sha256").update(payload).digest("hex")
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
};
|
|
480
|
+
walk("");
|
|
481
|
+
return entries;
|
|
482
|
+
}
|
|
483
|
+
async function putObject(url, data, shaB64) {
|
|
484
|
+
const resp = await fetch(url, {
|
|
485
|
+
method: "PUT",
|
|
486
|
+
headers: {
|
|
487
|
+
"Content-Type": "application/gzip",
|
|
488
|
+
"x-amz-checksum-sha256": shaB64
|
|
489
|
+
},
|
|
490
|
+
body: new Uint8Array(data)
|
|
491
|
+
});
|
|
492
|
+
if (!resp.ok) {
|
|
493
|
+
throw new Error(`object PUT -> ${resp.status}: ${(await resp.text()).slice(0, 300)}`);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
async function push(dir, opts = {}) {
|
|
497
|
+
const options = opts.options ?? {};
|
|
498
|
+
if (!fs.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
499
|
+
const localMeta = readLocalMeta(dir) ?? {};
|
|
500
|
+
const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
|
|
501
|
+
const baseVersionId = localMeta.versionId ?? null;
|
|
502
|
+
const contentHash = computeContentHash(dir);
|
|
503
|
+
const pyproject = readPyproject(dir);
|
|
504
|
+
const slug = opts.slug ?? pyproject.name ?? path.basename(path.resolve(dir));
|
|
505
|
+
const resolve2 = await apiRequest(
|
|
506
|
+
options,
|
|
507
|
+
"POST",
|
|
508
|
+
"/api/public/benchmark-containers/resolve",
|
|
509
|
+
{ slug, name: pyproject.name ?? slug }
|
|
510
|
+
);
|
|
511
|
+
const tarball = buildSourceTarball(dir);
|
|
512
|
+
const bundleSha256 = sha256b64(tarball);
|
|
513
|
+
const version = await apiRequest(options, "POST", "/api/public/benchmark-containers/versions", {
|
|
514
|
+
containerId: resolve2.containerId,
|
|
515
|
+
contentHash,
|
|
516
|
+
bundleSha256,
|
|
517
|
+
bundleSizeBytes: tarball.length,
|
|
518
|
+
...pyproject.version ? { semanticVersion: pyproject.version } : {},
|
|
519
|
+
pyprojectMeta: pyproject,
|
|
520
|
+
fileManifest: buildFileManifest(dir),
|
|
521
|
+
branch,
|
|
522
|
+
baseVersionId,
|
|
523
|
+
...opts.force ? { force: true } : {},
|
|
524
|
+
...opts.message ? { changeReason: opts.message } : {},
|
|
525
|
+
...opts.author ? { author: opts.author } : {}
|
|
526
|
+
});
|
|
527
|
+
if (version.uploadUrl) await putObject(version.uploadUrl, tarball, bundleSha256);
|
|
528
|
+
await apiRequest(
|
|
529
|
+
options,
|
|
530
|
+
"POST",
|
|
531
|
+
`/api/public/benchmark-containers/versions/${version.versionId}/finalize`,
|
|
532
|
+
{ branch, baseVersionId, ...opts.force ? { force: true } : {} }
|
|
533
|
+
);
|
|
534
|
+
writeLocalMeta(dir, {
|
|
535
|
+
...localMeta,
|
|
536
|
+
slug,
|
|
537
|
+
ref: branch,
|
|
538
|
+
branch,
|
|
539
|
+
versionId: version.versionId,
|
|
540
|
+
contentHash,
|
|
541
|
+
semanticVersion: pyproject.version,
|
|
542
|
+
forkedFromVersionId: baseVersionId,
|
|
543
|
+
baseFiles: snapshotTreeHashes(dir)
|
|
544
|
+
});
|
|
545
|
+
return { slug, versionId: version.versionId, contentHash, branch };
|
|
546
|
+
}
|
|
547
|
+
function extractBundle(bundle, dir) {
|
|
548
|
+
const tar = gunzipSync(bundle);
|
|
549
|
+
let offset = 0;
|
|
550
|
+
let longName = null;
|
|
551
|
+
let written = 0;
|
|
552
|
+
while (offset + 512 <= tar.length) {
|
|
553
|
+
const header = tar.subarray(offset, offset + 512);
|
|
554
|
+
offset += 512;
|
|
555
|
+
if (header.every((b) => b === 0)) break;
|
|
556
|
+
const rawName = header.subarray(0, 100).toString("utf-8").replace(/\0.*$/, "");
|
|
557
|
+
const size = parseInt(header.subarray(124, 136).toString("ascii").replace(/\0.*$/, "").trim(), 8) || 0;
|
|
558
|
+
const typeflag = header.subarray(156, 157).toString("ascii");
|
|
559
|
+
const body = tar.subarray(offset, offset + size);
|
|
560
|
+
offset += size + (size % 512 === 0 ? 0 : 512 - size % 512);
|
|
561
|
+
if (typeflag === "L") {
|
|
562
|
+
longName = body.toString("utf-8").replace(/\0.*$/, "");
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
const name = (longName ?? rawName).replace(/^\.\//, "");
|
|
566
|
+
longName = null;
|
|
567
|
+
if (typeflag !== "0" && typeflag !== "") continue;
|
|
568
|
+
if (name.startsWith("/") || name.split("/").includes("..")) continue;
|
|
569
|
+
const target = path.join(dir, name);
|
|
570
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
571
|
+
fs.writeFileSync(target, body);
|
|
572
|
+
written++;
|
|
573
|
+
}
|
|
574
|
+
return written;
|
|
575
|
+
}
|
|
576
|
+
async function pull(slugRef, target, options = {}) {
|
|
577
|
+
const at = slugRef.indexOf("@");
|
|
578
|
+
const slug = at === -1 ? slugRef : slugRef.slice(0, at);
|
|
579
|
+
const ref = at === -1 ? "latest" : slugRef.slice(at + 1);
|
|
580
|
+
const details = await apiRequest(options, "POST", "/api/public/benchmark-containers/resolve-version", {
|
|
581
|
+
slug,
|
|
582
|
+
ref
|
|
583
|
+
});
|
|
584
|
+
if (!details.downloadUrl) {
|
|
585
|
+
throw new Error(`${slug}@${ref} resolved but has no uploaded bundle`);
|
|
586
|
+
}
|
|
587
|
+
let dir = target ?? path.join(process.cwd(), slug);
|
|
588
|
+
if (!target) {
|
|
589
|
+
let i = 1;
|
|
590
|
+
const base = dir;
|
|
591
|
+
while (fs.existsSync(dir)) dir = `${base}-${i++}`;
|
|
592
|
+
}
|
|
593
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
594
|
+
const resp = await fetch(details.downloadUrl);
|
|
595
|
+
if (!resp.ok) throw new Error(`bundle download -> ${resp.status}`);
|
|
596
|
+
const bundle = Buffer.from(await resp.arrayBuffer());
|
|
597
|
+
if (details.bundleSha256 && sha256b64(bundle) !== details.bundleSha256) {
|
|
598
|
+
throw new Error(
|
|
599
|
+
`bundle integrity check failed for ${slug}@${ref}: downloaded sha256 != recorded bundleSha256`
|
|
600
|
+
);
|
|
601
|
+
}
|
|
602
|
+
const files = extractBundle(bundle, dir);
|
|
603
|
+
const branch = ref === "latest" || /^\d|^[0-9a-f]{8,}$/.test(ref) ? DEFAULT_BRANCH : ref;
|
|
604
|
+
writeLocalMeta(dir, {
|
|
605
|
+
slug,
|
|
606
|
+
ref,
|
|
607
|
+
branch,
|
|
608
|
+
versionId: details.versionId,
|
|
609
|
+
contentHash: details.contentHash,
|
|
610
|
+
semanticVersion: details.semanticVersion ?? void 0,
|
|
611
|
+
baseFiles: snapshotTreeHashes(dir)
|
|
612
|
+
});
|
|
613
|
+
return { slug, dir, versionId: details.versionId, files };
|
|
614
|
+
}
|
|
615
|
+
async function propose(dir, opts) {
|
|
616
|
+
const meta = readLocalMeta(dir);
|
|
617
|
+
if (!meta?.slug || !meta.baseFiles) {
|
|
618
|
+
throw new Error("This dir has no pull/push base \u2014 run pull first, then edit, then propose.");
|
|
619
|
+
}
|
|
620
|
+
const state = localTreeState(dir);
|
|
621
|
+
if (state.clean) throw new Error("Nothing to propose \u2014 the tree matches its base.");
|
|
622
|
+
const changes = state.dirty.map((relPath) => {
|
|
623
|
+
const abs = path.join(dir, relPath);
|
|
624
|
+
if (!fs.existsSync(abs)) return { path: relPath, action: "delete" };
|
|
625
|
+
return {
|
|
626
|
+
path: relPath,
|
|
627
|
+
action: "write",
|
|
628
|
+
newContent: fs.readFileSync(abs, "utf-8")
|
|
629
|
+
};
|
|
630
|
+
});
|
|
631
|
+
const res = await apiRequest(
|
|
632
|
+
opts.options ?? {},
|
|
633
|
+
"POST",
|
|
634
|
+
"/api/public/benchmark-containers/proposals",
|
|
635
|
+
{
|
|
636
|
+
slug: meta.slug,
|
|
637
|
+
title: opts.title,
|
|
638
|
+
changeReason: opts.reason,
|
|
639
|
+
baseVersionId: meta.versionId,
|
|
640
|
+
...opts.targetBranch ? { targetBranch: opts.targetBranch } : {},
|
|
641
|
+
changes
|
|
642
|
+
}
|
|
643
|
+
);
|
|
644
|
+
return res;
|
|
645
|
+
}
|
|
646
|
+
async function createRef(slug, name, kind, versionId, options = {}) {
|
|
647
|
+
return apiRequest(options, "POST", "/api/public/benchmark-containers/refs", {
|
|
648
|
+
slug,
|
|
649
|
+
name,
|
|
650
|
+
kind,
|
|
651
|
+
...versionId ? { versionId } : {}
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
var DEFAULT_BRANCH, META_RELPATH, SKIP_DIRS, ROOT_EXCLUDED_DIRS, sha256b64;
|
|
655
|
+
var init_hub = __esm({
|
|
656
|
+
"src/worlds/hub.ts"() {
|
|
657
|
+
"use strict";
|
|
658
|
+
init_worlds();
|
|
659
|
+
DEFAULT_BRANCH = "main";
|
|
660
|
+
META_RELPATH = path.join(".gateway", ".env-metadata.json");
|
|
661
|
+
SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
662
|
+
".git",
|
|
663
|
+
"__pycache__",
|
|
664
|
+
"node_modules",
|
|
665
|
+
".venv",
|
|
666
|
+
".mypy_cache",
|
|
667
|
+
"dist",
|
|
668
|
+
"build",
|
|
669
|
+
"outputs"
|
|
670
|
+
]);
|
|
671
|
+
ROOT_EXCLUDED_DIRS = /* @__PURE__ */ new Set(["dist", "build", "outputs", "__pycache__"]);
|
|
672
|
+
sha256b64 = (data) => createHash("sha256").update(data).digest("base64");
|
|
673
|
+
}
|
|
674
|
+
});
|
|
675
|
+
|
|
676
|
+
// src/worlds/validate.ts
|
|
677
|
+
function fmtScore(score) {
|
|
678
|
+
return score === null ? "no score" : score.toFixed(3);
|
|
679
|
+
}
|
|
680
|
+
function buildMarkdown(report, minScore) {
|
|
681
|
+
const lines = [];
|
|
682
|
+
lines.push(
|
|
683
|
+
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
684
|
+
);
|
|
685
|
+
lines.push("");
|
|
686
|
+
lines.push(
|
|
687
|
+
`Ran the pull request's own world source (version \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
688
|
+
);
|
|
689
|
+
lines.push("");
|
|
690
|
+
lines.push("| Model | Status | Score | Run |");
|
|
691
|
+
lines.push("| --- | --- | ---: | --- |");
|
|
692
|
+
for (const run of report.runs) {
|
|
693
|
+
const r = run.report;
|
|
694
|
+
const link = r?.url ? `[open](${r.url})` : "\u2014";
|
|
695
|
+
const status = run.failure ? `\u274C ${run.failure}` : `\u2705 ${r?.status ?? "COMPLETED"}`;
|
|
696
|
+
lines.push(
|
|
697
|
+
`| \`${run.model}\` | ${status} | ${fmtScore(r?.score ?? null)} | ${link} |`
|
|
698
|
+
);
|
|
699
|
+
}
|
|
700
|
+
const taskRows = report.runs.flatMap(
|
|
701
|
+
(run) => (run.report?.samples ?? []).map((sample) => ({
|
|
702
|
+
model: run.model,
|
|
703
|
+
task: sample.task ?? "?",
|
|
704
|
+
reward: sample.reward
|
|
705
|
+
}))
|
|
706
|
+
);
|
|
707
|
+
if (taskRows.length > 0 && taskRows.length <= 40) {
|
|
708
|
+
lines.push("");
|
|
709
|
+
lines.push("<details><summary>Per-task rewards</summary>");
|
|
710
|
+
lines.push("");
|
|
711
|
+
lines.push("| Task | Model | Reward |");
|
|
712
|
+
lines.push("| --- | --- | ---: |");
|
|
713
|
+
for (const row of taskRows) {
|
|
714
|
+
lines.push(
|
|
715
|
+
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
716
|
+
);
|
|
717
|
+
}
|
|
718
|
+
lines.push("");
|
|
719
|
+
lines.push("</details>");
|
|
720
|
+
}
|
|
721
|
+
return lines.join("\n");
|
|
722
|
+
}
|
|
723
|
+
async function validateWorld(dir, opts) {
|
|
724
|
+
const options = opts.options ?? {};
|
|
725
|
+
const progress = opts.onProgress ?? (() => {
|
|
726
|
+
});
|
|
727
|
+
const minScore = opts.minScore ?? null;
|
|
728
|
+
const provenance = detectProvenance();
|
|
729
|
+
const branch = opts.branch ?? provenance.branch;
|
|
730
|
+
if (!branch)
|
|
731
|
+
throw new Error(
|
|
732
|
+
"No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
|
|
733
|
+
);
|
|
734
|
+
progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
|
|
735
|
+
const pushed = await push(dir, {
|
|
736
|
+
slug: opts.slug,
|
|
737
|
+
branch,
|
|
738
|
+
message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
|
|
739
|
+
...provenance.actor?.name ? { author: provenance.actor.name } : {},
|
|
740
|
+
options
|
|
741
|
+
});
|
|
742
|
+
progress(`pushed version ${pushed.versionId} (${pushed.contentHash.slice(0, 12)})`);
|
|
743
|
+
const configs = await listRunConfigs(options);
|
|
744
|
+
const runConfig = configs.find((c) => c.id === opts.runConfig) ?? configs.find((c) => c.name === opts.runConfig);
|
|
745
|
+
if (!runConfig) {
|
|
746
|
+
const known = configs.map((c) => c.name).join(", ") || "(none)";
|
|
747
|
+
throw new Error(
|
|
748
|
+
`Run config "${opts.runConfig}" not found in this project. Have: ${known}`
|
|
749
|
+
);
|
|
750
|
+
}
|
|
751
|
+
progress(`dispatching run config "${runConfig.name}" (${runConfig.models.join(", ")})\u2026`);
|
|
752
|
+
const dispatched = await apiRequest(options, "POST", "/api/public/benchmark-containers/run-configs/dispatch", {
|
|
753
|
+
runConfigId: runConfig.id,
|
|
754
|
+
versionId: pushed.versionId,
|
|
755
|
+
...provenance
|
|
756
|
+
});
|
|
757
|
+
const host = (options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai").replace(/\/+$/, "");
|
|
758
|
+
let projectId = null;
|
|
759
|
+
try {
|
|
760
|
+
const projects = await apiRequest(
|
|
761
|
+
options,
|
|
762
|
+
"GET",
|
|
763
|
+
"/api/public/projects"
|
|
764
|
+
);
|
|
765
|
+
projectId = projects.data?.[0]?.id ?? null;
|
|
766
|
+
} catch {
|
|
767
|
+
projectId = null;
|
|
768
|
+
}
|
|
769
|
+
const fallbackUrl = (report) => {
|
|
770
|
+
if (report.url) return report.url;
|
|
771
|
+
const evaluationId = report.evaluations[0]?.id;
|
|
772
|
+
return projectId && evaluationId ? `${host}/project/${projectId}/environment/runs/${encodeURIComponent(evaluationId)}` : null;
|
|
773
|
+
};
|
|
774
|
+
const runs = [];
|
|
775
|
+
for (const run of dispatched.runs) {
|
|
776
|
+
if (!run.ok) {
|
|
777
|
+
runs.push({
|
|
778
|
+
model: run.model,
|
|
779
|
+
runId: run.runId,
|
|
780
|
+
report: null,
|
|
781
|
+
failure: `dispatch failed: ${run.error ?? "unknown error"}`
|
|
782
|
+
});
|
|
783
|
+
continue;
|
|
784
|
+
}
|
|
785
|
+
progress(`waiting on ${run.model} (run ${run.runId})\u2026`);
|
|
786
|
+
let report = null;
|
|
787
|
+
let failure = null;
|
|
788
|
+
try {
|
|
789
|
+
report = await waitForRun(run.runId, {
|
|
790
|
+
timeoutMinutes: opts.timeoutMinutes ?? 45,
|
|
791
|
+
options,
|
|
792
|
+
onTick: (r) => progress(
|
|
793
|
+
`\u2026${run.model}: ${r.status.toLowerCase()} (${r.shardsCompleted ?? 0}/${r.parallelism} shards)`
|
|
794
|
+
)
|
|
795
|
+
});
|
|
796
|
+
report = { ...report, url: fallbackUrl(report) };
|
|
797
|
+
if (report.status.toUpperCase() !== "COMPLETED") {
|
|
798
|
+
failure = report.status;
|
|
799
|
+
} else if (minScore !== null) {
|
|
800
|
+
if (report.score === null) failure = "completed without a score";
|
|
801
|
+
else if (report.score < minScore)
|
|
802
|
+
failure = `score ${fmtScore(report.score)} < ${minScore}`;
|
|
803
|
+
}
|
|
804
|
+
} catch (error) {
|
|
805
|
+
failure = error instanceof Error ? error.message : String(error);
|
|
806
|
+
}
|
|
807
|
+
runs.push({ model: run.model, runId: run.runId, report, failure });
|
|
808
|
+
}
|
|
809
|
+
const base = {
|
|
810
|
+
ok: runs.length > 0 && runs.every((r) => r.failure === null),
|
|
811
|
+
slug: pushed.slug,
|
|
812
|
+
branch,
|
|
813
|
+
versionId: pushed.versionId,
|
|
814
|
+
contentHash: pushed.contentHash,
|
|
815
|
+
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
816
|
+
runs
|
|
817
|
+
};
|
|
818
|
+
return { ...base, markdown: buildMarkdown(base, minScore) };
|
|
819
|
+
}
|
|
820
|
+
var init_validate = __esm({
|
|
821
|
+
"src/worlds/validate.ts"() {
|
|
822
|
+
"use strict";
|
|
823
|
+
init_worlds();
|
|
824
|
+
init_hub();
|
|
825
|
+
}
|
|
826
|
+
});
|
|
827
|
+
|
|
314
828
|
// src/worlds/index.ts
|
|
315
829
|
var worlds_exports = {};
|
|
316
830
|
__export(worlds_exports, {
|
|
831
|
+
DEFAULT_BRANCH: () => DEFAULT_BRANCH,
|
|
317
832
|
WorldSession: () => WorldSession,
|
|
318
833
|
WorldSessionCallError: () => WorldSessionCallError,
|
|
319
834
|
WorldSessionTimeout: () => WorldSessionTimeout,
|
|
@@ -321,17 +836,30 @@ __export(worlds_exports, {
|
|
|
321
836
|
WorldTask: () => WorldTask,
|
|
322
837
|
apiRequest: () => apiRequest,
|
|
323
838
|
attachSession: () => attachSession,
|
|
839
|
+
buildSourceTarball: () => buildSourceTarball,
|
|
840
|
+
collectArchiveRelPaths: () => collectArchiveRelPaths,
|
|
841
|
+
computeContentHash: () => computeContentHash,
|
|
842
|
+
createRef: () => createRef,
|
|
324
843
|
deleteEnvSecret: () => deleteEnvSecret,
|
|
325
844
|
detectProvenance: () => detectProvenance,
|
|
326
845
|
getRun: () => getRun,
|
|
846
|
+
isIncludedInContentHash: () => isIncludedInContentHash,
|
|
327
847
|
listEnvSecretKeys: () => listEnvSecretKeys,
|
|
328
848
|
listRunConfigs: () => listRunConfigs,
|
|
849
|
+
localTreeState: () => localTreeState,
|
|
329
850
|
openSession: () => openSession,
|
|
330
851
|
openWorld: () => openWorld,
|
|
852
|
+
propose: () => propose,
|
|
853
|
+
pull: () => pull,
|
|
854
|
+
push: () => push,
|
|
855
|
+
readLocalMeta: () => readLocalMeta,
|
|
331
856
|
runSessions: () => runSessions,
|
|
332
857
|
schemaFromSignature: () => schemaFromSignature,
|
|
333
858
|
setEnvSecret: () => setEnvSecret,
|
|
334
|
-
|
|
859
|
+
snapshotTreeHashes: () => snapshotTreeHashes,
|
|
860
|
+
validateWorld: () => validateWorld,
|
|
861
|
+
waitForRun: () => waitForRun,
|
|
862
|
+
writeLocalMeta: () => writeLocalMeta
|
|
335
863
|
});
|
|
336
864
|
function resolveEnv(options) {
|
|
337
865
|
const host = options.host ?? process.env.GATEWAY_HOST ?? "https://withgateway.ai";
|
|
@@ -344,9 +872,9 @@ function resolveEnv(options) {
|
|
|
344
872
|
}
|
|
345
873
|
return { host: host.replace(/\/+$/, ""), publicKey, secretKey };
|
|
346
874
|
}
|
|
347
|
-
async function apiRequest(options, method,
|
|
875
|
+
async function apiRequest(options, method, path2, body) {
|
|
348
876
|
const { host, publicKey, secretKey } = resolveEnv(options);
|
|
349
|
-
const res = await fetch(`${host}${
|
|
877
|
+
const res = await fetch(`${host}${path2}`, {
|
|
350
878
|
method,
|
|
351
879
|
headers: {
|
|
352
880
|
"content-type": "application/json",
|
|
@@ -357,7 +885,7 @@ async function apiRequest(options, method, path, body) {
|
|
|
357
885
|
const text = await res.text();
|
|
358
886
|
if (!res.ok) {
|
|
359
887
|
throw new Error(
|
|
360
|
-
`${method} ${
|
|
888
|
+
`${method} ${path2} -> ${res.status}: ${text.slice(0, 400)}`
|
|
361
889
|
);
|
|
362
890
|
}
|
|
363
891
|
return JSON.parse(text);
|
|
@@ -419,7 +947,7 @@ async function waitForRun(runId, opts = {}) {
|
|
|
419
947
|
);
|
|
420
948
|
opts.onTick?.(run);
|
|
421
949
|
await new Promise(
|
|
422
|
-
(
|
|
950
|
+
(resolve2) => setTimeout(resolve2, (opts.intervalSeconds ?? 20) * 1e3)
|
|
423
951
|
);
|
|
424
952
|
}
|
|
425
953
|
}
|
|
@@ -493,6 +1021,8 @@ var WorldTask, WorldSet;
|
|
|
493
1021
|
var init_worlds = __esm({
|
|
494
1022
|
"src/worlds/index.ts"() {
|
|
495
1023
|
init_session();
|
|
1024
|
+
init_hub();
|
|
1025
|
+
init_validate();
|
|
496
1026
|
WorldTask = class {
|
|
497
1027
|
constructor(world, name, variant) {
|
|
498
1028
|
this.world = world;
|
|
@@ -556,6 +1086,7 @@ var init_worlds = __esm({
|
|
|
556
1086
|
});
|
|
557
1087
|
init_worlds();
|
|
558
1088
|
export {
|
|
1089
|
+
DEFAULT_BRANCH,
|
|
559
1090
|
WorldSession,
|
|
560
1091
|
WorldSessionCallError,
|
|
561
1092
|
WorldSessionTimeout,
|
|
@@ -563,16 +1094,29 @@ export {
|
|
|
563
1094
|
WorldTask,
|
|
564
1095
|
apiRequest,
|
|
565
1096
|
attachSession,
|
|
1097
|
+
buildSourceTarball,
|
|
1098
|
+
collectArchiveRelPaths,
|
|
1099
|
+
computeContentHash,
|
|
1100
|
+
createRef,
|
|
566
1101
|
deleteEnvSecret,
|
|
567
1102
|
detectProvenance,
|
|
568
1103
|
getRun,
|
|
1104
|
+
isIncludedInContentHash,
|
|
569
1105
|
listEnvSecretKeys,
|
|
570
1106
|
listRunConfigs,
|
|
1107
|
+
localTreeState,
|
|
571
1108
|
openSession,
|
|
572
1109
|
openWorld,
|
|
1110
|
+
propose,
|
|
1111
|
+
pull,
|
|
1112
|
+
push,
|
|
1113
|
+
readLocalMeta,
|
|
573
1114
|
runSessions,
|
|
574
1115
|
schemaFromSignature,
|
|
575
1116
|
setEnvSecret,
|
|
576
|
-
|
|
1117
|
+
snapshotTreeHashes,
|
|
1118
|
+
validateWorld,
|
|
1119
|
+
waitForRun,
|
|
1120
|
+
writeLocalMeta
|
|
577
1121
|
};
|
|
578
1122
|
//# sourceMappingURL=index.mjs.map
|