@withgateway/sdk 0.1.18 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +42 -10
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +42 -10
- package/dist/cli.mjs.map +1 -1
- package/dist/index.js +43 -11
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +43 -11
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +5 -0
- package/dist/worlds/index.d.ts +5 -0
- package/dist/worlds/index.js +42 -10
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +42 -10
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/cli.mjs
CHANGED
|
@@ -501,7 +501,7 @@ async function push(dir, opts = {}) {
|
|
|
501
501
|
if (!fs.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
502
502
|
const localMeta = readLocalMeta(dir) ?? {};
|
|
503
503
|
const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
|
|
504
|
-
const baseVersionId = localMeta.versionId ?? null;
|
|
504
|
+
const baseVersionId = opts.baseVersionId ?? localMeta.versionId ?? null;
|
|
505
505
|
const contentHash = computeContentHash(dir);
|
|
506
506
|
const pyproject = readPyproject(dir);
|
|
507
507
|
const slug = opts.slug ?? pyproject.name ?? path.basename(path.resolve(dir));
|
|
@@ -681,14 +681,15 @@ var init_hub = __esm({
|
|
|
681
681
|
function fmtScore(score) {
|
|
682
682
|
return score === null ? "no score" : score.toFixed(3);
|
|
683
683
|
}
|
|
684
|
-
function buildMarkdown(report, minScore) {
|
|
684
|
+
function buildMarkdown(report, minScore, source) {
|
|
685
685
|
const lines = [];
|
|
686
686
|
lines.push(
|
|
687
687
|
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
688
688
|
);
|
|
689
689
|
lines.push("");
|
|
690
|
+
const commitLink = source?.repoUrl && source?.commit ? `[\`${source.commit.slice(0, 12)}\`](${source.repoUrl}/commit/${source.commit})` : null;
|
|
690
691
|
lines.push(
|
|
691
|
-
`Ran the pull request's own world source (
|
|
692
|
+
`Ran the pull request's own world source` + (commitLink ? ` (${commitLink} on \`${report.branch}\`` : ` (`) + `${commitLink ? ", " : ""}pinned as \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
692
693
|
);
|
|
693
694
|
lines.push("");
|
|
694
695
|
lines.push("| Model | Status | Score | Run |");
|
|
@@ -708,19 +709,29 @@ function buildMarkdown(report, minScore) {
|
|
|
708
709
|
reward: sample.reward
|
|
709
710
|
}))
|
|
710
711
|
);
|
|
711
|
-
if (taskRows.length > 0
|
|
712
|
+
if (taskRows.length > 0) {
|
|
713
|
+
const fold = taskRows.length > 12;
|
|
712
714
|
lines.push("");
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
715
|
+
if (fold) {
|
|
716
|
+
lines.push(
|
|
717
|
+
`<details><summary>Per-scenario rewards (${taskRows.length})</summary>`
|
|
718
|
+
);
|
|
719
|
+
lines.push("");
|
|
720
|
+
} else {
|
|
721
|
+
lines.push(`**Scenarios (${taskRows.length})**`);
|
|
722
|
+
lines.push("");
|
|
723
|
+
}
|
|
724
|
+
lines.push("| Scenario | Model | Reward |");
|
|
716
725
|
lines.push("| --- | --- | ---: |");
|
|
717
726
|
for (const row of taskRows) {
|
|
718
727
|
lines.push(
|
|
719
728
|
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
720
729
|
);
|
|
721
730
|
}
|
|
722
|
-
|
|
723
|
-
|
|
731
|
+
if (fold) {
|
|
732
|
+
lines.push("");
|
|
733
|
+
lines.push("</details>");
|
|
734
|
+
}
|
|
724
735
|
}
|
|
725
736
|
return lines.join("\n");
|
|
726
737
|
}
|
|
@@ -736,9 +747,24 @@ async function validateWorld(dir, opts) {
|
|
|
736
747
|
"No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
|
|
737
748
|
);
|
|
738
749
|
progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
|
|
750
|
+
let tipVersionId = null;
|
|
751
|
+
if (opts.slug) {
|
|
752
|
+
try {
|
|
753
|
+
const tip = await apiRequest(
|
|
754
|
+
options,
|
|
755
|
+
"POST",
|
|
756
|
+
"/api/public/benchmark-containers/resolve-version",
|
|
757
|
+
{ slug: opts.slug, ref: branch }
|
|
758
|
+
);
|
|
759
|
+
tipVersionId = tip.versionId ?? null;
|
|
760
|
+
} catch {
|
|
761
|
+
tipVersionId = null;
|
|
762
|
+
}
|
|
763
|
+
}
|
|
739
764
|
const pushed = await push(dir, {
|
|
740
765
|
slug: opts.slug,
|
|
741
766
|
branch,
|
|
767
|
+
baseVersionId: tipVersionId,
|
|
742
768
|
message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
|
|
743
769
|
...provenance.actor?.name ? { author: provenance.actor.name } : {},
|
|
744
770
|
options
|
|
@@ -819,7 +845,13 @@ async function validateWorld(dir, opts) {
|
|
|
819
845
|
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
820
846
|
runs
|
|
821
847
|
};
|
|
822
|
-
return {
|
|
848
|
+
return {
|
|
849
|
+
...base,
|
|
850
|
+
markdown: buildMarkdown(base, minScore, {
|
|
851
|
+
repoUrl: provenance.repoUrl,
|
|
852
|
+
commit: provenance.commit
|
|
853
|
+
})
|
|
854
|
+
};
|
|
823
855
|
}
|
|
824
856
|
var init_validate = __esm({
|
|
825
857
|
"src/worlds/validate.ts"() {
|