@withgateway/sdk 0.1.18 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +42 -10
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +42 -10
- package/dist/cli.mjs.map +1 -1
- package/dist/index.js +43 -11
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +43 -11
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +5 -0
- package/dist/worlds/index.d.ts +5 -0
- package/dist/worlds/index.js +42 -10
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +42 -10
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -510,7 +510,7 @@ async function push(dir, opts = {}) {
|
|
|
510
510
|
if (!fs.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
511
511
|
const localMeta = readLocalMeta(dir) ?? {};
|
|
512
512
|
const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
|
|
513
|
-
const baseVersionId = localMeta.versionId ?? null;
|
|
513
|
+
const baseVersionId = opts.baseVersionId ?? localMeta.versionId ?? null;
|
|
514
514
|
const contentHash = computeContentHash(dir);
|
|
515
515
|
const pyproject = readPyproject(dir);
|
|
516
516
|
const slug = opts.slug ?? pyproject.name ?? path.basename(path.resolve(dir));
|
|
@@ -694,14 +694,15 @@ var init_hub = __esm({
|
|
|
694
694
|
function fmtScore(score) {
|
|
695
695
|
return score === null ? "no score" : score.toFixed(3);
|
|
696
696
|
}
|
|
697
|
-
function buildMarkdown(report, minScore) {
|
|
697
|
+
function buildMarkdown(report, minScore, source) {
|
|
698
698
|
const lines = [];
|
|
699
699
|
lines.push(
|
|
700
700
|
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
701
701
|
);
|
|
702
702
|
lines.push("");
|
|
703
|
+
const commitLink = source?.repoUrl && source?.commit ? `[\`${source.commit.slice(0, 12)}\`](${source.repoUrl}/commit/${source.commit})` : null;
|
|
703
704
|
lines.push(
|
|
704
|
-
`Ran the pull request's own world source (
|
|
705
|
+
`Ran the pull request's own world source` + (commitLink ? ` (${commitLink} on \`${report.branch}\`` : ` (`) + `${commitLink ? ", " : ""}pinned as \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
705
706
|
);
|
|
706
707
|
lines.push("");
|
|
707
708
|
lines.push("| Model | Status | Score | Run |");
|
|
@@ -721,19 +722,29 @@ function buildMarkdown(report, minScore) {
|
|
|
721
722
|
reward: sample.reward
|
|
722
723
|
}))
|
|
723
724
|
);
|
|
724
|
-
if (taskRows.length > 0
|
|
725
|
+
if (taskRows.length > 0) {
|
|
726
|
+
const fold = taskRows.length > 12;
|
|
725
727
|
lines.push("");
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
728
|
+
if (fold) {
|
|
729
|
+
lines.push(
|
|
730
|
+
`<details><summary>Per-scenario rewards (${taskRows.length})</summary>`
|
|
731
|
+
);
|
|
732
|
+
lines.push("");
|
|
733
|
+
} else {
|
|
734
|
+
lines.push(`**Scenarios (${taskRows.length})**`);
|
|
735
|
+
lines.push("");
|
|
736
|
+
}
|
|
737
|
+
lines.push("| Scenario | Model | Reward |");
|
|
729
738
|
lines.push("| --- | --- | ---: |");
|
|
730
739
|
for (const row of taskRows) {
|
|
731
740
|
lines.push(
|
|
732
741
|
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
733
742
|
);
|
|
734
743
|
}
|
|
735
|
-
|
|
736
|
-
|
|
744
|
+
if (fold) {
|
|
745
|
+
lines.push("");
|
|
746
|
+
lines.push("</details>");
|
|
747
|
+
}
|
|
737
748
|
}
|
|
738
749
|
return lines.join("\n");
|
|
739
750
|
}
|
|
@@ -749,9 +760,24 @@ async function validateWorld(dir, opts) {
|
|
|
749
760
|
"No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
|
|
750
761
|
);
|
|
751
762
|
progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
|
|
763
|
+
let tipVersionId = null;
|
|
764
|
+
if (opts.slug) {
|
|
765
|
+
try {
|
|
766
|
+
const tip = await apiRequest(
|
|
767
|
+
options,
|
|
768
|
+
"POST",
|
|
769
|
+
"/api/public/benchmark-containers/resolve-version",
|
|
770
|
+
{ slug: opts.slug, ref: branch }
|
|
771
|
+
);
|
|
772
|
+
tipVersionId = tip.versionId ?? null;
|
|
773
|
+
} catch {
|
|
774
|
+
tipVersionId = null;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
752
777
|
const pushed = await push(dir, {
|
|
753
778
|
slug: opts.slug,
|
|
754
779
|
branch,
|
|
780
|
+
baseVersionId: tipVersionId,
|
|
755
781
|
message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
|
|
756
782
|
...provenance.actor?.name ? { author: provenance.actor.name } : {},
|
|
757
783
|
options
|
|
@@ -832,7 +858,13 @@ async function validateWorld(dir, opts) {
|
|
|
832
858
|
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
833
859
|
runs
|
|
834
860
|
};
|
|
835
|
-
return {
|
|
861
|
+
return {
|
|
862
|
+
...base,
|
|
863
|
+
markdown: buildMarkdown(base, minScore, {
|
|
864
|
+
repoUrl: provenance.repoUrl,
|
|
865
|
+
commit: provenance.commit
|
|
866
|
+
})
|
|
867
|
+
};
|
|
836
868
|
}
|
|
837
869
|
var init_validate = __esm({
|
|
838
870
|
"src/worlds/validate.ts"() {
|