@withgateway/sdk 0.1.18 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +42 -10
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +42 -10
- package/dist/cli.mjs.map +1 -1
- package/dist/index.js +43 -11
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +43 -11
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +5 -0
- package/dist/worlds/index.d.ts +5 -0
- package/dist/worlds/index.js +42 -10
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +42 -10
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -513,7 +513,7 @@ async function push(dir, opts = {}) {
|
|
|
513
513
|
if (!fs.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
514
514
|
const localMeta = readLocalMeta(dir) ?? {};
|
|
515
515
|
const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
|
|
516
|
-
const baseVersionId = localMeta.versionId ?? null;
|
|
516
|
+
const baseVersionId = opts.baseVersionId ?? localMeta.versionId ?? null;
|
|
517
517
|
const contentHash = computeContentHash(dir);
|
|
518
518
|
const pyproject = readPyproject(dir);
|
|
519
519
|
const slug = opts.slug ?? pyproject.name ?? path.basename(path.resolve(dir));
|
|
@@ -697,14 +697,15 @@ var init_hub = __esm({
|
|
|
697
697
|
function fmtScore(score) {
|
|
698
698
|
return score === null ? "no score" : score.toFixed(3);
|
|
699
699
|
}
|
|
700
|
-
function buildMarkdown(report, minScore) {
|
|
700
|
+
function buildMarkdown(report, minScore, source) {
|
|
701
701
|
const lines = [];
|
|
702
702
|
lines.push(
|
|
703
703
|
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
704
704
|
);
|
|
705
705
|
lines.push("");
|
|
706
|
+
const commitLink = source?.repoUrl && source?.commit ? `[\`${source.commit.slice(0, 12)}\`](${source.repoUrl}/commit/${source.commit})` : null;
|
|
706
707
|
lines.push(
|
|
707
|
-
`Ran the pull request's own world source (
|
|
708
|
+
`Ran the pull request's own world source` + (commitLink ? ` (${commitLink} on \`${report.branch}\`` : ` (`) + `${commitLink ? ", " : ""}pinned as \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
708
709
|
);
|
|
709
710
|
lines.push("");
|
|
710
711
|
lines.push("| Model | Status | Score | Run |");
|
|
@@ -724,19 +725,29 @@ function buildMarkdown(report, minScore) {
|
|
|
724
725
|
reward: sample.reward
|
|
725
726
|
}))
|
|
726
727
|
);
|
|
727
|
-
if (taskRows.length > 0
|
|
728
|
+
if (taskRows.length > 0) {
|
|
729
|
+
const fold = taskRows.length > 12;
|
|
728
730
|
lines.push("");
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
731
|
+
if (fold) {
|
|
732
|
+
lines.push(
|
|
733
|
+
`<details><summary>Per-scenario rewards (${taskRows.length})</summary>`
|
|
734
|
+
);
|
|
735
|
+
lines.push("");
|
|
736
|
+
} else {
|
|
737
|
+
lines.push(`**Scenarios (${taskRows.length})**`);
|
|
738
|
+
lines.push("");
|
|
739
|
+
}
|
|
740
|
+
lines.push("| Scenario | Model | Reward |");
|
|
732
741
|
lines.push("| --- | --- | ---: |");
|
|
733
742
|
for (const row of taskRows) {
|
|
734
743
|
lines.push(
|
|
735
744
|
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
736
745
|
);
|
|
737
746
|
}
|
|
738
|
-
|
|
739
|
-
|
|
747
|
+
if (fold) {
|
|
748
|
+
lines.push("");
|
|
749
|
+
lines.push("</details>");
|
|
750
|
+
}
|
|
740
751
|
}
|
|
741
752
|
return lines.join("\n");
|
|
742
753
|
}
|
|
@@ -752,9 +763,24 @@ async function validateWorld(dir, opts) {
|
|
|
752
763
|
"No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
|
|
753
764
|
);
|
|
754
765
|
progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
|
|
766
|
+
let tipVersionId = null;
|
|
767
|
+
if (opts.slug) {
|
|
768
|
+
try {
|
|
769
|
+
const tip = await apiRequest(
|
|
770
|
+
options,
|
|
771
|
+
"POST",
|
|
772
|
+
"/api/public/benchmark-containers/resolve-version",
|
|
773
|
+
{ slug: opts.slug, ref: branch }
|
|
774
|
+
);
|
|
775
|
+
tipVersionId = tip.versionId ?? null;
|
|
776
|
+
} catch {
|
|
777
|
+
tipVersionId = null;
|
|
778
|
+
}
|
|
779
|
+
}
|
|
755
780
|
const pushed = await push(dir, {
|
|
756
781
|
slug: opts.slug,
|
|
757
782
|
branch,
|
|
783
|
+
baseVersionId: tipVersionId,
|
|
758
784
|
message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
|
|
759
785
|
...provenance.actor?.name ? { author: provenance.actor.name } : {},
|
|
760
786
|
options
|
|
@@ -835,7 +861,13 @@ async function validateWorld(dir, opts) {
|
|
|
835
861
|
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
836
862
|
runs
|
|
837
863
|
};
|
|
838
|
-
return {
|
|
864
|
+
return {
|
|
865
|
+
...base,
|
|
866
|
+
markdown: buildMarkdown(base, minScore, {
|
|
867
|
+
repoUrl: provenance.repoUrl,
|
|
868
|
+
commit: provenance.commit
|
|
869
|
+
})
|
|
870
|
+
};
|
|
839
871
|
}
|
|
840
872
|
var init_validate = __esm({
|
|
841
873
|
"src/worlds/validate.ts"() {
|
|
@@ -1111,7 +1143,7 @@ var require_package = __commonJS({
|
|
|
1111
1143
|
"package.json"(exports2, module2) {
|
|
1112
1144
|
module2.exports = {
|
|
1113
1145
|
name: "@withgateway/sdk",
|
|
1114
|
-
version: "0.1.
|
|
1146
|
+
version: "0.1.20",
|
|
1115
1147
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
1116
1148
|
main: "./dist/index.js",
|
|
1117
1149
|
module: "./dist/index.mjs",
|