@withgateway/sdk 0.1.18 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +42 -10
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +42 -10
- package/dist/cli.mjs.map +1 -1
- package/dist/index.js +43 -11
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +43 -11
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +5 -0
- package/dist/worlds/index.d.ts +5 -0
- package/dist/worlds/index.js +42 -10
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +42 -10
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -503,7 +503,7 @@ async function push(dir, opts = {}) {
|
|
|
503
503
|
if (!fs.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
504
504
|
const localMeta = readLocalMeta(dir) ?? {};
|
|
505
505
|
const branch = opts.branch ?? localMeta.branch ?? DEFAULT_BRANCH;
|
|
506
|
-
const baseVersionId = localMeta.versionId ?? null;
|
|
506
|
+
const baseVersionId = opts.baseVersionId ?? localMeta.versionId ?? null;
|
|
507
507
|
const contentHash = computeContentHash(dir);
|
|
508
508
|
const pyproject = readPyproject(dir);
|
|
509
509
|
const slug = opts.slug ?? pyproject.name ?? path.basename(path.resolve(dir));
|
|
@@ -683,14 +683,15 @@ var init_hub = __esm({
|
|
|
683
683
|
function fmtScore(score) {
|
|
684
684
|
return score === null ? "no score" : score.toFixed(3);
|
|
685
685
|
}
|
|
686
|
-
function buildMarkdown(report, minScore) {
|
|
686
|
+
function buildMarkdown(report, minScore, source) {
|
|
687
687
|
const lines = [];
|
|
688
688
|
lines.push(
|
|
689
689
|
`### World validation ${report.ok ? "passed" : "failed"} \u2014 \`${report.slug}@${report.branch}\``
|
|
690
690
|
);
|
|
691
691
|
lines.push("");
|
|
692
|
+
const commitLink = source?.repoUrl && source?.commit ? `[\`${source.commit.slice(0, 12)}\`](${source.repoUrl}/commit/${source.commit})` : null;
|
|
692
693
|
lines.push(
|
|
693
|
-
`Ran the pull request's own world source (
|
|
694
|
+
`Ran the pull request's own world source` + (commitLink ? ` (${commitLink} on \`${report.branch}\`` : ` (`) + `${commitLink ? ", " : ""}pinned as \`${report.contentHash.slice(0, 12)}\`) through run config **${report.runConfig.name}**` + (minScore !== null ? `, gating at score \u2265 ${minScore}.` : ".")
|
|
694
695
|
);
|
|
695
696
|
lines.push("");
|
|
696
697
|
lines.push("| Model | Status | Score | Run |");
|
|
@@ -710,19 +711,29 @@ function buildMarkdown(report, minScore) {
|
|
|
710
711
|
reward: sample.reward
|
|
711
712
|
}))
|
|
712
713
|
);
|
|
713
|
-
if (taskRows.length > 0
|
|
714
|
+
if (taskRows.length > 0) {
|
|
715
|
+
const fold = taskRows.length > 12;
|
|
714
716
|
lines.push("");
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
717
|
+
if (fold) {
|
|
718
|
+
lines.push(
|
|
719
|
+
`<details><summary>Per-scenario rewards (${taskRows.length})</summary>`
|
|
720
|
+
);
|
|
721
|
+
lines.push("");
|
|
722
|
+
} else {
|
|
723
|
+
lines.push(`**Scenarios (${taskRows.length})**`);
|
|
724
|
+
lines.push("");
|
|
725
|
+
}
|
|
726
|
+
lines.push("| Scenario | Model | Reward |");
|
|
718
727
|
lines.push("| --- | --- | ---: |");
|
|
719
728
|
for (const row of taskRows) {
|
|
720
729
|
lines.push(
|
|
721
730
|
`| ${row.task} | \`${row.model}\` | ${row.reward === null ? "no score" : row.reward.toFixed(2)} |`
|
|
722
731
|
);
|
|
723
732
|
}
|
|
724
|
-
|
|
725
|
-
|
|
733
|
+
if (fold) {
|
|
734
|
+
lines.push("");
|
|
735
|
+
lines.push("</details>");
|
|
736
|
+
}
|
|
726
737
|
}
|
|
727
738
|
return lines.join("\n");
|
|
728
739
|
}
|
|
@@ -738,9 +749,24 @@ async function validateWorld(dir, opts) {
|
|
|
738
749
|
"No branch to validate on: pass --branch or run inside a git checkout / GitHub Actions."
|
|
739
750
|
);
|
|
740
751
|
progress(`pushing ${dir} as ${opts.slug ?? "(pyproject name)"}@${branch}\u2026`);
|
|
752
|
+
let tipVersionId = null;
|
|
753
|
+
if (opts.slug) {
|
|
754
|
+
try {
|
|
755
|
+
const tip = await apiRequest(
|
|
756
|
+
options,
|
|
757
|
+
"POST",
|
|
758
|
+
"/api/public/benchmark-containers/resolve-version",
|
|
759
|
+
{ slug: opts.slug, ref: branch }
|
|
760
|
+
);
|
|
761
|
+
tipVersionId = tip.versionId ?? null;
|
|
762
|
+
} catch {
|
|
763
|
+
tipVersionId = null;
|
|
764
|
+
}
|
|
765
|
+
}
|
|
741
766
|
const pushed = await push(dir, {
|
|
742
767
|
slug: opts.slug,
|
|
743
768
|
branch,
|
|
769
|
+
baseVersionId: tipVersionId,
|
|
744
770
|
message: `PR validation push (${provenance.commit?.slice(0, 12) ?? "local"})`,
|
|
745
771
|
...provenance.actor?.name ? { author: provenance.actor.name } : {},
|
|
746
772
|
options
|
|
@@ -821,7 +847,13 @@ async function validateWorld(dir, opts) {
|
|
|
821
847
|
runConfig: { id: runConfig.id, name: runConfig.name },
|
|
822
848
|
runs
|
|
823
849
|
};
|
|
824
|
-
return {
|
|
850
|
+
return {
|
|
851
|
+
...base,
|
|
852
|
+
markdown: buildMarkdown(base, minScore, {
|
|
853
|
+
repoUrl: provenance.repoUrl,
|
|
854
|
+
commit: provenance.commit
|
|
855
|
+
})
|
|
856
|
+
};
|
|
825
857
|
}
|
|
826
858
|
var init_validate = __esm({
|
|
827
859
|
"src/worlds/validate.ts"() {
|
|
@@ -1097,7 +1129,7 @@ var require_package = __commonJS({
|
|
|
1097
1129
|
"package.json"(exports, module) {
|
|
1098
1130
|
module.exports = {
|
|
1099
1131
|
name: "@withgateway/sdk",
|
|
1100
|
-
version: "0.1.
|
|
1132
|
+
version: "0.1.20",
|
|
1101
1133
|
description: "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
1102
1134
|
main: "./dist/index.js",
|
|
1103
1135
|
module: "./dist/index.mjs",
|