modelrig 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +118 -0
- package/README.md +86 -0
- package/dist/bakeoff/disagreement.d.ts +93 -0
- package/dist/bakeoff/disagreement.d.ts.map +1 -0
- package/dist/bakeoff/disagreement.js +140 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +58 -2
- package/dist/bakeoff/pairwise.d.ts +102 -0
- package/dist/bakeoff/pairwise.d.ts.map +1 -0
- package/dist/bakeoff/pairwise.js +214 -0
- package/dist/bakeoff/replay.d.ts +6 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +6 -2
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +46 -0
- package/dist/bakeoff/types.d.ts +24 -0
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +9 -9
- package/dist/mask/apply.d.ts +80 -0
- package/dist/mask/apply.d.ts.map +1 -0
- package/dist/mask/apply.js +228 -0
- package/dist/mask/detect.d.ts +38 -0
- package/dist/mask/detect.d.ts.map +1 -0
- package/dist/mask/detect.js +167 -0
- package/dist/mask/index.d.ts +21 -0
- package/dist/mask/index.d.ts.map +1 -0
- package/dist/mask/index.js +54 -0
- package/dist/mask/mask-policy-cli.d.ts +11 -0
- package/dist/mask/mask-policy-cli.d.ts.map +1 -0
- package/dist/mask/mask-policy-cli.js +33 -0
- package/dist/mask/policy.d.ts +38 -0
- package/dist/mask/policy.d.ts.map +1 -0
- package/dist/mask/policy.js +187 -0
- package/dist/mask/proof.d.ts +54 -0
- package/dist/mask/proof.d.ts.map +1 -0
- package/dist/mask/proof.js +120 -0
- package/dist/mask/receipt.d.ts +30 -0
- package/dist/mask/receipt.d.ts.map +1 -0
- package/dist/mask/receipt.js +49 -0
- package/dist/mask/seam.d.ts +53 -0
- package/dist/mask/seam.d.ts.map +1 -0
- package/dist/mask/seam.js +120 -0
- package/dist/mask/types.d.ts +81 -0
- package/dist/mask/types.d.ts.map +1 -0
- package/dist/mask/types.js +10 -0
- package/dist/mask/vault.d.ts +50 -0
- package/dist/mask/vault.d.ts.map +1 -0
- package/dist/mask/vault.js +81 -0
- package/dist/receipts/types.d.ts +65 -0
- package/dist/receipts/types.d.ts.map +1 -0
- package/dist/receipts/types.js +134 -0
- package/dist/routes/loader-warnings.d.ts +8 -0
- package/dist/routes/loader-warnings.d.ts.map +1 -1
- package/dist/routes/loader-warnings.js +23 -0
- package/dist/routes/loader.d.ts +1 -1
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +24 -25
- package/dist/routes/mask-loader.d.ts +18 -0
- package/dist/routes/mask-loader.d.ts.map +1 -0
- package/dist/routes/mask-loader.js +39 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +51 -17
- package/dist/routes/screen-loader.d.ts +23 -0
- package/dist/routes/screen-loader.d.ts.map +1 -0
- package/dist/routes/screen-loader.js +79 -0
- package/dist/routes/template.d.ts +10 -0
- package/dist/routes/template.d.ts.map +1 -1
- package/dist/routes/template.js +10 -3
- package/dist/run/accept.d.ts +4 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +2 -0
- package/dist/run/dispatch.d.ts +35 -0
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +33 -0
- package/dist/run/grounding.d.ts +47 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +113 -6
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +6 -1
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +12 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +19 -16
- package/dist/screen/provenance.d.ts +59 -0
- package/dist/screen/provenance.d.ts.map +1 -0
- package/dist/screen/provenance.js +104 -0
- package/dist/screen/types.d.ts +71 -0
- package/dist/screen/types.d.ts.map +1 -0
- package/dist/screen/types.js +9 -0
- package/dist/status-cli.d.ts +11 -0
- package/dist/status-cli.d.ts.map +1 -0
- package/dist/status-cli.js +21 -0
- package/dist/status-runs.d.ts +38 -0
- package/dist/status-runs.d.ts.map +1 -0
- package/dist/status-runs.js +80 -0
- package/dist/status.d.ts +7 -2
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +11 -2
- package/dist/telemetry/ingest-protocol.d.ts +8 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +11 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +12 -3
- package/dist/types/provider.d.ts +11 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +9 -0
- package/dist/types/route.d.ts +55 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +7 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +13 -0
- package/dist/verify/types.d.ts +6 -0
- package/dist/verify/types.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +131 -0
- package/docs/how-it-fits.md +34 -0
- package/docs/instrumentation.md +221 -0
- package/docs/llms.txt +276 -0
- package/docs/migration-playbook.md +512 -0
- package/docs/quickstart.md +339 -0
- package/package.json +18 -2
- package/receipts.d.ts +12 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,kBAAkB,CAAC;AAEvB,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;wCAEoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;kFAM8E;IAC9E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;wFACoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;uDACmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;wEACoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;oDAGgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;oFACgF;IAChF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C;;6DAEyD;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;oDACgD;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;uDACmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;;;;+EAM2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;mEAC+D;IAC/D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;sDAGkD;IAClD,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;;;uDAKmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC;;;;uEAImE;IACnE,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC;CACvC;AAED;;6EAE6E;AAC7E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;gDAE4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;cAIc;AACd,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;IAClE;;;6EAGyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;CAClC;AAED;;;gDAGgD;AAChD,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,cAAc,GAAG,UAAU,CAAC;IACjD,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED;;mFAEmF;AACnF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;kEAG8D;IAC9D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC;CAC5C;AAED,4DAA4D;AAC5D,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;mDAC+C;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,eAAe,GAAG,aAAa,GAAG,OAAO,CAAC;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,6EAA6E;AAC7E,MAAM,MAAM,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,QAAQ,CAAC;AAE/E;;+EAE+E;AAC/E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,GAAG,EAAE,MAAM,OAAO,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC;IAC7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9C,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC;IAC1D,KAAK,IAAI,IAAI,CAAC;CACf"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;
|
|
1
|
+
{"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAUH,wBAAgB,cAAc,IAAI,MAAM,CAwEvC"}
|
package/dist/validate-cli.js
CHANGED
|
@@ -17,6 +17,7 @@ const rawonly_1 = require("./run/rawonly");
|
|
|
17
17
|
const loader_1 = require("./routes/loader");
|
|
18
18
|
const manifest_1 = require("./routes/manifest");
|
|
19
19
|
const attempt_meta_1 = require("./telemetry/attempt-meta");
|
|
20
|
+
const status_runs_1 = require("./status-runs");
|
|
20
21
|
function runValidateCli() {
|
|
21
22
|
const config = (0, config_1.loadConfigFromEnv)();
|
|
22
23
|
const routesDir = (0, rawonly_1.requireRoutesDir)(config, "modelrig validate");
|
|
@@ -66,6 +67,18 @@ function runValidateCli() {
|
|
|
66
67
|
console.error(`FAIL ${r.route ?? r.file}: ${r.problems.join("; ")}`);
|
|
67
68
|
}
|
|
68
69
|
}
|
|
70
|
+
// GS-1 teaching hint: a rig that is USED (has recorded inferences) but never
|
|
71
|
+
// wrapped an execution in run.start / run.scope records no runs, so /runs stays
|
|
72
|
+
// empty and /projects shows "(no run)". A WARN, never a failure — the config is
|
|
73
|
+
// valid; runs are the standard path (get-started v5). Best-effort + silent when
|
|
74
|
+
// there is no local buffer to read yet.
|
|
75
|
+
const adoption = (0, status_runs_1.readRunAdoption)(config.telemetryDbPath);
|
|
76
|
+
if (adoption !== null && adoption.inferences > 0 && adoption.runs === 0) {
|
|
77
|
+
console.warn(`WARN this rig has recorded ${adoption.inferences} call${adoption.inferences === 1 ? "" : "s"} ` +
|
|
78
|
+
"but has never started a run — wrap each pipeline execution in run.start()/run.scope() " +
|
|
79
|
+
"(one step() per model-call family, artifact.save for work products) so it appears on /runs. " +
|
|
80
|
+
"Runs are the standard path: https://modelrig.dev/instrumentation.html");
|
|
81
|
+
}
|
|
69
82
|
if (failures === 0) {
|
|
70
83
|
console.log(`\n${results.length} route(s) valid — no provider key required for this check.`);
|
|
71
84
|
return 0;
|
package/dist/verify/types.d.ts
CHANGED
|
@@ -75,6 +75,12 @@ export interface SampleReport extends SampleDiff {
|
|
|
75
75
|
export interface EquivalenceReport {
|
|
76
76
|
readonly schemaVersion: 1;
|
|
77
77
|
readonly kind: "shadow-equivalence";
|
|
78
|
+
/** What change this equivalence run proves, when it is not a plain model swap
|
|
79
|
+
* (the default, absent). `screen-assembly` marks a SC-0 B3 run proving an
|
|
80
|
+
* assembly change — grounding `legacy`→`spotlight`, or an `x-trust: untrusted`
|
|
81
|
+
* variable flip — is output-equivalent to the prior assembly on the pinned
|
|
82
|
+
* model (spec §5), so the flip ships through the bake-off like a swap. Additive. */
|
|
83
|
+
readonly subject?: "swap" | "mask-policy" | "screen-assembly";
|
|
78
84
|
readonly callSiteId: string;
|
|
79
85
|
readonly route: string;
|
|
80
86
|
readonly routeVersion: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/verify/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;;6BAE6B;AAC7B,MAAM,WAAW,aAAa;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;oEACoE;AACpE,MAAM,WAAW,oBAAoB;IACnC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;mEAC+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAC3C,4EAA4E;IAC5E,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C;;;sDAGkD;IAClD,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpD;AAED,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GACrB,YAAY,GACZ,kCAAkC,GAClC,WAAW,CAAC;AAEhB;kDACkD;AAClD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3D,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC;;wEAEoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;4EACwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;+CAC2C;IAC3C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;QAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;sEACkE;IAClE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/verify/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;;6BAE6B;AAC7B,MAAM,WAAW,aAAa;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;oEACoE;AACpE,MAAM,WAAW,oBAAoB;IACnC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;mEAC+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAC3C,4EAA4E;IAC5E,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C;;;sDAGkD;IAClD,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpD;AAED,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GACrB,YAAY,GACZ,kCAAkC,GAClC,WAAW,CAAC;AAEhB;kDACkD;AAClD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3D,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC;;wEAEoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;4EACwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;IACpC;;;;wFAIoF;IACpF,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,iBAAiB,CAAC;IAC9D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;+CAC2C;IAC3C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;QAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;sEACkE;IAClE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: modelrig
|
|
3
|
+
description: Use when working with LLM/model selection, cost, or reliability in a codebase — choosing a model, migrating direct provider SDK calls to routes, explaining what a model call actually costs, or debugging schema/grounding/caching failures. Answers from PROBED measurements rather than provider claims.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# ModelRig
|
|
7
|
+
|
|
8
|
+
**A pointer skill.** The canonical playbook lives at
|
|
9
|
+
<https://modelrig.dev/llms.txt> and is regenerated whenever the registry
|
|
10
|
+
is rebuilt. Fetch it rather than trusting anything cached here — model
|
|
11
|
+
capabilities change, and a stale answer about a model is worse than no answer.
|
|
12
|
+
|
|
13
|
+
One copy, every ecosystem: this file is the same in Claude Code, Cursor,
|
|
14
|
+
Codex, and anywhere else skills load. It deliberately holds almost no facts.
|
|
15
|
+
|
|
16
|
+
## Start here
|
|
17
|
+
|
|
18
|
+
1. **Fetch <https://modelrig.dev/llms.txt>.** It carries the current probed
|
|
19
|
+
facts, the model list with `as_of` stamps, and the tool advertisements.
|
|
20
|
+
2. **Prefer the MCP tools over parsing text.** If the ModelRig oracle is
|
|
21
|
+
registered, call it — the answers are typed, dated, and scoped:
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
query_registry which models satisfy a capability + price bound
|
|
25
|
+
get_leaderboard standings by effective cost of conformance
|
|
26
|
+
explain_pricing list price vs what conformance actually costs
|
|
27
|
+
get_call_notes per-model quirks that make requests fail
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Not registered? Add it — it needs no account for those four:
|
|
31
|
+
|
|
32
|
+
```json
|
|
33
|
+
{ "mcpServers": { "modelrig": { "command": "npx", "args": ["modelrig-oracle"] } } }
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
3. **Fall back to the raw registry** when neither is available:
|
|
37
|
+
<https://raw.githubusercontent.com/modelrig/modelrig/main/registry/registry.json>
|
|
38
|
+
|
|
39
|
+
## Rules that matter more than speed
|
|
40
|
+
|
|
41
|
+
- **Never state a model's capabilities from memory.** Look them up. Every
|
|
42
|
+
fact in the registry carries an `as_of` date and a sample count; quote both.
|
|
43
|
+
- **Absence is not a limit.** A model missing from an answer is one nobody has
|
|
44
|
+
probed — say that, rather than implying it can't do the thing.
|
|
45
|
+
- **Probed beats declared.** Where a provider's claim and our measurement
|
|
46
|
+
disagree, the entry's `discrepancies` field says so. Surface the
|
|
47
|
+
disagreement; it is usually the useful part.
|
|
48
|
+
- **Compare on effective cost of conformance,** not list price. A model at half
|
|
49
|
+
the price that conforms two-thirds as often is not cheaper.
|
|
50
|
+
|
|
51
|
+
## Just want to see your runs first? (observe, no migration)
|
|
52
|
+
|
|
53
|
+
Before migrating anything, you can just **watch** an existing pipeline. ModelRig
|
|
54
|
+
records each run as a run → step → artifact chain you click through in the
|
|
55
|
+
console — no routes moved, no model calls rerouted, and with the flag off the
|
|
56
|
+
pipeline behaves byte-for-byte as before. Scaffold it:
|
|
57
|
+
`npx modelrig observe [path] --scope <one pipeline>` writes a self-contained
|
|
58
|
+
`record-step` seam and an `OBSERVE.md` whose wiring is emitted as diffs you
|
|
59
|
+
apply. The instrumentation is a code-level SDK seam (no zero-code observe), but
|
|
60
|
+
you no longer hand-write it — see <https://modelrig.dev/instrumentation.html>.
|
|
61
|
+
The namespace is **on by default once a control plane is configured** (a
|
|
62
|
+
`rig_sk_` API key, or the self-host Supabase pair); a pure local-only rig stays
|
|
63
|
+
off — force it on there with `MODELRIG_ARTIFACTS=1`, and `MODELRIG_ARTIFACTS=0`
|
|
64
|
+
opts out anywhere. Observing the runs in the console needs a free
|
|
65
|
+
account + key from <https://app.modelrig.ai> — the one step a human has to do
|
|
66
|
+
(console login to mint the key), so plan the handoff rather than stalling on it.
|
|
67
|
+
|
|
68
|
+
## Migrating a repo — the T0–T2 playbook
|
|
69
|
+
|
|
70
|
+
When you import this skill to migrate a codebase, **fetch the canonical
|
|
71
|
+
playbook** and follow it: <https://modelrig.dev/migration-playbook.html>. It is
|
|
72
|
+
a four-tier autonomy ladder — T0 gateway-pointing and T1 instrumentation are
|
|
73
|
+
behaviour-preserving (apply after a plan review); T2 route extraction replaces a
|
|
74
|
+
direct provider call with `rig.run()` **always same-model**, and is
|
|
75
|
+
behaviour-affecting, so the agent *proposes* it, proves it with
|
|
76
|
+
`modelrig verify-swap` (shadow-equivalence), embeds the report in the PR, does
|
|
77
|
+
**one call site per PR**, and a **human merges**. Model optimization (T3) is
|
|
78
|
+
never the skill's job.
|
|
79
|
+
|
|
80
|
+
**Runs are the standard path, not an appendix.** Every pipeline execution should
|
|
81
|
+
be a **run**: wrap it in `rig.artifacts.run.start` / `run.scope`, name one step
|
|
82
|
+
per model-call family (pass `stepKey`), `artifact.save` the work products, and tag
|
|
83
|
+
`subject` / `feature`. Inside a run context the SDK auto-stamps `tags.run_id` for
|
|
84
|
+
you, so the run fills `/runs` **and** `/projects`; acceptance is the run appearing
|
|
85
|
+
on `/runs` with its steps. (Inside a run context the SDK also auto-stamps
|
|
86
|
+
`tags.step`; a value you pass always wins, so name the step with `stepKey` or
|
|
87
|
+
`tags.step` when you want a specific label.) This is Tier 1 and it is **not
|
|
88
|
+
optional for a raw-lane (Lane-B) seam** — a pipeline that reaches the model
|
|
89
|
+
through `rig.runRaw` without a `run.start` around it records inferences but no
|
|
90
|
+
run. The full API is the reference at
|
|
91
|
+
<https://modelrig.dev/instrumentation.html>.
|
|
92
|
+
|
|
93
|
+
The machinery underneath: `npx modelrig init [path] [--scope <glob>]` scans for
|
|
94
|
+
provider call sites and drafts a route bundle for each, plus a `MIGRATION.md`
|
|
95
|
+
mapping call site → route → the replacement `rig.run` line. It writes only under
|
|
96
|
+
`modelrig/routes/drafts/` and **never edits source** — applying the diff is the
|
|
97
|
+
human's decision, one call site at a time. The replaced call is **deleted** (not
|
|
98
|
+
commented out) with a one-line breadcrumb; rollback is `git revert`.
|
|
99
|
+
|
|
100
|
+
## Capabilities beyond migration (the full surface)
|
|
101
|
+
|
|
102
|
+
Shipped features a newbie migration won't stumble onto — one line each, with the
|
|
103
|
+
reference doc. The canonical detail is always <https://modelrig.dev/llms.txt>.
|
|
104
|
+
|
|
105
|
+
- **Runs & projects** — the run → step → artifact graph at `/runs`, grouped by tag
|
|
106
|
+
at `/projects`: <https://modelrig.dev/instrumentation.html>.
|
|
107
|
+
- **Tenants & statements** — a `subject` tag or project-scoped key becomes a
|
|
108
|
+
per-customer tenant with a derived statement: <https://modelrig.dev/tenants.html>.
|
|
109
|
+
- **Published receipts** — a signed, login-free receipt of a bake-off or route at
|
|
110
|
+
`/r/<id>`, verifiable at `/v1/verify/<id>`, content never leaves:
|
|
111
|
+
<https://modelrig.dev/receipts.html>.
|
|
112
|
+
- **Analytics query API** — self-describing, read-only, metadata-only SELECT over
|
|
113
|
+
your own telemetry (`GET /v1/analytics/meta`, `POST /v1/analytics/query`):
|
|
114
|
+
<https://modelrig.dev/optimization-loop.html>.
|
|
115
|
+
- **Provenance & trust** — provenance-first prompt assembly plus per-variable
|
|
116
|
+
`x-trust`: <https://modelrig.dev/screening.html>.
|
|
117
|
+
- **Bake-offs — pairwise, disagreement & regret** — replay your own traffic, judge
|
|
118
|
+
pairwise with disagreement shown, propose the lowest-regret arm under a quality
|
|
119
|
+
gate: <https://modelrig.dev/bakeoffs.html> and <https://modelrig.dev/run-verdicts.html>.
|
|
120
|
+
|
|
121
|
+
Content custody stays early access (metadata and hashes); no capability here is a
|
|
122
|
+
claim beyond what ships.
|
|
123
|
+
|
|
124
|
+
## What ModelRig is, in three sentences
|
|
125
|
+
|
|
126
|
+
A route is a task your code calls by name; the route's YAML declares which
|
|
127
|
+
models are allowed to serve it, the schema the output must match, and the
|
|
128
|
+
retry and budget policy. Your code calls the task, not the model, so swapping
|
|
129
|
+
models is a config change with evidence behind it rather than a code change
|
|
130
|
+
with hope behind it. Nothing switches what serves a route without a bake-off
|
|
131
|
+
on your own traffic and an explicit human approval.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# How it fits
|
|
2
|
+
|
|
3
|
+
One content source (this file) renders in the console `/setup` page and on modelrig.dev — edit here, surfaces follow.
|
|
4
|
+
|
|
5
|
+
ModelRig sits between your code and the model providers. Your repo declares **routes** — small YAML files naming a prompt, an output schema, and the models allowed to serve the task. Your app calls `rig.run("task.name")`; ModelRig renders the prompt, picks a model from the route's candidates using verified registry facts, enforces the schema, retries typed failures within per-class budgets, and hard-stops spending at your envelope caps. How the candidate ladder actually runs — ordering, validate → repair → fall-through, and what is not built — is on [Routing & reliability](routing-reliability.md).
|
|
6
|
+
|
|
7
|
+
Every call writes one telemetry row to a local SQLite file first — the run never blocks on the cloud. An async exporter mirrors rows to the console you are reading. Provider keys are the one thing that deliberately never leaves your machine; the exporter has no code path to them.
|
|
8
|
+
|
|
9
|
+
Routing is half of it. ModelRig also records the **execution graph** of your workstream — the run, its steps, and the artifacts each step produced — versioned, hashed, and lineaged, into an inspectable store you can open: when someone asks why the pipeline did that, you pull up the run and show them. This is the **standard path**, not an add-on: every pipeline execution is a run — wrap it in `run.start` / `run.scope`, name a step per model-call family (name it with `stepKey`), save the work products, and inside that context the SDK auto-stamps `run_id` for you, so the run appears on `/runs` and `/projects` with no hand-written id (the SDK auto-stamps the `step` key inside a run context too; the `stepKey` you pass names the step, and always wins). The namespace is **on by default once a control plane is configured** (a ModelRig API key, or the self-host Supabase pair); a pure local-only rig stays off, and `MODELRIG_ARTIFACTS=0` opts out anywhere. Artifact *metadata and lineage* ride the same exporter; artifact *content* moves only with per-route consent, off by default. A failing artifact can be promoted to a regression case and replayed as a bake-off — see → prove → save is the loop, and the store is where seeing happens.
|
|
10
|
+
|
|
11
|
+
What is retained beyond that is your choice, per route, and there are two lanes.
|
|
12
|
+
|
|
13
|
+
**Pure router** (capture off). Nothing retained. Real, supported, selectable, and the honest answer for a ZDR-strict or data-residency-bound workload.
|
|
14
|
+
|
|
15
|
+
**Optimization on** (recommended). Your traffic is what makes the routing smarter and the bill smaller. **What is kept:** per-inference telemetry rows, and bounded review samples for the bake-offs you create. **Where:** your organization, row-scoped by the database, not by our application code. **How you leave:** see it all in the console, delete it, export it, take your routes and go — they were always YAML in your git. **What you get back:** community priors on your very first bake-off and RigIndex-grade evidence on your own routes — at the same flat 2% either way; the corpus earns its place by being useful, not by being a discount.
|
|
16
|
+
|
|
17
|
+
Captures power **bake-offs**: replay your own recent traffic through route variants — a cheaper model, a different prompt — and get the effective cost of conformance for each, with confidence intervals. A winning variant is a proposal for you to apply in your YAML; nothing swaps models automatically.
|
|
18
|
+
|
|
19
|
+
The **registry** feeds routing from the side: declared claims, probed measurements, and observed traffic per model, with disagreements shown. A probed fact beats a declared claim — a model that declares schema support but failed our probes is not eligible to serve a schema route natively.
|
|
20
|
+
|
|
21
|
+
The loop, end to end: routes in your repo → `rig.run` in your app → local telemetry (+ captures on the routes where you turn Optimization on) → synced console → bake-offs prove cheaper variants → you edit the YAML. The console shows the truth; actions live in your code.
|
|
22
|
+
|
|
23
|
+
## More capabilities
|
|
24
|
+
|
|
25
|
+
The five-step get-started is the front door; these are the shipped surfaces it does not stop to teach. One line each, with the reference doc:
|
|
26
|
+
|
|
27
|
+
- **Runs & projects** — the run → step → artifact graph on `/runs`, grouped by tag on `/projects`. The standard path above; full API in [instrumentation](instrumentation.md).
|
|
28
|
+
- **Tenants & statements** — a `subject` tag or a project-scoped key becomes a per-customer tenant with a derived statement (attempted, verified, billable, true cost), on `/tenants`. See [Tenants & statements](tenants.md).
|
|
29
|
+
- **Published receipts** — publish a signed, login-free receipt of a bake-off or route at `/r/<id>` — counts, scores and hashes only, verifiable at `/v1/verify/<id>`; names aliased, content never leaves. See [Published receipts](receipts.md).
|
|
30
|
+
- **Analytics query API** — a self-describing, read-only query surface over your own telemetry (`GET /v1/analytics/meta`, `POST /v1/analytics/query`): an allowlisted SELECT grammar that returns metadata only, never content. See [Optimization loop](optimization-loop.md).
|
|
31
|
+
- **Provenance & trust** — provenance-first prompt assembly keeps third-party content out of the most-trusted slot; annotate a variable `x-trust: untrusted` to spotlight it. See [Provenance & trust](screening.md).
|
|
32
|
+
- **Bake-offs — pairwise, disagreement & regret** — replay your own traffic, judge candidates pairwise with disagreement shown, and propose the lowest-**regret** arm under a quality gate (near-optimal, not merely cheapest). See [Bake-offs & replay](bakeoffs.md) and [Run verdicts](run-verdicts.md).
|
|
33
|
+
|
|
34
|
+
New claims are held to the same bar as the five steps: content custody stays early access (metadata and hashes), and pricing stays the launch-terms sentence.
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
# Artifact instrumentation (runs, steps, artifacts)
|
|
2
|
+
|
|
3
|
+
ModelRig can record the **execution graph** of any pipeline your code runs — the
|
|
4
|
+
run, its steps, and the artifacts each step produced (the prompt, the raw model
|
|
5
|
+
text, the parsed output, the evidence it grounded on) — with lineage,
|
|
6
|
+
evaluations, and an integrity hash for each. The console then renders that run as
|
|
7
|
+
a semantic chain you click through: *what your pipeline actually did*.
|
|
8
|
+
|
|
9
|
+
It is **on by default when a control plane is configured** (an api-mode
|
|
10
|
+
`rig_sk_` key, or the direct-mode Supabase pair) — metadata + hashes only. A
|
|
11
|
+
**pure local-only rig** (no sink to ship to) stays off. Either way it is
|
|
12
|
+
additive; set `MODELRIG_ARTIFACTS=0` to opt out, and with the namespace off it
|
|
13
|
+
is inert: `run.start` is a no-op, `artifact.save` returns null, no rows are
|
|
14
|
+
written, and your pipeline behaves byte-for-byte as before. `modelrig status`
|
|
15
|
+
prints the current posture (on/off + why).
|
|
16
|
+
|
|
17
|
+
> **Metadata + hashes only, this release.** The serialized value you save is
|
|
18
|
+
> hashed (sha256) and the bytes are **discarded** — nothing but the metadata row
|
|
19
|
+
> is persisted. Content custody (the blob path) ships in a later release; the
|
|
20
|
+
> hash proves integrity today. A `zeroRetention` run refuses every artifact
|
|
21
|
+
> fail-closed at the SDK gate.
|
|
22
|
+
|
|
23
|
+
## The namespace toggle
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
# On by default once a control plane is configured — nothing to set.
|
|
27
|
+
export MODELRIG_ARTIFACTS=0 # opt OUT (or force ON with =1 on a
|
|
28
|
+
# local-only rig)
|
|
29
|
+
export MODELRIG_ARTIFACT_STEPS=step_a,step_b # optional: instrument a subset
|
|
30
|
+
# (unset/empty = every instrumented step)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Rows land in the local telemetry buffer and export to the control plane through
|
|
34
|
+
the standard sink (`MODELRIG_API_KEY` + `MODELRIG_INGEST_URL`, or the direct-mode
|
|
35
|
+
Supabase pair). With no sink configured they stay local.
|
|
36
|
+
|
|
37
|
+
An agent can do everything here except mint that `MODELRIG_API_KEY` — creating
|
|
38
|
+
it requires a human console login at <https://app.modelrig.ai>, so an
|
|
39
|
+
agent-driven setup should plan that one handoff rather than stall on it.
|
|
40
|
+
|
|
41
|
+
## Instrument a pipeline
|
|
42
|
+
|
|
43
|
+
The namespace hangs off `rig.artifacts`. Two moving parts: a **run context**
|
|
44
|
+
around the whole pipeline, and a **per-step save** of the work products.
|
|
45
|
+
|
|
46
|
+
```ts
|
|
47
|
+
import { createRig, loadConfigFromEnv } from "modelrig";
|
|
48
|
+
|
|
49
|
+
const rig = createRig(loadConfigFromEnv());
|
|
50
|
+
|
|
51
|
+
// 1. Open a run context around the pipeline. episodeKey bridges feedback;
|
|
52
|
+
// it is a grouping key, not a unique id — a natural choice is your own
|
|
53
|
+
// per-run attempt id.
|
|
54
|
+
const run = rig.artifacts.run.start({
|
|
55
|
+
pipeline: "my-pipeline",
|
|
56
|
+
episodeKey: runAttemptId,
|
|
57
|
+
environment: process.env.NODE_ENV,
|
|
58
|
+
});
|
|
59
|
+
try {
|
|
60
|
+
for (const step of steps) {
|
|
61
|
+
const prompt = assemble(step);
|
|
62
|
+
const result = await callModel(prompt); // your own model call
|
|
63
|
+
|
|
64
|
+
// 2. Save the step's artifacts. stepKey groups them in the console's
|
|
65
|
+
// chain view; task carries the semantic task name.
|
|
66
|
+
const p = rig.artifacts.artifact.save(prompt, { name: `${step}.prompt`, type: "prompt", stepKey: step });
|
|
67
|
+
const raw = rig.artifacts.artifact.save(result.rawText, { name: `${step}.raw`, type: "raw_response", stepKey: step });
|
|
68
|
+
const parsed = rig.artifacts.artifact.save(result.json, {
|
|
69
|
+
name: `${step}.output`, type: "step_output", task: step, stepKey: step, schema: step.schema,
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// 3. Lineage: the parsed output derives from the prompt + raw; this step's
|
|
73
|
+
// prompt consumed the previous step's output.
|
|
74
|
+
if (p && parsed) rig.artifacts.artifact.link(parsed, p, "derived_from");
|
|
75
|
+
if (raw && parsed) rig.artifacts.artifact.link(parsed, raw, "derived_from");
|
|
76
|
+
|
|
77
|
+
// 4. Evaluations (optional, deterministic): flag conformance, degraded
|
|
78
|
+
// provenance, anything you can decide without an LLM judge.
|
|
79
|
+
if (parsed) rig.artifacts.artifact.evaluate(parsed, {
|
|
80
|
+
evaluator: "schema-check", kind: "deterministic", passed: conforms(result.json),
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
run.end("succeeded");
|
|
84
|
+
} catch (err) {
|
|
85
|
+
run.end("failed");
|
|
86
|
+
throw err;
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
For **concurrent** runs in one process, use `rig.artifacts.run.scope(opts, fn)`
|
|
91
|
+
instead of `start`/`end` — each scope keeps its own ambient context so
|
|
92
|
+
overlapping runs never cross-attribute. `run.start` uses `enterWith`, which is
|
|
93
|
+
right for one run at a time on a request/async path.
|
|
94
|
+
|
|
95
|
+
## Steps vs. artifacts
|
|
96
|
+
|
|
97
|
+
- An **artifact** is a work product — a durable thing your pipeline made. It
|
|
98
|
+
always saves (metadata + hash), whether or not the model call went through the
|
|
99
|
+
rig.
|
|
100
|
+
- A **step** is an execution row. `rig.run()` auto-creates a **ground-truth**
|
|
101
|
+
step (its inference is real, so cost and latency are measured) when it runs
|
|
102
|
+
inside a run context. A stage whose model call does *not* go through the rig
|
|
103
|
+
produces artifacts with `step_key` set and **no step row** — which is correct
|
|
104
|
+
and honest. Never route a stage through the rig just to mint a step row; the
|
|
105
|
+
console renders "cost n/a" for un-routed steps rather than inventing a number.
|
|
106
|
+
|
|
107
|
+
## Tags, dimensions, and the graph
|
|
108
|
+
|
|
109
|
+
ModelRig has **two** grouping mechanisms, and they compose:
|
|
110
|
+
|
|
111
|
+
- **Tags** are a flat filter/rollup. Every `rig.run` / `rig.runRaw` takes a `tags`
|
|
112
|
+
object; those key/values ride the telemetry row and drive `/costs` and the route
|
|
113
|
+
dashboard (group cost by `project`, by `environment`, by whatever you tag).
|
|
114
|
+
- **The graph** is the per-run tree at `/runs/[id]` — the ground-truth steps a run
|
|
115
|
+
recorded, in order, with their inferences, costs, and the artifacts attached to
|
|
116
|
+
each step.
|
|
117
|
+
|
|
118
|
+
**Declare your dimensions** in `modelrig/rig.yaml` so the cost views know which
|
|
119
|
+
tags matter:
|
|
120
|
+
|
|
121
|
+
```yaml
|
|
122
|
+
name: my-rig
|
|
123
|
+
dimensions:
|
|
124
|
+
- { key: project, label: Project, required: true }
|
|
125
|
+
- { key: environment, label: Environment, required: false }
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
A `required` dimension that is missing from a run's `tags` is **warned once per
|
|
129
|
+
process** (`required dimension "project" missing from run tags — costs … will be
|
|
130
|
+
under-attributed on /costs`) — on the routed lane and the raw lane alike. It is a
|
|
131
|
+
hygiene warning, never an error: the run proceeds.
|
|
132
|
+
|
|
133
|
+
**Conventional tag keys** (documented, not enforced): `subject` (what a run is
|
|
134
|
+
*about* — a ticker, a client alias), `feature`, `project` (flat grouping),
|
|
135
|
+
`step` (names the step this call belongs to), and `run_id` (bridges a call to a
|
|
136
|
+
run's `episodeKey`). **Inside a `run.start` / `run.scope` context the SDK stamps
|
|
137
|
+
both `run_id` and `step` for you** — `run_id` = the run's
|
|
138
|
+
`episodeKey`, or its id when you gave none; `step` = the current step key (your
|
|
139
|
+
explicit `stepKey`, else `tags.step`, else the auto `route#n` / `provider/model#n`
|
|
140
|
+
key) — on every attempt row of both lanes, unless you passed your own value (yours
|
|
141
|
+
always wins). Outside a run context nothing is stamped, so the row is
|
|
142
|
+
byte-identical. This is why runs are the standard path: a `run.start` alone fills
|
|
143
|
+
both `/runs` (the graph) and `/projects` (the tag tree), with no hand-written id.
|
|
144
|
+
ModelRig owns `lane`, `zdr_enforced`, `rung`, and — on the hosted gateway —
|
|
145
|
+
`key_ownership`; you cannot override those.
|
|
146
|
+
|
|
147
|
+
**How they compose.** Inside a `run.start` context, both `rig.run` and
|
|
148
|
+
`rig.runRaw` record a **ground-truth step** and carry the run's `run_id` tag, so
|
|
149
|
+
the same calls appear on `/runs/[id]` (the graph) **and** under that run on
|
|
150
|
+
`/projects` (the tag tree). Name it with `stepKey` (either lane)
|
|
151
|
+
— explicit `stepKey` wins, else `tags.step`, else an auto key (`route#n` /
|
|
152
|
+
`provider/model#n`). An `artifact.save` **without** a `stepKey` lands in the
|
|
153
|
+
**ambient step** (the most recent step in the context), so a raw call and the
|
|
154
|
+
artifacts you save around it end up in one group on `/runs/[id]`:
|
|
155
|
+
|
|
156
|
+
```ts
|
|
157
|
+
const run = rig.artifacts.run.start({ pipeline: "report" });
|
|
158
|
+
// A raw (BYOK) call names its step; the save with no stepKey joins it.
|
|
159
|
+
const out = await rig.runRaw({ provider, model, apiKey, systemPrompt, userPrompt,
|
|
160
|
+
tags: { project: "acme" }, stepKey: "earnings_5a" });
|
|
161
|
+
rig.artifacts.artifact.save(out.output, { name: "earnings.output", type: "step_output" });
|
|
162
|
+
run.end("succeeded");
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
> **Raw lane and the tree.** A `rig.runRaw` / `rig.runRawStream` call inside a run
|
|
166
|
+
> context records a ground-truth step **since modelrig 0.5.0**, on success and on
|
|
167
|
+
> failure. Before 0.5.0 the raw lane never read the run context, so a pipeline
|
|
168
|
+
> that wrapped raw calls in `run.start` got an **empty tree**. (Raw steps are
|
|
169
|
+
> metadata only — no artifact *content* is written on the raw lane.)
|
|
170
|
+
|
|
171
|
+
## Never throws into your pipeline
|
|
172
|
+
|
|
173
|
+
Every entry point is fail-open: a telemetry-buffer write that fails is logged and
|
|
174
|
+
dropped, never thrown into your code. Instrumentation that runs *after* an
|
|
175
|
+
expensive model call must never be the thing that kills the run.
|
|
176
|
+
|
|
177
|
+
## See it
|
|
178
|
+
|
|
179
|
+
The console's **Runs** tab lists every instrumented run; `/runs/[id]` is the
|
|
180
|
+
semantic chain (artifacts grouped by step, with hashes, costs, and evaluation
|
|
181
|
+
badges) and `/artifacts/[id]` shows one artifact's lineage, versions, and
|
|
182
|
+
evaluations. Enabling the flag also lights up the `/setup` artifacts line.
|
|
183
|
+
|
|
184
|
+
## A reusable per-step seam
|
|
185
|
+
|
|
186
|
+
Most pipelines want one small helper that saves the three work products of a
|
|
187
|
+
step and wires their lineage, so the call sites stay a single line.
|
|
188
|
+
**`modelrig observe [path] --scope <one pipeline>` generates this seam for you**
|
|
189
|
+
(a `record-step` file) plus an `OBSERVE.md` whose per-call-site wiring is emitted
|
|
190
|
+
as diffs to apply — it never edits your source. The seam is equally yours to
|
|
191
|
+
write by hand; here it is, self-contained:
|
|
192
|
+
|
|
193
|
+
```ts
|
|
194
|
+
import { createRig, loadConfigFromEnv, type Rig } from "modelrig";
|
|
195
|
+
|
|
196
|
+
const rig: Rig = createRig(loadConfigFromEnv());
|
|
197
|
+
|
|
198
|
+
/** Save prompt + raw + parsed for one step, link lineage, flag conformance. */
|
|
199
|
+
function recordStep(
|
|
200
|
+
stepKey: string,
|
|
201
|
+
work: { prompt: string; rawText: string; json: unknown; conforms: boolean },
|
|
202
|
+
): void {
|
|
203
|
+
const p = rig.artifacts.artifact.save(work.prompt, { name: `${stepKey}.prompt`, type: "prompt", stepKey });
|
|
204
|
+
const raw = rig.artifacts.artifact.save(work.rawText, { name: `${stepKey}.raw`, type: "raw_response", stepKey });
|
|
205
|
+
const parsed = rig.artifacts.artifact.save(work.json, { name: `${stepKey}.output`, type: "step_output", task: stepKey, stepKey });
|
|
206
|
+
if (p && parsed) rig.artifacts.artifact.link(parsed, p, "derived_from");
|
|
207
|
+
if (raw && parsed) rig.artifacts.artifact.link(parsed, raw, "derived_from");
|
|
208
|
+
if (parsed) rig.artifacts.artifact.evaluate(parsed, { evaluator: "schema-check", kind: "deterministic", passed: work.conforms });
|
|
209
|
+
}
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Call `recordStep(step, { … })` once per stage inside the `run.start` / `run.end`
|
|
213
|
+
context above, and every entry point stays fail-open — a save that fails is
|
|
214
|
+
logged and dropped, never thrown into your pipeline.
|
|
215
|
+
|
|
216
|
+
> **In-repo example (reconstructed via onboarding-test run-2).** ModelRig's own
|
|
217
|
+
> customer-zero pipeline (the InferWealth v3 report generator) was stripped back
|
|
218
|
+
> to a clean slate to be re-onboarded through this exact doc. Its live seam is
|
|
219
|
+
> reconstructed as part of onboarding-test run-2; until then, the snippet above
|
|
220
|
+
> is the reference. Do not resurrect a deleted seam from git history — writing a
|
|
221
|
+
> fresh one is the point.
|