modelrig 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +227 -0
- package/README.md +51 -27
- package/data/PROVENANCE.md +14 -0
- package/data/pricing-map.json +26 -0
- package/dist/adapters/attempt-meta-map.d.ts +8 -1
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -1
- package/dist/adapters/attempt-meta-map.js +4 -0
- package/dist/adapters/build-adapters.js +2 -2
- package/dist/adapters/gemini.d.ts +28 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +38 -9
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +53 -4
- package/dist/adapters/openai-compat.d.ts +8 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +8 -1
- package/dist/adapters/openai-stream.d.ts +10 -2
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +5 -1
- package/dist/adapters/openai.d.ts +10 -1
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +57 -7
- package/dist/adapters/responses-api.d.ts +61 -0
- package/dist/adapters/responses-api.d.ts.map +1 -0
- package/dist/adapters/responses-api.js +124 -0
- package/dist/artifacts/artifact.d.ts +20 -0
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +207 -171
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +8 -1
- package/dist/bakeoff/groundedness.d.ts +58 -0
- package/dist/bakeoff/groundedness.d.ts.map +1 -0
- package/dist/bakeoff/groundedness.js +165 -0
- package/dist/bakeoff/quality.d.ts +14 -1
- package/dist/bakeoff/quality.d.ts.map +1 -1
- package/dist/bakeoff/quality.js +10 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +4 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +8 -0
- package/dist/config.d.ts +31 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +23 -0
- package/dist/doctor-cli.d.ts +7 -0
- package/dist/doctor-cli.d.ts.map +1 -0
- package/dist/doctor-cli.js +68 -0
- package/dist/doctor-preflight.d.ts +64 -0
- package/dist/doctor-preflight.d.ts.map +1 -0
- package/dist/doctor-preflight.js +167 -0
- package/dist/feedback/coverage.d.ts.map +1 -1
- package/dist/feedback/coverage.js +9 -2
- package/dist/feedback/grade.d.ts +8 -1
- package/dist/feedback/grade.d.ts.map +1 -1
- package/dist/feedback/grade.js +4 -0
- package/dist/gateway/http-protocol.d.ts +19 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -1
- package/dist/gateway/serve-client.d.ts +99 -0
- package/dist/gateway/serve-client.d.ts.map +1 -0
- package/dist/gateway/serve-client.js +360 -0
- package/dist/index.d.ts +31 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +71 -8
- package/dist/prompts/manifest.d.ts +128 -0
- package/dist/prompts/manifest.d.ts.map +1 -0
- package/dist/prompts/manifest.js +214 -0
- package/dist/prompts/prompt-fetch.d.ts +84 -0
- package/dist/prompts/prompt-fetch.d.ts.map +1 -0
- package/dist/prompts/prompt-fetch.js +120 -0
- package/dist/prompts/prompt-resolve.d.ts +76 -0
- package/dist/prompts/prompt-resolve.d.ts.map +1 -0
- package/dist/prompts/prompt-resolve.js +93 -0
- package/dist/prompts/slot-resolver.d.ts +91 -0
- package/dist/prompts/slot-resolver.d.ts.map +1 -0
- package/dist/prompts/slot-resolver.js +135 -0
- package/dist/prompts/tools-register.d.ts +60 -0
- package/dist/prompts/tools-register.d.ts.map +1 -0
- package/dist/prompts/tools-register.js +48 -0
- package/dist/prompts/version-hash.d.ts +29 -0
- package/dist/prompts/version-hash.d.ts.map +1 -0
- package/dist/prompts/version-hash.js +54 -0
- package/dist/prompts-export-cli.d.ts +21 -0
- package/dist/prompts-export-cli.d.ts.map +1 -0
- package/dist/prompts-export-cli.js +130 -0
- package/dist/registry/cache-policy.d.ts.map +1 -1
- package/dist/registry/cache-policy.js +22 -0
- package/dist/registry/pricing-shape.d.ts +38 -8
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +40 -12
- package/dist/registry/pricing.d.ts +6 -5
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +9 -5
- package/dist/routes/destination-loader.d.ts +16 -0
- package/dist/routes/destination-loader.d.ts.map +1 -0
- package/dist/routes/destination-loader.js +29 -0
- package/dist/routes/draft.d.ts +7 -0
- package/dist/routes/draft.d.ts.map +1 -1
- package/dist/routes/draft.js +12 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +27 -54
- package/dist/routes/output-loader.d.ts +42 -0
- package/dist/routes/output-loader.d.ts.map +1 -0
- package/dist/routes/output-loader.js +112 -0
- package/dist/routes/prompt-loader.d.ts +49 -0
- package/dist/routes/prompt-loader.d.ts.map +1 -0
- package/dist/routes/prompt-loader.js +113 -0
- package/dist/routes/resolved-bundle.d.ts +19 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +24 -0
- package/dist/routes/screen-loader.d.ts +1 -1
- package/dist/routes/screen-loader.d.ts.map +1 -1
- package/dist/routes/screen-loader.js +8 -1
- package/dist/run/accept.d.ts +3 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +1 -0
- package/dist/run/cost.d.ts +31 -2
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +57 -2
- package/dist/run/destination.d.ts +39 -0
- package/dist/run/destination.d.ts.map +1 -0
- package/dist/run/destination.js +45 -0
- package/dist/run/dispatch.d.ts +25 -3
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +26 -4
- package/dist/run/raw-defaults.d.ts.map +1 -1
- package/dist/run/raw-defaults.js +6 -1
- package/dist/run/raw-finalize.d.ts +96 -0
- package/dist/run/raw-finalize.d.ts.map +1 -0
- package/dist/run/raw-finalize.js +283 -0
- package/dist/run/raw-knobs.d.ts +36 -0
- package/dist/run/raw-knobs.d.ts.map +1 -0
- package/dist/run/raw-knobs.js +99 -0
- package/dist/run/raw-request.d.ts +24 -0
- package/dist/run/raw-request.d.ts.map +1 -0
- package/dist/run/raw-request.js +69 -0
- package/dist/run/raw-schema.d.ts +76 -0
- package/dist/run/raw-schema.d.ts.map +1 -0
- package/dist/run/raw-schema.js +166 -0
- package/dist/run/raw-search-cost.d.ts +49 -0
- package/dist/run/raw-search-cost.d.ts.map +1 -0
- package/dist/run/raw-search-cost.js +79 -0
- package/dist/run/raw-support.d.ts +64 -6
- package/dist/run/raw-support.d.ts.map +1 -1
- package/dist/run/raw-support.js +84 -11
- package/dist/run/raw.d.ts +25 -26
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +30 -351
- package/dist/run/record-attempt.d.ts +8 -0
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +26 -12
- package/dist/run/record-call.d.ts +104 -0
- package/dist/run/record-call.d.ts.map +1 -0
- package/dist/run/record-call.js +277 -0
- package/dist/run/repair-attempt.d.ts +68 -0
- package/dist/run/repair-attempt.d.ts.map +1 -0
- package/dist/run/repair-attempt.js +180 -0
- package/dist/run/resolve-destination.d.ts +49 -0
- package/dist/run/resolve-destination.d.ts.map +1 -0
- package/dist/run/resolve-destination.js +76 -0
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +9 -4
- package/dist/run/run.d.ts +30 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +75 -148
- package/dist/run/telemetry-row.d.ts +92 -0
- package/dist/run/telemetry-row.d.ts.map +1 -0
- package/dist/run/telemetry-row.js +81 -0
- package/dist/run/validate.d.ts +12 -1
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +31 -0
- package/dist/run/web-search-pricing.d.ts +43 -0
- package/dist/run/web-search-pricing.d.ts.map +1 -0
- package/dist/run/web-search-pricing.js +56 -0
- package/dist/settings/grading-guide.d.ts +27 -1
- package/dist/settings/grading-guide.d.ts.map +1 -1
- package/dist/status.d.ts +6 -0
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +4 -1
- package/dist/telemetry/artifact-store.d.ts.map +1 -1
- package/dist/telemetry/artifact-store.js +11 -0
- package/dist/telemetry/attempt-meta.d.ts +66 -1
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +27 -1
- package/dist/telemetry/exporter-setup.d.ts +2 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -1
- package/dist/telemetry/exporter-setup.js +6 -1
- package/dist/telemetry/grade-mappers.d.ts +5 -2
- package/dist/telemetry/grade-mappers.d.ts.map +1 -1
- package/dist/telemetry/grade-mappers.js +8 -2
- package/dist/telemetry/grades-sql.d.ts +4 -1
- package/dist/telemetry/grades-sql.d.ts.map +1 -1
- package/dist/telemetry/grades-sql.js +26 -3
- package/dist/telemetry/ingest-protocol.d.ts +26 -2
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +32 -29
- package/dist/telemetry/lanes.d.ts +31 -0
- package/dist/telemetry/lanes.d.ts.map +1 -0
- package/dist/telemetry/lanes.js +41 -0
- package/dist/telemetry/org-identity.d.ts +47 -0
- package/dist/telemetry/org-identity.d.ts.map +1 -0
- package/dist/telemetry/org-identity.js +100 -0
- package/dist/telemetry/route-registration.d.ts +48 -0
- package/dist/telemetry/route-registration.d.ts.map +1 -0
- package/dist/telemetry/route-registration.js +77 -0
- package/dist/telemetry/sinks.d.ts +10 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +52 -6
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +13 -0
- package/dist/tenancy/api-key-format.d.ts +16 -1
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +17 -0
- package/dist/types/provider.d.ts +20 -3
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +1 -0
- package/dist/types/route.d.ts +76 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +30 -2
- package/dist/types/telemetry.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +75 -5
- package/docs/instrumentation.md +98 -36
- package/docs/llms.txt +96 -16
- package/docs/managed-serving.md +150 -0
- package/docs/mcp-oracle.md +75 -2
- package/docs/migration-playbook.md +5 -4
- package/docs/quickstart.md +169 -83
- package/package.json +3 -2
- package/registry/registry.json +82 -0
- package/dist/feedback/feedback.d.ts +0 -35
- package/dist/feedback/feedback.d.ts.map +0 -1
- package/dist/feedback/feedback.js +0 -39
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElE,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAGnE,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAExE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxE,YAAY,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,SAAS,EACT,gBAAgB,EAChB,cAAc,EACd,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,YAAY,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElE,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAGnE,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAExE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxE,YAAY,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,SAAS,EACT,gBAAgB,EAChB,cAAc,EACd,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,YAAY,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,uBAAuB,EACvB,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC/F,YAAY,EACV,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,kBAAkB,EAClB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,YAAY,EACV,mBAAmB,EACnB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAC/E,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAGvF,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,SAAS,CAAC;AACjB,YAAY,EACV,SAAS,EACT,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,YAAY,GACb,MAAM,SAAS,CAAC;AACjB,YAAY,EACV,MAAM,EACN,OAAO,EACP,WAAW,EACX,eAAe,EACf,WAAW,EACX,cAAc,GACf,MAAM,oBAAoB,CAAC;AAE5B;oDACoD;AACpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;CAChC;AAED;;;2DAG2D;AAC3D,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;IACrC;6EACyE;IACzE,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACjD,GAAG;IAAE,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAa/F;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,qBAAqB,GAAG,SAAS,EAC1C,QAAQ,EAAE,oBAAoB,GAAG,IAAI,GACpC,mBAAmB,GAAG,SAAS,CAcjC;AAED;;2EAE2E;AAC3E,wBAAgB,uBAAuB,CAAC,IAAI,EAAE;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;4BAEwB;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,qBAAqB,CAAC;IACzC;;kEAE8D;IAC9D,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACjD,GAAG,iBAAiB,CAcpB"}
|
package/dist/artifacts/index.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* lifecycle + `@step`; chunk B adds the artifact primitives (save/load/link).
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.TAINT_DEFAULT_CAP = exports.TAINT_DEFAULT_MAX_DEPTH = exports.computeDownstreamClosure = exports.normalizeClassification = exports.isDataClassification = exports.SENSITIVE_CLASSIFICATIONS = exports.DATA_CLASSIFICATIONS = exports.scrubDetectors = exports.detectSpans = exports.luhnValid = exports.scrubContent = exports.uploadArtifactContent = exports.openArtifactStore = exports.createArtifactApi = exports.step = exports.createRunNamespace = exports.getActiveRunContext = void 0;
|
|
10
|
+
exports.TAINT_DEFAULT_CAP = exports.TAINT_DEFAULT_MAX_DEPTH = exports.computeDownstreamClosure = exports.normalizeClassification = exports.isDataClassification = exports.SENSITIVE_CLASSIFICATIONS = exports.DATA_CLASSIFICATIONS = exports.scrubDetectors = exports.detectSpans = exports.luhnValid = exports.scrubContent = exports.uploadArtifactContent = exports.parseAuthoringDoc = exports.assertValidManifest = exports.validateManifest = exports.openArtifactStore = exports.createArtifactApi = exports.step = exports.createRunNamespace = exports.getActiveRunContext = void 0;
|
|
11
11
|
exports.wireRigArtifacts = wireRigArtifacts;
|
|
12
12
|
exports.buildArtifactContentDeps = buildArtifactContentDeps;
|
|
13
13
|
exports.createArtifactNamespace = createArtifactNamespace;
|
|
@@ -24,6 +24,13 @@ var artifact_2 = require("./artifact");
|
|
|
24
24
|
Object.defineProperty(exports, "createArtifactApi", { enumerable: true, get: function () { return artifact_2.createArtifactApi; } });
|
|
25
25
|
var artifact_store_1 = require("../telemetry/artifact-store");
|
|
26
26
|
Object.defineProperty(exports, "openArtifactStore", { enumerable: true, get: function () { return artifact_store_1.openArtifactStore; } });
|
|
27
|
+
// PSR-1 — the structure-only assembly-manifest model (spec §2.1). Re-exported
|
|
28
|
+
// here so `export * from "./artifacts"` in index.ts puts it on the public
|
|
29
|
+
// `modelrig` surface for the InferWealth inference + a future MCP write path.
|
|
30
|
+
var manifest_1 = require("../prompts/manifest");
|
|
31
|
+
Object.defineProperty(exports, "validateManifest", { enumerable: true, get: function () { return manifest_1.validateManifest; } });
|
|
32
|
+
Object.defineProperty(exports, "assertValidManifest", { enumerable: true, get: function () { return manifest_1.assertValidManifest; } });
|
|
33
|
+
Object.defineProperty(exports, "parseAuthoringDoc", { enumerable: true, get: function () { return manifest_1.parseAuthoringDoc; } });
|
|
27
34
|
var content_uploader_2 = require("./content-uploader");
|
|
28
35
|
Object.defineProperty(exports, "uploadArtifactContent", { enumerable: true, get: function () { return content_uploader_2.uploadArtifactContent; } });
|
|
29
36
|
var scrub_1 = require("./scrub");
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evidence groundedness grader (call-QA F3 / research §4 G6, spec §5). A
|
|
3
|
+
* DETERMINISTIC bake-off tier that scores a scorecard OUTPUT by whether its
|
|
4
|
+
* evidence quotes are actually present in the replayed INPUT (the transcript /
|
|
5
|
+
* source). A scorer that invents justifications — quotes that never appear in
|
|
6
|
+
* the input — is penalized; this is what stops a model from fabricating
|
|
7
|
+
* evidence to defend a score.
|
|
8
|
+
*
|
|
9
|
+
* Unlike the golden tiers (exact / field-level / llm-judge in quality.ts), this
|
|
10
|
+
* grades the output against the INPUT, not a corrected golden, so it applies
|
|
11
|
+
* whether or not an eval-case `expected` value is present.
|
|
12
|
+
*
|
|
13
|
+
* The scorecard OUTPUT shape (call-QA F2 emit):
|
|
14
|
+
* { criteria: [{ id, score, evidence: [{ quote, turn_id? }], comment }], score, comment }
|
|
15
|
+
*
|
|
16
|
+
* Grounding is a normalized SUBSTRING match against the DECODED input values
|
|
17
|
+
* (inputHaystack): the string leaf values the model was actually shown — never
|
|
18
|
+
* the JSON envelope's escaping or its keys — so a truthful quote spanning escaped
|
|
19
|
+
* chars is grounded and a fabricated quote equal to a key is not. Trivially short
|
|
20
|
+
* quotes never ground (MIN_GROUNDED_QUOTE_LEN).
|
|
21
|
+
*
|
|
22
|
+
* SCOPE NOTE (F3 vs F5): `turn_id` on an evidence entry is carried but NOT
|
|
23
|
+
* hard-validated — turn-scoped matching against the diarized transcript-with-
|
|
24
|
+
* turns artifact is F5's job (spec §7). F3 is substring grounding only.
|
|
25
|
+
*
|
|
26
|
+
* Pure; no I/O. Unit-tested directly (groundedness.test.ts).
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* Pull every `criteria[].evidence[].quote` string from a scorecard output.
|
|
30
|
+
* Tolerant of missing / malformed shapes: a non-object output, a missing or
|
|
31
|
+
* non-array `criteria`, a criterion without an `evidence` array, or an evidence
|
|
32
|
+
* entry without a string `quote` are all skipped rather than throwing. Returns
|
|
33
|
+
* [] for anything it cannot read. Empty (whitespace-only) quotes are dropped —
|
|
34
|
+
* they carry no evidence to ground.
|
|
35
|
+
*/
|
|
36
|
+
export declare function extractEvidenceQuotes(output: unknown): string[];
|
|
37
|
+
/**
|
|
38
|
+
* The fraction in [0,1] of a scorecard's evidence quotes that are grounded in
|
|
39
|
+
* the input, or null when there is nothing to ground.
|
|
40
|
+
*
|
|
41
|
+
* A quote is GROUNDED when its whitespace-collapsed, case-insensitive form is a
|
|
42
|
+
* substring of the DECODED input's whitespace-collapsed, case-insensitive text
|
|
43
|
+
* ({@link inputHaystack} — the string leaf VALUES the model was shown, not the
|
|
44
|
+
* serialized envelope or its keys) AND it is at least {@link MIN_GROUNDED_QUOTE_LEN}
|
|
45
|
+
* chars long (a too-trivial quote is never grounded).
|
|
46
|
+
*
|
|
47
|
+
* Edge cases (explicit, tested):
|
|
48
|
+
* • ZERO evidence quotes across the whole scorecard (none present, or a
|
|
49
|
+
* malformed shape yields none) ⇒ null. There is nothing to ground, so the
|
|
50
|
+
* caller falls through to the next quality tier — NOT a spurious 1.0 or 0.0.
|
|
51
|
+
* • An empty / absent input WITH quotes ⇒ every quote is ungrounded ⇒ 0.
|
|
52
|
+
* • Quotes present but ALL below the min length ⇒ 0, NOT null — they were
|
|
53
|
+
* emitted, they just carry no evidentiary weight (null is only for ZERO
|
|
54
|
+
* emitted quotes).
|
|
55
|
+
* • All quotes grounded ⇒ 1; some grounded ⇒ grounded / total.
|
|
56
|
+
*/
|
|
57
|
+
export declare function groundednessScore(output: unknown, input: unknown): number | null;
|
|
58
|
+
//# sourceMappingURL=groundedness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"groundedness.d.ts","sourceRoot":"","sources":["../../src/bakeoff/groundedness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAoEH;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,EAAE,CAgB/D;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAYhF"}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Evidence groundedness grader (call-QA F3 / research §4 G6, spec §5). A
|
|
4
|
+
* DETERMINISTIC bake-off tier that scores a scorecard OUTPUT by whether its
|
|
5
|
+
* evidence quotes are actually present in the replayed INPUT (the transcript /
|
|
6
|
+
* source). A scorer that invents justifications — quotes that never appear in
|
|
7
|
+
* the input — is penalized; this is what stops a model from fabricating
|
|
8
|
+
* evidence to defend a score.
|
|
9
|
+
*
|
|
10
|
+
* Unlike the golden tiers (exact / field-level / llm-judge in quality.ts), this
|
|
11
|
+
* grades the output against the INPUT, not a corrected golden, so it applies
|
|
12
|
+
* whether or not an eval-case `expected` value is present.
|
|
13
|
+
*
|
|
14
|
+
* The scorecard OUTPUT shape (call-QA F2 emit):
|
|
15
|
+
* { criteria: [{ id, score, evidence: [{ quote, turn_id? }], comment }], score, comment }
|
|
16
|
+
*
|
|
17
|
+
* Grounding is a normalized SUBSTRING match against the DECODED input values
|
|
18
|
+
* (inputHaystack): the string leaf values the model was actually shown — never
|
|
19
|
+
* the JSON envelope's escaping or its keys — so a truthful quote spanning escaped
|
|
20
|
+
* chars is grounded and a fabricated quote equal to a key is not. Trivially short
|
|
21
|
+
* quotes never ground (MIN_GROUNDED_QUOTE_LEN).
|
|
22
|
+
*
|
|
23
|
+
* SCOPE NOTE (F3 vs F5): `turn_id` on an evidence entry is carried but NOT
|
|
24
|
+
* hard-validated — turn-scoped matching against the diarized transcript-with-
|
|
25
|
+
* turns artifact is F5's job (spec §7). F3 is substring grounding only.
|
|
26
|
+
*
|
|
27
|
+
* Pure; no I/O. Unit-tested directly (groundedness.test.ts).
|
|
28
|
+
*/
|
|
29
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
30
|
+
exports.extractEvidenceQuotes = extractEvidenceQuotes;
|
|
31
|
+
exports.groundednessScore = groundednessScore;
|
|
32
|
+
function isObject(v) {
|
|
33
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
34
|
+
}
|
|
35
|
+
/** Collapse all runs of whitespace to a single space, trim, and lowercase — the
|
|
36
|
+
* comparison basis for grounding. This makes a quote grounded regardless of
|
|
37
|
+
* incidental whitespace (newlines, tabs, doubled spaces) or letter case, while
|
|
38
|
+
* never doing a fuzzy match: a quote whose WORDS are not in the input is still
|
|
39
|
+
* ungrounded. Mirrors quality.ts's "normalized, never fuzzy" discipline. */
|
|
40
|
+
function collapse(text) {
|
|
41
|
+
return text.replace(/\s+/g, " ").trim().toLowerCase();
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Quotes whose collapsed length is below this count as UNGROUNDED, never
|
|
45
|
+
* grounded (they still count toward the denominator — see {@link groundednessScore}).
|
|
46
|
+
* A 1–2 char fragment ("a", "no", "ok") is a substring of almost any transcript,
|
|
47
|
+
* so a raw substring match would let a fabricator score 1.0 with trivial
|
|
48
|
+
* "evidence"; too-short quotes carry no evidentiary weight. 12 chars ≈ a few
|
|
49
|
+
* words — enough to be a real span, not an accidental hit. */
|
|
50
|
+
const MIN_GROUNDED_QUOTE_LEN = 12;
|
|
51
|
+
/** Collect every STRING LEAF from a decoded JSON value — object VALUES and array
|
|
52
|
+
* items, NEVER object KEYS. This is the text the model was actually shown (the
|
|
53
|
+
* decoded variable values), so grounding is checked against values, and a
|
|
54
|
+
* fabricated quote equal to a structural KEY (e.g. "transcript") cannot
|
|
55
|
+
* false-pass. */
|
|
56
|
+
function collectStringLeaves(v, out) {
|
|
57
|
+
if (typeof v === "string") {
|
|
58
|
+
out.push(v);
|
|
59
|
+
}
|
|
60
|
+
else if (Array.isArray(v)) {
|
|
61
|
+
for (const item of v)
|
|
62
|
+
collectStringLeaves(item, out);
|
|
63
|
+
}
|
|
64
|
+
else if (isObject(v)) {
|
|
65
|
+
for (const key of Object.keys(v))
|
|
66
|
+
collectStringLeaves(v[key], out);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Render the replayed input to the haystack grounding is checked against: the
|
|
71
|
+
* DECODED string leaf values, joined by spaces — NOT the serialized envelope.
|
|
72
|
+
*
|
|
73
|
+
* The model was shown the decoded variable VALUES, not the JSON wrapper, so:
|
|
74
|
+
* • a string input (the serialized route variables) is JSON.parse'd first,
|
|
75
|
+
* which un-escapes `\"`→`"` and `\n`→newline, then its string leaves are
|
|
76
|
+
* collected — a parse failure means a bare (non-JSON) transcript, used as-is;
|
|
77
|
+
* • an object/array input is walked directly.
|
|
78
|
+
* Object KEYS are excluded (leaves are values only), so a quote equal to a key
|
|
79
|
+
* ("transcript") is not grounded. `undefined`/`null` ⇒ "" (every quote
|
|
80
|
+
* ungrounded). */
|
|
81
|
+
function inputHaystack(input) {
|
|
82
|
+
if (input === undefined || input === null)
|
|
83
|
+
return "";
|
|
84
|
+
if (typeof input === "string") {
|
|
85
|
+
try {
|
|
86
|
+
const parsed = JSON.parse(input);
|
|
87
|
+
const leaves = [];
|
|
88
|
+
collectStringLeaves(parsed, leaves);
|
|
89
|
+
return leaves.join(" ");
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return input; // not JSON — a bare transcript string; use as-is.
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const leaves = [];
|
|
96
|
+
collectStringLeaves(input, leaves);
|
|
97
|
+
return leaves.join(" ");
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Pull every `criteria[].evidence[].quote` string from a scorecard output.
|
|
101
|
+
* Tolerant of missing / malformed shapes: a non-object output, a missing or
|
|
102
|
+
* non-array `criteria`, a criterion without an `evidence` array, or an evidence
|
|
103
|
+
* entry without a string `quote` are all skipped rather than throwing. Returns
|
|
104
|
+
* [] for anything it cannot read. Empty (whitespace-only) quotes are dropped —
|
|
105
|
+
* they carry no evidence to ground.
|
|
106
|
+
*/
|
|
107
|
+
function extractEvidenceQuotes(output) {
|
|
108
|
+
if (!isObject(output))
|
|
109
|
+
return [];
|
|
110
|
+
const criteria = output.criteria;
|
|
111
|
+
if (!Array.isArray(criteria))
|
|
112
|
+
return [];
|
|
113
|
+
const quotes = [];
|
|
114
|
+
for (const criterion of criteria) {
|
|
115
|
+
if (!isObject(criterion))
|
|
116
|
+
continue;
|
|
117
|
+
const evidence = criterion.evidence;
|
|
118
|
+
if (!Array.isArray(evidence))
|
|
119
|
+
continue;
|
|
120
|
+
for (const item of evidence) {
|
|
121
|
+
if (!isObject(item))
|
|
122
|
+
continue;
|
|
123
|
+
const quote = item.quote;
|
|
124
|
+
if (typeof quote === "string" && quote.trim() !== "")
|
|
125
|
+
quotes.push(quote);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return quotes;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* The fraction in [0,1] of a scorecard's evidence quotes that are grounded in
|
|
132
|
+
* the input, or null when there is nothing to ground.
|
|
133
|
+
*
|
|
134
|
+
* A quote is GROUNDED when its whitespace-collapsed, case-insensitive form is a
|
|
135
|
+
* substring of the DECODED input's whitespace-collapsed, case-insensitive text
|
|
136
|
+
* ({@link inputHaystack} — the string leaf VALUES the model was shown, not the
|
|
137
|
+
* serialized envelope or its keys) AND it is at least {@link MIN_GROUNDED_QUOTE_LEN}
|
|
138
|
+
* chars long (a too-trivial quote is never grounded).
|
|
139
|
+
*
|
|
140
|
+
* Edge cases (explicit, tested):
|
|
141
|
+
* • ZERO evidence quotes across the whole scorecard (none present, or a
|
|
142
|
+
* malformed shape yields none) ⇒ null. There is nothing to ground, so the
|
|
143
|
+
* caller falls through to the next quality tier — NOT a spurious 1.0 or 0.0.
|
|
144
|
+
* • An empty / absent input WITH quotes ⇒ every quote is ungrounded ⇒ 0.
|
|
145
|
+
* • Quotes present but ALL below the min length ⇒ 0, NOT null — they were
|
|
146
|
+
* emitted, they just carry no evidentiary weight (null is only for ZERO
|
|
147
|
+
* emitted quotes).
|
|
148
|
+
* • All quotes grounded ⇒ 1; some grounded ⇒ grounded / total.
|
|
149
|
+
*/
|
|
150
|
+
function groundednessScore(output, input) {
|
|
151
|
+
const quotes = extractEvidenceQuotes(output);
|
|
152
|
+
if (quotes.length === 0)
|
|
153
|
+
return null;
|
|
154
|
+
const haystack = collapse(inputHaystack(input));
|
|
155
|
+
let grounded = 0;
|
|
156
|
+
for (const q of quotes) {
|
|
157
|
+
const needle = collapse(q);
|
|
158
|
+
// Too-trivial quotes never ground (they still count in the denominator).
|
|
159
|
+
if (needle.length < MIN_GROUNDED_QUOTE_LEN)
|
|
160
|
+
continue;
|
|
161
|
+
if (haystack.includes(needle))
|
|
162
|
+
grounded += 1;
|
|
163
|
+
}
|
|
164
|
+
return grounded / quotes.length;
|
|
165
|
+
}
|
|
@@ -20,8 +20,13 @@ import type { ValueCheck } from "../types/route";
|
|
|
20
20
|
* as of LRE-2), and `llm-judge` (a PAIRWISE comparison of the output against the
|
|
21
21
|
* golden — NOT a deterministic tier, so this resolver returns null for it and the
|
|
22
22
|
* caller's golden judge (golden-judge.ts, LRE-2) scores it against `expected`).
|
|
23
|
+
*
|
|
24
|
+
* `evidence-grounded` (call-QA F3 / G6) is a fourth, GOLDEN-FREE deterministic
|
|
25
|
+
* kind: it scores a scorecard output by whether its evidence quotes appear in
|
|
26
|
+
* the replayed INPUT (groundedness.ts), not against a corrected golden. It is
|
|
27
|
+
* dispatched here regardless of whether an `expected` value is present.
|
|
23
28
|
*/
|
|
24
|
-
export type GraderType = "exact" | "field-level" | "llm-judge";
|
|
29
|
+
export type GraderType = "exact" | "field-level" | "llm-judge" | "evidence-grounded";
|
|
25
30
|
/**
|
|
26
31
|
* Schema-aware field accuracy in [0,1]: the fraction of the EXPECTED value's leaf
|
|
27
32
|
* fields that the actual output matches. Recurses into objects/arrays keyed by
|
|
@@ -50,6 +55,11 @@ export declare function checkPasses(output: unknown, check: ValueCheck): boolean
|
|
|
50
55
|
export interface DeterministicQualityInput {
|
|
51
56
|
readonly output: unknown;
|
|
52
57
|
readonly valueChecks?: readonly ValueCheck[];
|
|
58
|
+
/** The replayed capture INPUT (transcript / source). Consulted ONLY by the
|
|
59
|
+
* `evidence-grounded` grader (F3/G6), which grounds the output's evidence
|
|
60
|
+
* quotes against it. A string is used as-is; anything else is JSON.stringify'd
|
|
61
|
+
* (groundedness.ts). Ignored by every other grader kind. */
|
|
62
|
+
readonly input?: unknown;
|
|
53
63
|
/** The corrected/golden expected output for this capture, when a bake-off runs
|
|
54
64
|
* from eval cases with a corrected artifact. */
|
|
55
65
|
readonly expected?: unknown;
|
|
@@ -63,6 +73,9 @@ export interface DeterministicQualityInput {
|
|
|
63
73
|
* • `llm-judge` → NOT a deterministic tier; returns null so the caller's
|
|
64
74
|
* golden judge (golden-judge.ts) scores the output against the golden via
|
|
65
75
|
* the N5/N6 pairwise judge (LRE-2).
|
|
76
|
+
* • `evidence-grounded` → GOLDEN-FREE (F3/G6); scores the output's evidence
|
|
77
|
+
* quotes against {@link DeterministicQualityInput.input}, independent of
|
|
78
|
+
* `expected`/`hasExpected`. See {@link groundednessScore}.
|
|
66
79
|
* Absent ⇒ `field-level` behavior (byte-identical to pre-LRE-1b default). */
|
|
67
80
|
readonly graderType?: GraderType;
|
|
68
81
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"quality.d.ts","sourceRoot":"","sources":["../../src/bakeoff/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"quality.d.ts","sourceRoot":"","sources":["../../src/bakeoff/quality.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,aAAa,GAAG,WAAW,GAAG,mBAAmB,CAAC;AAiBrF;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAgBxE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAErE;AAED;iEACiE;AACjE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAmB3E;AAED,uFAAuF;AACvF,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAWvE;AAYD,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C;;;gEAG4D;IAC5D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB;oDACgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;8CAC0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;;;;;;;iFAU6E;IAC7E,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,GAAG,MAAM,GAAG,IAAI,CAuB3F"}
|
package/dist/bakeoff/quality.js
CHANGED
|
@@ -19,6 +19,7 @@ exports.exactMatch = exactMatch;
|
|
|
19
19
|
exports.resolveJsonPointer = resolveJsonPointer;
|
|
20
20
|
exports.checkPasses = checkPasses;
|
|
21
21
|
exports.resolveDeterministicQuality = resolveDeterministicQuality;
|
|
22
|
+
const groundedness_1 = require("./groundedness");
|
|
22
23
|
/** Normalize a scalar for comparison: trim strings; leave numbers/booleans as-is.
|
|
23
24
|
* "normalized match" (spec §4.3) means insensitivity to surrounding whitespace,
|
|
24
25
|
* never a fuzzy match — a wrong value is still wrong. */
|
|
@@ -129,6 +130,15 @@ function normalizeDeep(v) {
|
|
|
129
130
|
* expected value nor a value check applies (the caller then tries the judge).
|
|
130
131
|
*/
|
|
131
132
|
function resolveDeterministicQuality(input) {
|
|
133
|
+
// evidence-grounded (F3/G6): GOLDEN-FREE. Scores the output's evidence quotes
|
|
134
|
+
// against the replayed INPUT, NOT a golden — so it runs whether or not an
|
|
135
|
+
// eval-case `expected` value is present, and does not disturb the exact/
|
|
136
|
+
// field-level/llm-judge/hasExpected/valueChecks precedence below. A null
|
|
137
|
+
// (no evidence quotes to ground) defers to the next tier exactly like the
|
|
138
|
+
// other deterministic kinds.
|
|
139
|
+
if (input.graderType === "evidence-grounded") {
|
|
140
|
+
return (0, groundedness_1.groundednessScore)(input.output, input.input);
|
|
141
|
+
}
|
|
132
142
|
if (input.hasExpected === true) {
|
|
133
143
|
// llm-judge is not deterministic — defer to the judge tier (LRE-2 seam).
|
|
134
144
|
if (input.graderType === "llm-judge")
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;sEAEkE;IAClE,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;sFAGkF;IAClF,QAAQ,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC;IACzC;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../src/bakeoff/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAI1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;uCAEmC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;kFAI8E;IAC9E,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C;;yEAEqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;sEAEkE;IAClE,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;sFACkF;IAClF,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;sFAGkF;IAClF,QAAQ,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC;IACzC;;;;;iFAK6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;wEACoE;IACpE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;yDACqD;IACrD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;sDACkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,SAAS,kBAAkB,EAAE,CAAC,CAoJxC"}
|
package/dist/bakeoff/replay.js
CHANGED
|
@@ -110,6 +110,10 @@ async function replayVariant(deps, captures, req) {
|
|
|
110
110
|
const expected = req.expected?.get(capture.inferenceId);
|
|
111
111
|
const det = (0, quality_1.resolveDeterministicQuality)({
|
|
112
112
|
output: outputValue,
|
|
113
|
+
// The replayed capture INPUT, for the `evidence-grounded` grader (F3/G6)
|
|
114
|
+
// — the serialized route variables (which carry the transcript/source),
|
|
115
|
+
// used as-is since it is already text. Ignored by every other grader.
|
|
116
|
+
input: capture.variablesJson,
|
|
113
117
|
...(route.valueChecks !== undefined ? { valueChecks: route.valueChecks } : {}),
|
|
114
118
|
...(expected !== undefined ? { hasExpected: true, expected: expected.value } : {}),
|
|
115
119
|
...(req.graderType !== undefined ? { graderType: req.graderType } : {}),
|
package/dist/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;GAgBG;AAsDH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACpC;AAeD,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAuBxD;AA+eD,wBAAsB,IAAI,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CA+CnE"}
|
package/dist/cli.js
CHANGED
|
@@ -31,6 +31,7 @@ const failures_teaching_1 = require("./types/failures-teaching");
|
|
|
31
31
|
const config_1 = require("./config");
|
|
32
32
|
const index_1 = require("./index");
|
|
33
33
|
const status_cli_1 = require("./status-cli");
|
|
34
|
+
const doctor_cli_1 = require("./doctor-cli");
|
|
34
35
|
const engine_1 = require("./bakeoff/engine");
|
|
35
36
|
const samples_1 = require("./bakeoff/samples");
|
|
36
37
|
const report_1 = require("./bakeoff/report");
|
|
@@ -59,6 +60,7 @@ const state_store_1 = require("./watch/state-store");
|
|
|
59
60
|
const swaps_client_1 = require("./savings/swaps-client");
|
|
60
61
|
const swap_execute_1 = require("./savings/swap-execute");
|
|
61
62
|
const cli_query_1 = require("./cli-query");
|
|
63
|
+
const prompts_export_cli_1 = require("./prompts-export-cli");
|
|
62
64
|
const cli_judge_1 = require("./cli-judge");
|
|
63
65
|
const regret_monitor_cli_1 = require("./watch/regret-monitor-cli");
|
|
64
66
|
const screen_cli_1 = require("./screen/screen-cli");
|
|
@@ -507,6 +509,7 @@ const USAGE = [
|
|
|
507
509
|
" modelrig screen pull <classifier:name@version> [--url <weights-url>] (SC-1a: fetch an optional classifier's weights to the local cache — never bundled, A3)",
|
|
508
510
|
" modelrig screen probe [--detectors <ids>] [--corpus <ids>] (SC-3: measured TPR/FP per license-clean synthetic corpus — counts + rates only)",
|
|
509
511
|
" modelrig status (print export/artifact/otlp posture)",
|
|
512
|
+
" modelrig doctor [--expect-org <slug>] (confirm which org MODELRIG_API_KEY resolves to via /v1/whoami — ✓ or a loud 401; exits non-zero on a bad key or, with --expect-org, a different org)",
|
|
510
513
|
" modelrig serve (needs the modelrig-server package)",
|
|
511
514
|
" modelrig bakeoff --route <route> --variants a,b,c --replay-last N [--budget-usd usd] [--objective quality-first|balanced|cost-first] [--keep-outputs[=all]] [--json] [--no-mirror]",
|
|
512
515
|
" modelrig bakeoff --from-proposal <id> [--replay-last N] [--keep-outputs[=all]] [--json]",
|
|
@@ -518,6 +521,7 @@ const USAGE = [
|
|
|
518
521
|
" modelrig swap revert <swapId>",
|
|
519
522
|
" modelrig judge <runId> (E5: score a run with the bounded model-judge; explicit + metered)",
|
|
520
523
|
' modelrig query "<grammar>" | query --saved <id> (WI-1b: telemetry analytics — metadata only, never content)',
|
|
524
|
+
" modelrig prompts export --route <route> [--out modelrig/routes] [--label prod | --version N] [--segment default] [--json] (PSR-3: export a prompt version back to git, byte-faithful — proves the round-trip)",
|
|
521
525
|
" modelrig samples purge [--bakeoff <id>]",
|
|
522
526
|
].join("\n");
|
|
523
527
|
async function main(argv) {
|
|
@@ -533,6 +537,8 @@ async function main(argv) {
|
|
|
533
537
|
switch (command) {
|
|
534
538
|
case "status":
|
|
535
539
|
return (0, status_cli_1.runStatusCli)();
|
|
540
|
+
case "doctor":
|
|
541
|
+
return (0, doctor_cli_1.runDoctorCli)(flags.named.has("expect-org") ? { expectOrg: flags.named.get("expect-org") ?? "" } : {});
|
|
536
542
|
case "validate":
|
|
537
543
|
return (0, validate_cli_1.runValidateCli)();
|
|
538
544
|
case "screen":
|
|
@@ -559,6 +565,8 @@ async function main(argv) {
|
|
|
559
565
|
return (0, regret_monitor_cli_1.cmdRegretMonitor)(flags);
|
|
560
566
|
case "serve":
|
|
561
567
|
return cmdServe();
|
|
568
|
+
case "prompts":
|
|
569
|
+
return (0, prompts_export_cli_1.cmdPromptsExport)(flags);
|
|
562
570
|
default:
|
|
563
571
|
console.error(USAGE);
|
|
564
572
|
return 1;
|
package/dist/config.d.ts
CHANGED
|
@@ -35,6 +35,9 @@ export interface RigConfig {
|
|
|
35
35
|
/** A1 aggregator hosts (generic openai-compatible adapter). */
|
|
36
36
|
readonly deepinfra?: string;
|
|
37
37
|
readonly fireworks?: string;
|
|
38
|
+
/** Z.ai / Zhipu (B4a) — env: ZAI_API_KEY (unprefixed vendor key). Reached via
|
|
39
|
+
* the generic openai-compatible adapter (same family as the A1 hosts). */
|
|
40
|
+
readonly zai?: string;
|
|
38
41
|
};
|
|
39
42
|
/** Budget for envelopes first opened by this rig (default: envelope.ts's $25). */
|
|
40
43
|
readonly envelopeBudgetUsd?: number;
|
|
@@ -101,6 +104,34 @@ export interface RigConfig {
|
|
|
101
104
|
* spell out the off case.
|
|
102
105
|
*/
|
|
103
106
|
readonly artifacts?: ArtifactsConfig;
|
|
107
|
+
/**
|
|
108
|
+
* Managed serving (Unified Route U3, ruling D-2) — the switch for
|
|
109
|
+
* `destination: modelrig-served`. Set IN CODE at construction, NOT via an env
|
|
110
|
+
* var (Matt, 2026-09-09): `createRig({ ...loadConfigFromEnv(), managedServing: true })`
|
|
111
|
+
* or `loadConfigFromEnv({ managedServing: true })`. `loadConfigFromEnv` defaults
|
|
112
|
+
* it to `false` and reads NO `process.env` key for it; like `otlp`/`artifacts`
|
|
113
|
+
* it is ALWAYS resolved so `/status` can answer "enabled|disabled" honestly.
|
|
114
|
+
* Optional so a hand-built config may omit it — an absent value reads as `false`.
|
|
115
|
+
* When off, a run whose resolved destination is `modelrig-served` FAILS CLOSED
|
|
116
|
+
* pre-dispatch (run/resolve-destination.ts) — never a silent `direct` fallback
|
|
117
|
+
* (fail-closed-but-discoverable). Threaded to the routed run path as
|
|
118
|
+
* RunDeps.managedServingEnabled. Turning it on is a deliberate operator/owner
|
|
119
|
+
* action (P9 money/isolation path); the per-org console toggle is the future GA
|
|
120
|
+
* mechanism (server-resolved). CODE ONLY — no GA, no outward copy; true
|
|
121
|
+
* customer-keyless serving is a hosted-server capability, deferred (spec §4 U3).
|
|
122
|
+
*/
|
|
123
|
+
readonly managedServing?: boolean;
|
|
124
|
+
/**
|
|
125
|
+
* The org slug this process expects its `MODELRIG_API_KEY` to resolve to
|
|
126
|
+
* (REL-1, ruling M5). On the first successful flush the api sink asks
|
|
127
|
+
* `GET /v1/whoami` and logs the org it is exporting to; when the resolved slug
|
|
128
|
+
* differs from `expectOrg` it logs `[modelrig] ORG MISMATCH …` loudly on every
|
|
129
|
+
* flush. It NEVER throws and NEVER drops telemetry — the
|
|
130
|
+
* rows still land in the key's org. Set IN CODE, not via an env var (P15):
|
|
131
|
+
* `createRig({ ...loadConfigFromEnv(), expectOrg: "inferwealth" })`. Absent ⇒
|
|
132
|
+
* the org line is still logged, with no comparison.
|
|
133
|
+
*/
|
|
134
|
+
readonly expectOrg?: string;
|
|
104
135
|
}
|
|
105
136
|
/** Scrub mode applied to content BEFORE upload (E2-C chunk B). Default `detect`
|
|
106
137
|
* when content custody is on. The EFFECTIVE mode is resolved per task/account
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;kFAIkF;AAClF,wBAAgB,kBAAkB,IAAI,MAAM,GAAG,SAAS,CAMvD;AAED,MAAM,WAAW,SAAS;IACxB;;;;;4EAKwE;IACxE,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;;qFAIiF;IACjF,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,qDAAqD;QACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QACvB,+DAA+D;QAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;kFAIkF;AAClF,wBAAgB,kBAAkB,IAAI,MAAM,GAAG,SAAS,CAMvD;AAED,MAAM,WAAW,SAAS;IACxB;;;;;4EAKwE;IACxE,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;;qFAIiF;IACjF,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,qDAAqD;QACrD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QACvB,+DAA+D;QAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B;kFAC0E;QAC1E,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,kFAAkF;IAClF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;8EAC0E;IAC1E,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC;;;6CAGyC;IACzC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,CAAC,CAAC,CAAC;IACtF;8DAC0D;IAC1D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B;;eAEW;IACX,QAAQ,CAAC,MAAM,CAAC,EAAE;QAChB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;QAC9B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;KAC7C,CAAC;IACF;;wCAEoC;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE;QACf,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;KAChC,CAAC;IACF;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,eAAe,GAAG,kBAAkB,CAAC;IACvD;;;;;;;;OAQG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;IAClC;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;IACrC;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAClC;;;;;;;;;OASG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;uBAIuB;AACvB,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D;;;8DAG8D;AAC9D,eAAO,MAAM,0BAA0B,EAAE,SAAoB,CAAC;AAE9D;;;;kFAIkF;AAClF,MAAM,WAAW,qBAAqB;IACpC,qEAAqE;IACrE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;6EAC6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,qBAAqB,CAAC;CAC1C;AAID;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,sBAAsB,CACpC,GAAG,EAAE;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AACD;;4EAE4E;AAC5E,sBAAsB,UAAQ;AAC9B;iFACiF;AACjF,UAAU,UAAQ,GACjB,eAAe,CAsBjB;AAKD;;;;;wDAKwD;AACxD,wBAAgB,wBAAwB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAGtE;AAED;uEACuE;AACvE,MAAM,WAAW,UAAU;IACzB,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,mFAAmF;IACnF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAYhF;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,UAAU,GAAG,IAAI,CAGnB;AAED,UAAU,YAAY;IACpB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IACrB;+EAC2E;IAC3E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;;uCAEmC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED;;kEAEkE;AAClE,eAAO,MAAM,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAa7D,CAAC;AAKF,wBAAgB,iBAAiB,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,SAAS,CA4K3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,MAAM,GACX;IAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAgB/F"}
|
package/dist/config.js
CHANGED
|
@@ -137,6 +137,11 @@ exports.PROVIDER_KEY_ENV = {
|
|
|
137
137
|
grok: "XAI_API_KEY",
|
|
138
138
|
deepinfra: "MODELRIG_DEEPINFRA_API_KEY",
|
|
139
139
|
fireworks: "MODELRIG_FIREWORKS_API_KEY",
|
|
140
|
+
// Z.ai (B4a): UNPREFIXED on purpose — matches the XAI_API_KEY/ANTHROPIC_API_KEY
|
|
141
|
+
// vendor-key convention (only the aggregator hosts deepinfra/fireworks + tavily
|
|
142
|
+
// carry the MODELRIG_ prefix, for host-app env collision-avoidance, which does
|
|
143
|
+
// not apply to a first-party vendor key).
|
|
144
|
+
zai: "ZAI_API_KEY",
|
|
140
145
|
};
|
|
141
146
|
const DEFAULT_ROUTES_DIR = "./modelrig/routes";
|
|
142
147
|
const DEFAULT_TELEMETRY_DB = "./.modelrig/telemetry.db";
|
|
@@ -150,6 +155,19 @@ function loadConfigFromEnv(overrides) {
|
|
|
150
155
|
// OTLP span emission (E1) — off unless an endpoint is set. Additive to
|
|
151
156
|
// `export`; a rig may emit OTLP with no control-plane sink and vice-versa.
|
|
152
157
|
const otlp = resolveOtlpConfig(env.MODELRIG_OTLP_ENDPOINT, env.MODELRIG_OTLP_HEADERS);
|
|
158
|
+
// Managed serving (Unified Route U3, ruling D-2) is deliberately NOT an env var
|
|
159
|
+
// (Matt, 2026-09-09): `config.managedServing` is set IN CODE at construction —
|
|
160
|
+
// `createRig({ ...loadConfigFromEnv(), managedServing: true })`, or
|
|
161
|
+
// `loadConfigFromEnv({ managedServing: true })` (overrides win below) — and
|
|
162
|
+
// defaults OFF (`false` in `fromEnv`). It is intentionally read from NO
|
|
163
|
+
// `process.env` key. The per-org console/settings toggle is the future GA
|
|
164
|
+
// mechanism (server-resolved, like `artifact_posture`). When off/absent, a
|
|
165
|
+
// resolved `modelrig-served` FAILS CLOSED pre-dispatch (run/resolve-destination.ts)
|
|
166
|
+
// — never a silent `direct` fallback. This is NOT a customer-key override: the
|
|
167
|
+
// fireworks/deepinfra adapters already carry ModelRig-owned keys on the routed
|
|
168
|
+
// lane, so U3 is a config-flag + capability SEAM, not a key switch (true
|
|
169
|
+
// customer-keyless serving now ships as a hosted-server capability behind this
|
|
170
|
+
// same flag — no longer deferred).
|
|
153
171
|
// Artifact / execution-graph namespace (E2; P15 CP-3) — ON by default when a
|
|
154
172
|
// control plane is configured (api key OR the direct supabase pair). The ORG's
|
|
155
173
|
// account posture (`artifact_posture`), resolved from the control plane at
|
|
@@ -190,6 +208,9 @@ function loadConfigFromEnv(overrides) {
|
|
|
190
208
|
telemetryDbPath: env.MODELRIG_TELEMETRY_DB ?? DEFAULT_TELEMETRY_DB,
|
|
191
209
|
otlp,
|
|
192
210
|
artifacts,
|
|
211
|
+
// U3 (D-2): default OFF; enabled in code via `createRig`/`loadConfigFromEnv`
|
|
212
|
+
// overrides (spread below), NOT from an env var (Matt, 2026-09-09).
|
|
213
|
+
managedServing: false,
|
|
193
214
|
...(env.MODELRIG_REGISTRY_PATH ? { registryPath: env.MODELRIG_REGISTRY_PATH } : {}),
|
|
194
215
|
keys: {
|
|
195
216
|
...(env.GEMINI_API_KEY ? { gemini: env.GEMINI_API_KEY } : {}),
|
|
@@ -200,6 +221,8 @@ function loadConfigFromEnv(overrides) {
|
|
|
200
221
|
// A1 hosts: ModelRig-scoped names on purpose (same doctrine as Tavily).
|
|
201
222
|
...(env.MODELRIG_DEEPINFRA_API_KEY ? { deepinfra: env.MODELRIG_DEEPINFRA_API_KEY } : {}),
|
|
202
223
|
...(env.MODELRIG_FIREWORKS_API_KEY ? { fireworks: env.MODELRIG_FIREWORKS_API_KEY } : {}),
|
|
224
|
+
// Z.ai (B4a): unprefixed vendor key (see PROVIDER_KEY_ENV).
|
|
225
|
+
...(env.ZAI_API_KEY ? { zai: env.ZAI_API_KEY } : {}),
|
|
203
226
|
},
|
|
204
227
|
...(budget !== undefined && Number.isFinite(budget) ? { envelopeBudgetUsd: budget } : {}),
|
|
205
228
|
...(env.MODELRIG_UNPRICED_RATE_USD_PER_MTOK &&
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export interface DoctorOptions {
|
|
2
|
+
/** `--expect-org <slug>`: exit 1 (printing both slugs) when the key resolves
|
|
3
|
+
* to a different org — the valid-key-wrong-org mistake (M5). */
|
|
4
|
+
readonly expectOrg?: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function runDoctorCli(opts?: DoctorOptions): Promise<number>;
|
|
7
|
+
//# sourceMappingURL=doctor-cli.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor-cli.d.ts","sourceRoot":"","sources":["../src/doctor-cli.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,aAAa;IAC5B;oEACgE;IAChE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,wBAAsB,YAAY,CAAC,IAAI,GAAE,aAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAwD5E"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runDoctorCli = runDoctorCli;
|
|
4
|
+
/**
|
|
5
|
+
* `modelrig doctor` — confirm, in one line, which org your key resolves to.
|
|
6
|
+
*
|
|
7
|
+
* The explicit form of the auth preflight (doctor-preflight.ts). Run it in a
|
|
8
|
+
* worker shell, a CI step, or setup step 0 to answer the question the missing-
|
|
9
|
+
* runs incident could not: is MODELRIG_API_KEY valid, and does it belong to the
|
|
10
|
+
* org you expect? Prints the key → org line (✓ on success, a loud line on 401),
|
|
11
|
+
* and exits non-zero when the key is rejected or unconfirmed — so CI can gate on
|
|
12
|
+
* it. `--expect-org <slug>` also exits non-zero when the key is valid but
|
|
13
|
+
* belongs to a different org, printing both slugs. Never prints the key itself
|
|
14
|
+
* (last-four only, via redactApiKey).
|
|
15
|
+
*
|
|
16
|
+
* Kept in its own file so cli.ts stays under the LOC cap (like status-cli.ts).
|
|
17
|
+
*/
|
|
18
|
+
const config_1 = require("./config");
|
|
19
|
+
const doctor_preflight_1 = require("./doctor-preflight");
|
|
20
|
+
async function runDoctorCli(opts = {}) {
|
|
21
|
+
// An empty --expect-org would make the gate unable to fail: refuse it (usage error).
|
|
22
|
+
if (opts.expectOrg !== undefined && opts.expectOrg.trim() === "") {
|
|
23
|
+
console.error("[modelrig] usage: modelrig doctor --expect-org <slug> — the slug must not be empty.");
|
|
24
|
+
return 2;
|
|
25
|
+
}
|
|
26
|
+
// config.ts is the only env reader (package-invariants) — doctor reads the
|
|
27
|
+
// resolved config, never process.env.
|
|
28
|
+
const config = (0, config_1.loadConfigFromEnv)();
|
|
29
|
+
const exp = config.export;
|
|
30
|
+
if (exp === undefined) {
|
|
31
|
+
console.error([
|
|
32
|
+
"[modelrig] no export sink configured — nothing to check.",
|
|
33
|
+
"Set MODELRIG_API_KEY (a rig_sk_ key minted at app.modelrig.ai) so the SDK",
|
|
34
|
+
"ships telemetry to the hosted API. Without it, runs are recorded locally only.",
|
|
35
|
+
].join("\n"));
|
|
36
|
+
return 1;
|
|
37
|
+
}
|
|
38
|
+
if (exp.mode !== "api") {
|
|
39
|
+
console.log(`[modelrig] export sink is ${exp.mode} mode (self-hosted control plane) — whoami is an ` +
|
|
40
|
+
"api-mode check, so there is nothing to confirm here.");
|
|
41
|
+
return 0;
|
|
42
|
+
}
|
|
43
|
+
const result = await (0, doctor_preflight_1.runAuthPreflight)(config);
|
|
44
|
+
if (result === null) {
|
|
45
|
+
// Unreachable given the api-mode guard above, but keep the contract total.
|
|
46
|
+
console.log("[modelrig] nothing to check.");
|
|
47
|
+
return 0;
|
|
48
|
+
}
|
|
49
|
+
if (result.ok) {
|
|
50
|
+
const expectOrg = opts.expectOrg?.trim();
|
|
51
|
+
if (expectOrg !== undefined && expectOrg !== "") {
|
|
52
|
+
const actual = (0, doctor_preflight_1.expectedOrgMismatch)(result, expectOrg);
|
|
53
|
+
if (actual !== null) {
|
|
54
|
+
console.error(`[modelrig] ORG MISMATCH: key resolves to «${actual}», --expect-org is «${expectOrg}» — ` +
|
|
55
|
+
`telemetry from this key lands in «${actual}». Mint a key in the «${expectOrg}» org at ` +
|
|
56
|
+
"app.modelrig.ai, or fix --expect-org.");
|
|
57
|
+
return 1;
|
|
58
|
+
}
|
|
59
|
+
console.log(`${result.line} (org «${expectOrg}» as expected)`);
|
|
60
|
+
return 0;
|
|
61
|
+
}
|
|
62
|
+
console.log(result.line);
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
console.error(result.line);
|
|
66
|
+
// L6 NIT-6: the wrong kind of secret is a usage error (exit 2), not a failed check.
|
|
67
|
+
return result.status === "not_rig_key" ? 2 : 1;
|
|
68
|
+
}
|