modelrig 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +179 -0
- package/README.md +51 -27
- package/data/PROVENANCE.md +14 -0
- package/data/pricing-map.json +26 -0
- package/dist/adapters/attempt-meta-map.d.ts +8 -1
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -1
- package/dist/adapters/attempt-meta-map.js +4 -0
- package/dist/adapters/build-adapters.js +2 -2
- package/dist/adapters/gemini.d.ts +28 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +38 -9
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +53 -4
- package/dist/adapters/openai-compat.d.ts +8 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +8 -1
- package/dist/adapters/openai-stream.d.ts +10 -2
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +5 -1
- package/dist/adapters/openai.d.ts +10 -1
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +57 -7
- package/dist/adapters/responses-api.d.ts +61 -0
- package/dist/adapters/responses-api.d.ts.map +1 -0
- package/dist/adapters/responses-api.js +124 -0
- package/dist/artifacts/artifact.d.ts +20 -0
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +207 -171
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +8 -1
- package/dist/bakeoff/groundedness.d.ts +58 -0
- package/dist/bakeoff/groundedness.d.ts.map +1 -0
- package/dist/bakeoff/groundedness.js +165 -0
- package/dist/bakeoff/quality.d.ts +14 -1
- package/dist/bakeoff/quality.d.ts.map +1 -1
- package/dist/bakeoff/quality.js +10 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +4 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +8 -0
- package/dist/config.d.ts +31 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +23 -0
- package/dist/doctor-cli.d.ts +7 -0
- package/dist/doctor-cli.d.ts.map +1 -0
- package/dist/doctor-cli.js +67 -0
- package/dist/doctor-preflight.d.ts +52 -0
- package/dist/doctor-preflight.d.ts.map +1 -0
- package/dist/doctor-preflight.js +112 -0
- package/dist/feedback/grade.d.ts +8 -1
- package/dist/feedback/grade.d.ts.map +1 -1
- package/dist/feedback/grade.js +4 -0
- package/dist/gateway/http-protocol.d.ts +19 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -1
- package/dist/gateway/serve-client.d.ts +99 -0
- package/dist/gateway/serve-client.d.ts.map +1 -0
- package/dist/gateway/serve-client.js +360 -0
- package/dist/index.d.ts +25 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -8
- package/dist/prompts/manifest.d.ts +128 -0
- package/dist/prompts/manifest.d.ts.map +1 -0
- package/dist/prompts/manifest.js +214 -0
- package/dist/prompts/prompt-fetch.d.ts +84 -0
- package/dist/prompts/prompt-fetch.d.ts.map +1 -0
- package/dist/prompts/prompt-fetch.js +120 -0
- package/dist/prompts/prompt-resolve.d.ts +76 -0
- package/dist/prompts/prompt-resolve.d.ts.map +1 -0
- package/dist/prompts/prompt-resolve.js +93 -0
- package/dist/prompts/slot-resolver.d.ts +91 -0
- package/dist/prompts/slot-resolver.d.ts.map +1 -0
- package/dist/prompts/slot-resolver.js +135 -0
- package/dist/prompts/tools-register.d.ts +60 -0
- package/dist/prompts/tools-register.d.ts.map +1 -0
- package/dist/prompts/tools-register.js +48 -0
- package/dist/prompts/version-hash.d.ts +29 -0
- package/dist/prompts/version-hash.d.ts.map +1 -0
- package/dist/prompts/version-hash.js +54 -0
- package/dist/prompts-export-cli.d.ts +21 -0
- package/dist/prompts-export-cli.d.ts.map +1 -0
- package/dist/prompts-export-cli.js +130 -0
- package/dist/registry/cache-policy.d.ts.map +1 -1
- package/dist/registry/cache-policy.js +22 -0
- package/dist/registry/pricing-shape.d.ts +38 -8
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +40 -12
- package/dist/registry/pricing.d.ts +6 -5
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +9 -5
- package/dist/routes/destination-loader.d.ts +16 -0
- package/dist/routes/destination-loader.d.ts.map +1 -0
- package/dist/routes/destination-loader.js +29 -0
- package/dist/routes/draft.d.ts +7 -0
- package/dist/routes/draft.d.ts.map +1 -1
- package/dist/routes/draft.js +12 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +27 -54
- package/dist/routes/output-loader.d.ts +42 -0
- package/dist/routes/output-loader.d.ts.map +1 -0
- package/dist/routes/output-loader.js +112 -0
- package/dist/routes/prompt-loader.d.ts +49 -0
- package/dist/routes/prompt-loader.d.ts.map +1 -0
- package/dist/routes/prompt-loader.js +113 -0
- package/dist/routes/resolved-bundle.d.ts +19 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +24 -0
- package/dist/routes/screen-loader.d.ts +1 -1
- package/dist/routes/screen-loader.d.ts.map +1 -1
- package/dist/routes/screen-loader.js +8 -1
- package/dist/run/accept.d.ts +3 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +1 -0
- package/dist/run/cost.d.ts +31 -2
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +57 -2
- package/dist/run/destination.d.ts +39 -0
- package/dist/run/destination.d.ts.map +1 -0
- package/dist/run/destination.js +45 -0
- package/dist/run/dispatch.d.ts +25 -3
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +26 -4
- package/dist/run/raw-defaults.d.ts.map +1 -1
- package/dist/run/raw-defaults.js +6 -1
- package/dist/run/raw-finalize.d.ts +50 -0
- package/dist/run/raw-finalize.d.ts.map +1 -0
- package/dist/run/raw-finalize.js +266 -0
- package/dist/run/raw-knobs.d.ts +36 -0
- package/dist/run/raw-knobs.d.ts.map +1 -0
- package/dist/run/raw-knobs.js +99 -0
- package/dist/run/raw-request.d.ts +24 -0
- package/dist/run/raw-request.d.ts.map +1 -0
- package/dist/run/raw-request.js +69 -0
- package/dist/run/raw-schema.d.ts +76 -0
- package/dist/run/raw-schema.d.ts.map +1 -0
- package/dist/run/raw-schema.js +166 -0
- package/dist/run/raw-search-cost.d.ts +48 -0
- package/dist/run/raw-search-cost.d.ts.map +1 -0
- package/dist/run/raw-search-cost.js +78 -0
- package/dist/run/raw-support.d.ts +64 -6
- package/dist/run/raw-support.d.ts.map +1 -1
- package/dist/run/raw-support.js +84 -11
- package/dist/run/raw.d.ts +25 -26
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +30 -351
- package/dist/run/record-attempt.d.ts +8 -0
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +26 -12
- package/dist/run/repair-attempt.d.ts +68 -0
- package/dist/run/repair-attempt.d.ts.map +1 -0
- package/dist/run/repair-attempt.js +180 -0
- package/dist/run/resolve-destination.d.ts +49 -0
- package/dist/run/resolve-destination.d.ts.map +1 -0
- package/dist/run/resolve-destination.js +76 -0
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +9 -4
- package/dist/run/run.d.ts +30 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +75 -148
- package/dist/run/telemetry-row.d.ts +92 -0
- package/dist/run/telemetry-row.d.ts.map +1 -0
- package/dist/run/telemetry-row.js +81 -0
- package/dist/run/validate.d.ts +12 -1
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +31 -0
- package/dist/run/web-search-pricing.d.ts +43 -0
- package/dist/run/web-search-pricing.d.ts.map +1 -0
- package/dist/run/web-search-pricing.js +56 -0
- package/dist/settings/grading-guide.d.ts +27 -1
- package/dist/settings/grading-guide.d.ts.map +1 -1
- package/dist/status.d.ts +6 -0
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +4 -1
- package/dist/telemetry/artifact-store.d.ts.map +1 -1
- package/dist/telemetry/artifact-store.js +11 -0
- package/dist/telemetry/attempt-meta.d.ts +55 -1
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +24 -1
- package/dist/telemetry/exporter-setup.d.ts +2 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -1
- package/dist/telemetry/exporter-setup.js +6 -1
- package/dist/telemetry/grade-mappers.d.ts +5 -2
- package/dist/telemetry/grade-mappers.d.ts.map +1 -1
- package/dist/telemetry/grade-mappers.js +8 -2
- package/dist/telemetry/grades-sql.d.ts +4 -1
- package/dist/telemetry/grades-sql.d.ts.map +1 -1
- package/dist/telemetry/grades-sql.js +26 -3
- package/dist/telemetry/ingest-protocol.d.ts +26 -2
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +32 -29
- package/dist/telemetry/org-identity.d.ts +44 -0
- package/dist/telemetry/org-identity.d.ts.map +1 -0
- package/dist/telemetry/org-identity.js +98 -0
- package/dist/telemetry/route-registration.d.ts +48 -0
- package/dist/telemetry/route-registration.d.ts.map +1 -0
- package/dist/telemetry/route-registration.js +77 -0
- package/dist/telemetry/sinks.d.ts +10 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +52 -6
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +13 -0
- package/dist/tenancy/api-key-format.d.ts +16 -1
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +17 -0
- package/dist/types/provider.d.ts +20 -3
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +1 -0
- package/dist/types/route.d.ts +76 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +30 -2
- package/dist/types/telemetry.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +73 -5
- package/docs/instrumentation.md +81 -36
- package/docs/llms.txt +95 -16
- package/docs/managed-serving.md +150 -0
- package/docs/mcp-oracle.md +75 -2
- package/docs/migration-playbook.md +5 -4
- package/docs/quickstart.md +169 -83
- package/package.json +3 -2
- package/registry/registry.json +82 -0
- package/dist/feedback/feedback.d.ts +0 -35
- package/dist/feedback/feedback.d.ts.map +0 -1
- package/dist/feedback/feedback.js +0 -39
|
@@ -22,6 +22,7 @@ const run_context_1 = require("./run-context");
|
|
|
22
22
|
const scrub_1 = require("./scrub");
|
|
23
23
|
const classification_1 = require("./classification");
|
|
24
24
|
const task_settings_1 = require("../settings/task-settings");
|
|
25
|
+
const manifest_1 = require("../prompts/manifest");
|
|
25
26
|
/** The configured scrub mode for one artifact save (BEFORE the S8-c PII/PHI
|
|
26
27
|
* floor). Resolved EXACTLY as capture-write.ts / samples.ts do: a configured
|
|
27
28
|
* control plane WINS (`resolver.get(task).scrub`), else the rig's default. A
|
|
@@ -43,6 +44,9 @@ function contentTypeFor(type) {
|
|
|
43
44
|
return "text/csv";
|
|
44
45
|
if (t === "html")
|
|
45
46
|
return "text/html; charset=utf-8";
|
|
47
|
+
// PSR-1: a structure-only assembly manifest is stored as a linked JSON artifact.
|
|
48
|
+
if (t === "manifest")
|
|
49
|
+
return "application/json";
|
|
46
50
|
return "application/octet-stream";
|
|
47
51
|
}
|
|
48
52
|
function idOf(ref) {
|
|
@@ -56,6 +60,7 @@ function warn(scope, err) {
|
|
|
56
60
|
function inertApi() {
|
|
57
61
|
return {
|
|
58
62
|
save: () => null,
|
|
63
|
+
saveManifest: () => null,
|
|
59
64
|
load: () => { },
|
|
60
65
|
link: () => { },
|
|
61
66
|
promoteToEval: () => null,
|
|
@@ -160,188 +165,219 @@ function createArtifactApi(deps) {
|
|
|
160
165
|
}
|
|
161
166
|
}
|
|
162
167
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
classification
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
}
|
|
168
|
+
// Extracted so `saveManifest` (PSR-1) can compose them without re-implementing
|
|
169
|
+
// the custody/lineage save path or the edges_no_self link. The bodies are the
|
|
170
|
+
// original `save`/`link` verbatim — the returned object references these.
|
|
171
|
+
const save = (value, opts) => {
|
|
172
|
+
const ctx = (0, run_context_1.getActiveRunContext)();
|
|
173
|
+
// zeroRetention: refuse fail-closed BEFORE hashing or building any row.
|
|
174
|
+
if (ctx !== null && ctx.zeroRetention) {
|
|
175
|
+
skippedZeroRetention += 1;
|
|
176
|
+
console.warn(`[modelrig artifacts] zeroRetention run — artifact "${opts.name}" NOT recorded ` +
|
|
177
|
+
"(fail-closed at the SDK gate; no metadata, no hash persisted).");
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
// Hash the serialization, then DISCARD the bytes — metadata + hash only.
|
|
181
|
+
const serialized = typeof value === "string" ? value : JSON.stringify(value ?? null);
|
|
182
|
+
const bytes = Buffer.byteLength(serialized, "utf8");
|
|
183
|
+
const contentHash = `sha256:${(0, node_crypto_1.createHash)("sha256").update(serialized).digest("hex")}`;
|
|
184
|
+
const schema = opts.schema ?? null;
|
|
185
|
+
const id = (0, node_crypto_1.randomUUID)();
|
|
186
|
+
const version = opts.supersedes !== undefined ? opts.supersedes.version + 1 : 1;
|
|
187
|
+
// G1 (spec §3): an explicit stepKey wins; absent, the artifact lands in the
|
|
188
|
+
// AMBIENT step — the most recent auto-step in this run context — so
|
|
189
|
+
// `artifact.save` and the run/@step graph compose. Null outside a step (or
|
|
190
|
+
// outside a run context), exactly as before.
|
|
191
|
+
const stepKey = opts.stepKey ?? ctx?.currentStepKey() ?? null;
|
|
192
|
+
// S8-b: validate data_classification against the shared enum. A bad value
|
|
193
|
+
// would fail the DB CHECK (00058) and drop the row at ingest, so catch it
|
|
194
|
+
// here with a teaching warning and drop the LABEL (the artifact still
|
|
195
|
+
// saves — a metadata typo must not lose the artifact). Normalize first so
|
|
196
|
+
// "PHI"/" pii " keep their meaning (and their scrub floor) rather than
|
|
197
|
+
// silently dropping.
|
|
198
|
+
let classification = (0, classification_1.normalizeClassification)(opts.classification);
|
|
199
|
+
if (opts.classification !== undefined && classification !== null && !(0, classification_1.isDataClassification)(classification)) {
|
|
200
|
+
console.warn(`[modelrig artifacts] artifact "${opts.name}" has an unknown data_classification ` +
|
|
201
|
+
`${JSON.stringify(opts.classification)} — label dropped (artifact still saved). ` +
|
|
202
|
+
`fix_hint: use one of ${classification_1.DATA_CLASSIFICATIONS.join(", ")}.`);
|
|
203
|
+
classification = null;
|
|
204
|
+
}
|
|
205
|
+
// Custody applies only when a rig content policy exists AND the caller left
|
|
206
|
+
// the bytes to us (a caller-supplied contentRef = their own store; managed
|
|
207
|
+
// custody must not double-home it).
|
|
208
|
+
const custodyApplies = content !== undefined && opts.contentRef === undefined;
|
|
209
|
+
// Scrub is computed HERE (before the row is written) so its counts-only
|
|
210
|
+
// summary can ride the immutable artifact row (00039). Only when custody
|
|
211
|
+
// applies; null otherwise (custody off ⇒ nothing was scanned). The
|
|
212
|
+
// configured mode comes from the per-task/account control-plane setting
|
|
213
|
+
// (P15 CP-3b — resolver wins; `content.defaultScrub` fallback local/direct).
|
|
214
|
+
// S8-c: a `pii`/`phi` artifact forces the scrub mode to a floor of
|
|
215
|
+
// `redact`, so PII/PHI content is never uploaded verbatim even when the
|
|
216
|
+
// task's configured mode is `off`/`detect`. A stronger `block` stays.
|
|
217
|
+
const effScrub = content !== undefined
|
|
218
|
+
? (0, task_settings_1.effectiveScrubMode)(resolveContentScrubMode(content, opts.task), classification)
|
|
219
|
+
: null;
|
|
220
|
+
const scrubbed = custodyApplies && content !== undefined && effScrub !== null ? (0, scrub_1.scrubContent)(serialized, effScrub) : null;
|
|
221
|
+
const scrubSummary = scrubbed !== null && effScrub !== null ? (0, scrub_1.summarizeScrub)(effScrub, scrubbed) : null;
|
|
222
|
+
if (scrubbed !== null)
|
|
223
|
+
scrubFindings += scrubbed.findings.reduce((n, f) => n + f.count, 0);
|
|
224
|
+
try {
|
|
225
|
+
store.recordArtifact({
|
|
226
|
+
id,
|
|
227
|
+
runId: ctx?.runId ?? null,
|
|
228
|
+
// E3: step attribution is an explicit caller-supplied key; G1 adds the
|
|
229
|
+
// ambient-step fallback (resolved above) so a save with no stepKey joins
|
|
230
|
+
// the current run/@step. Absent both ⇒ null, exactly as before.
|
|
231
|
+
stepKey,
|
|
232
|
+
attemptNo: opts.attemptNo ?? null,
|
|
233
|
+
task: opts.task ?? null,
|
|
234
|
+
name: opts.name,
|
|
235
|
+
type: opts.type,
|
|
236
|
+
version,
|
|
237
|
+
supersedesArtifactId: opts.supersedes?.id ?? null,
|
|
238
|
+
declaredSchema: schema,
|
|
239
|
+
fingerprintCell: (0, fingerprint_1.fingerprintSchema)(schema)?.cellKey ?? null,
|
|
240
|
+
contentRef: opts.contentRef ?? null,
|
|
241
|
+
contentHash,
|
|
242
|
+
contentBytes: bytes,
|
|
243
|
+
dataClassification: classification,
|
|
244
|
+
expiresAt: opts.expiresAt ?? null,
|
|
245
|
+
// Per-row scrub findings ride the original (scanned) row (00039).
|
|
246
|
+
scrub: scrubSummary,
|
|
247
|
+
});
|
|
248
|
+
saves += 1;
|
|
249
|
+
if (opts.task !== undefined)
|
|
250
|
+
savedTasks.set(id, opts.task); // for promoteToEval (chunk A)
|
|
251
|
+
// Lineage v0: derived_from edges to everything loaded since the last save.
|
|
252
|
+
if (ctx !== null) {
|
|
253
|
+
for (const loadedId of ctx.takeLoadedIds()) {
|
|
254
|
+
if (loadedId === id)
|
|
255
|
+
continue; // edges_no_self
|
|
256
|
+
store.recordArtifactEdge({ fromArtifactId: id, toArtifactId: loadedId, kind: "derived_from" });
|
|
257
|
+
inferredEdges += 1;
|
|
252
258
|
}
|
|
253
259
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
warn("save", err);
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
// ── Content custody (E2-C) ──────────────────────────────────────────
|
|
266
|
+
// Off by default: only when a rig has configured content custody. The
|
|
267
|
+
// metadata row above is written regardless — content is layered on, never
|
|
268
|
+
// a precondition. zeroRetention already returned above, so we never reach
|
|
269
|
+
// here for a zeroRetention run (structural, not re-checked).
|
|
270
|
+
if (content !== undefined && scrubbed !== null) {
|
|
271
|
+
if (scrubbed.blocked) {
|
|
272
|
+
// `block` mode found PII/PHI → content upload refused (counted). The
|
|
273
|
+
// artifact keeps its metadata + hash; it simply stores no content.
|
|
274
|
+
contentBlocked += 1;
|
|
275
|
+
console.warn(`[modelrig artifacts] scrub blocked content for "${opts.name}" ` +
|
|
276
|
+
`(${scrubbed.findings.length} finding type(s)); metadata + hash kept, no content stored.`);
|
|
257
277
|
}
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
supersedesArtifactId: id, // supersedes the original (ruling c column)
|
|
292
|
-
declaredSchema: schema,
|
|
293
|
-
fingerprintCell: (0, fingerprint_1.fingerprintSchema)(schema)?.cellKey ?? null,
|
|
294
|
-
contentRef: null,
|
|
295
|
-
contentHash: maskedHash,
|
|
296
|
-
contentBytes: maskedBytes,
|
|
297
|
-
dataClassification: classification,
|
|
298
|
-
expiresAt: opts.expiresAt ?? null,
|
|
299
|
-
// Findings ride the original scanned row; the masked version stores
|
|
300
|
-
// no PII, so its own scrub is null (avoid double-counting).
|
|
301
|
-
scrub: null,
|
|
302
|
-
});
|
|
303
|
-
saves += 1;
|
|
304
|
-
fireUpload({
|
|
305
|
-
artifactId: redactedId,
|
|
306
|
-
contentType: contentTypeFor(opts.type),
|
|
307
|
-
body: Buffer.from(maskedSerialized, "utf8"),
|
|
308
|
-
contentHash: maskedHash,
|
|
309
|
-
}, content.upload);
|
|
310
|
-
// The live version is the redacted one — hand its handle back.
|
|
311
|
-
return { id: redactedId, name: opts.name, type: opts.type, version: redactedVersion };
|
|
312
|
-
}
|
|
313
|
-
catch (err) {
|
|
314
|
-
warn("save (redacted version)", err);
|
|
315
|
-
// Fall through to returning the original handle (metadata-only).
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
else {
|
|
319
|
-
// off / detect / redact-with-no-findings: upload the bytes verbatim.
|
|
278
|
+
else if (effScrub === "redact" && scrubbed.findings.length > 0 && scrubbed.text !== null) {
|
|
279
|
+
// Redaction is a SUPERSEDING version (immutability): the original row
|
|
280
|
+
// keeps its hash and stores NO content; a new version records the
|
|
281
|
+
// masked content's hash and uploads the masked bytes.
|
|
282
|
+
const maskedSerialized = scrubbed.text;
|
|
283
|
+
const maskedBytes = Buffer.byteLength(maskedSerialized, "utf8");
|
|
284
|
+
const maskedHash = `sha256:${(0, node_crypto_1.createHash)("sha256").update(maskedSerialized).digest("hex")}`;
|
|
285
|
+
const redactedId = (0, node_crypto_1.randomUUID)();
|
|
286
|
+
const redactedVersion = version + 1;
|
|
287
|
+
try {
|
|
288
|
+
store.recordArtifact({
|
|
289
|
+
id: redactedId,
|
|
290
|
+
runId: ctx?.runId ?? null,
|
|
291
|
+
// Same ambient-step resolution as the original row it supersedes.
|
|
292
|
+
stepKey,
|
|
293
|
+
attemptNo: opts.attemptNo ?? null,
|
|
294
|
+
task: opts.task ?? null,
|
|
295
|
+
name: opts.name,
|
|
296
|
+
type: opts.type,
|
|
297
|
+
version: redactedVersion,
|
|
298
|
+
supersedesArtifactId: id, // supersedes the original (ruling c column)
|
|
299
|
+
declaredSchema: schema,
|
|
300
|
+
fingerprintCell: (0, fingerprint_1.fingerprintSchema)(schema)?.cellKey ?? null,
|
|
301
|
+
contentRef: null,
|
|
302
|
+
contentHash: maskedHash,
|
|
303
|
+
contentBytes: maskedBytes,
|
|
304
|
+
dataClassification: classification,
|
|
305
|
+
expiresAt: opts.expiresAt ?? null,
|
|
306
|
+
// Findings ride the original scanned row; the masked version stores
|
|
307
|
+
// no PII, so its own scrub is null (avoid double-counting).
|
|
308
|
+
scrub: null,
|
|
309
|
+
});
|
|
310
|
+
saves += 1;
|
|
320
311
|
fireUpload({
|
|
321
|
-
artifactId:
|
|
312
|
+
artifactId: redactedId,
|
|
322
313
|
contentType: contentTypeFor(opts.type),
|
|
323
|
-
body: Buffer.from(
|
|
324
|
-
contentHash,
|
|
314
|
+
body: Buffer.from(maskedSerialized, "utf8"),
|
|
315
|
+
contentHash: maskedHash,
|
|
325
316
|
}, content.upload);
|
|
317
|
+
// The live version is the redacted one — hand its handle back.
|
|
318
|
+
return { id: redactedId, name: opts.name, type: opts.type, version: redactedVersion };
|
|
319
|
+
}
|
|
320
|
+
catch (err) {
|
|
321
|
+
warn("save (redacted version)", err);
|
|
322
|
+
// Fall through to returning the original handle (metadata-only).
|
|
326
323
|
}
|
|
327
324
|
}
|
|
328
|
-
|
|
329
|
-
|
|
325
|
+
else {
|
|
326
|
+
// off / detect / redact-with-no-findings: upload the bytes verbatim.
|
|
327
|
+
fireUpload({
|
|
328
|
+
artifactId: id,
|
|
329
|
+
contentType: contentTypeFor(opts.type),
|
|
330
|
+
body: Buffer.from(scrubbed.text ?? serialized, "utf8"),
|
|
331
|
+
contentHash,
|
|
332
|
+
}, content.upload);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
return { id, name: opts.name, type: opts.type, version };
|
|
336
|
+
};
|
|
337
|
+
const link = (from, to, kind) => {
|
|
338
|
+
const fromId = idOf(from);
|
|
339
|
+
const toId = idOf(to);
|
|
340
|
+
if (fromId === toId)
|
|
341
|
+
return; // edges_no_self
|
|
342
|
+
try {
|
|
343
|
+
store.recordArtifactEdge({ fromArtifactId: fromId, toArtifactId: toId, kind });
|
|
344
|
+
}
|
|
345
|
+
catch (err) {
|
|
346
|
+
warn("link", err);
|
|
347
|
+
}
|
|
348
|
+
};
|
|
349
|
+
// PSR-1: validate a structure-only manifest, save it as a linked `manifest`
|
|
350
|
+
// artifact (ZERO-DDL free-text type), and chain `prompt → manifest` via
|
|
351
|
+
// derived_from. Never throws; an invalid manifest is a counted, teaching warn.
|
|
352
|
+
const saveManifest = (prompt, manifest, opts) => {
|
|
353
|
+
const result = (0, manifest_1.validateManifest)(manifest);
|
|
354
|
+
if (!result.ok) {
|
|
355
|
+
console.warn(`[modelrig artifacts] manifest for prompt "${idOf(prompt)}" is invalid — not emitted: ` +
|
|
356
|
+
`${result.errors.join("; ")}`);
|
|
357
|
+
return null;
|
|
358
|
+
}
|
|
359
|
+
const promptName = typeof prompt === "string" ? undefined : prompt.name;
|
|
360
|
+
const name = opts?.name ?? (promptName !== undefined ? `${promptName}.manifest` : "manifest");
|
|
361
|
+
const handle = save(JSON.stringify(result.manifest), {
|
|
362
|
+
name,
|
|
363
|
+
type: "manifest",
|
|
364
|
+
...(opts?.task !== undefined ? { task: opts.task } : {}),
|
|
365
|
+
...(opts?.stepKey !== undefined ? { stepKey: opts.stepKey } : {}),
|
|
366
|
+
...(opts?.attemptNo !== undefined ? { attemptNo: opts.attemptNo } : {}),
|
|
367
|
+
});
|
|
368
|
+
if (handle === null)
|
|
369
|
+
return null; // custody/zeroRetention refused — no edge.
|
|
370
|
+
// Chain prompt → manifest: the prompt is assembled per this manifest recipe.
|
|
371
|
+
link(prompt, handle, "derived_from");
|
|
372
|
+
return handle;
|
|
373
|
+
};
|
|
374
|
+
return {
|
|
375
|
+
save,
|
|
376
|
+
saveManifest,
|
|
330
377
|
load(ref) {
|
|
331
378
|
(0, run_context_1.getActiveRunContext)()?.noteLoad(idOf(ref));
|
|
332
379
|
},
|
|
333
|
-
link
|
|
334
|
-
const fromId = idOf(from);
|
|
335
|
-
const toId = idOf(to);
|
|
336
|
-
if (fromId === toId)
|
|
337
|
-
return; // edges_no_self
|
|
338
|
-
try {
|
|
339
|
-
store.recordArtifactEdge({ fromArtifactId: fromId, toArtifactId: toId, kind });
|
|
340
|
-
}
|
|
341
|
-
catch (err) {
|
|
342
|
-
warn("link", err);
|
|
343
|
-
}
|
|
344
|
-
},
|
|
380
|
+
link,
|
|
345
381
|
promoteToEval(ref, opts) {
|
|
346
382
|
const sourceArtifactId = idOf(ref);
|
|
347
383
|
// Resolve the task: explicit wins, else the task this artifact was saved
|
|
@@ -16,7 +16,9 @@ export { step } from "./step";
|
|
|
16
16
|
export { createArtifactApi } from "./artifact";
|
|
17
17
|
export { openArtifactStore } from "../telemetry/artifact-store";
|
|
18
18
|
export type { ArtifactStore } from "../telemetry/artifact-store";
|
|
19
|
-
export type { ArtifactApi, ArtifactHandle, ArtifactSaveOptions, ArtifactPromoteOptions, ArtifactContentDeps, ArtifactContentUploader, } from "./artifact";
|
|
19
|
+
export type { ArtifactApi, ArtifactHandle, ArtifactSaveOptions, ArtifactManifestOptions, ArtifactPromoteOptions, ArtifactContentDeps, ArtifactContentUploader, } from "./artifact";
|
|
20
|
+
export { validateManifest, assertValidManifest, parseAuthoringDoc } from "../prompts/manifest";
|
|
21
|
+
export type { Slot, SlotKind, Manifest, DynamicSource, DynamicSourceKind, CachePolicy, StaticFrom, ManifestValidation, AuthoringDoc, } from "../prompts/manifest";
|
|
20
22
|
export { uploadArtifactContent } from "./content-uploader";
|
|
21
23
|
export type { ContentUploadConfig, ContentUploadInput, ContentUploadOutcome, } from "./content-uploader";
|
|
22
24
|
export { scrubContent, luhnValid, detectSpans, scrubDetectors } from "./scrub";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElE,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAGnE,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAExE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxE,YAAY,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,SAAS,EACT,gBAAgB,EAChB,cAAc,EACd,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,YAAY,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElE,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAGnE,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAExE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxE,YAAY,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,SAAS,EACT,gBAAgB,EAChB,cAAc,EACd,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,YAAY,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,uBAAuB,EACvB,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC/F,YAAY,EACV,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,kBAAkB,EAClB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,YAAY,EACV,mBAAmB,EACnB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAC/E,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAGvF,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,SAAS,CAAC;AACjB,YAAY,EACV,SAAS,EACT,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,YAAY,GACb,MAAM,SAAS,CAAC;AACjB,YAAY,EACV,MAAM,EACN,OAAO,EACP,WAAW,EACX,eAAe,EACf,WAAW,EACX,cAAc,GACf,MAAM,oBAAoB,CAAC;AAE5B;oDACoD;AACpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;CAChC;AAED;;;2DAG2D;AAC3D,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;IACrC;6EACyE;IACzE,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACjD,GAAG;IAAE,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAa/F;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,qBAAqB,GAAG,SAAS,EAC1C,QAAQ,EAAE,oBAAoB,GAAG,IAAI,GACpC,mBAAmB,GAAG,SAAS,CAcjC;AAED;;2EAE2E;AAC3E,wBAAgB,uBAAuB,CAAC,IAAI,EAAE;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;4BAEwB;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,qBAAqB,CAAC;IACzC;;kEAE8D;IAC9D,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACjD,GAAG,iBAAiB,CAcpB"}
|
package/dist/artifacts/index.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* lifecycle + `@step`; chunk B adds the artifact primitives (save/load/link).
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.TAINT_DEFAULT_CAP = exports.TAINT_DEFAULT_MAX_DEPTH = exports.computeDownstreamClosure = exports.normalizeClassification = exports.isDataClassification = exports.SENSITIVE_CLASSIFICATIONS = exports.DATA_CLASSIFICATIONS = exports.scrubDetectors = exports.detectSpans = exports.luhnValid = exports.scrubContent = exports.uploadArtifactContent = exports.openArtifactStore = exports.createArtifactApi = exports.step = exports.createRunNamespace = exports.getActiveRunContext = void 0;
|
|
10
|
+
exports.TAINT_DEFAULT_CAP = exports.TAINT_DEFAULT_MAX_DEPTH = exports.computeDownstreamClosure = exports.normalizeClassification = exports.isDataClassification = exports.SENSITIVE_CLASSIFICATIONS = exports.DATA_CLASSIFICATIONS = exports.scrubDetectors = exports.detectSpans = exports.luhnValid = exports.scrubContent = exports.uploadArtifactContent = exports.parseAuthoringDoc = exports.assertValidManifest = exports.validateManifest = exports.openArtifactStore = exports.createArtifactApi = exports.step = exports.createRunNamespace = exports.getActiveRunContext = void 0;
|
|
11
11
|
exports.wireRigArtifacts = wireRigArtifacts;
|
|
12
12
|
exports.buildArtifactContentDeps = buildArtifactContentDeps;
|
|
13
13
|
exports.createArtifactNamespace = createArtifactNamespace;
|
|
@@ -24,6 +24,13 @@ var artifact_2 = require("./artifact");
|
|
|
24
24
|
Object.defineProperty(exports, "createArtifactApi", { enumerable: true, get: function () { return artifact_2.createArtifactApi; } });
|
|
25
25
|
var artifact_store_1 = require("../telemetry/artifact-store");
|
|
26
26
|
Object.defineProperty(exports, "openArtifactStore", { enumerable: true, get: function () { return artifact_store_1.openArtifactStore; } });
|
|
27
|
+
// PSR-1 — the structure-only assembly-manifest model (spec §2.1). Re-exported
|
|
28
|
+
// here so `export * from "./artifacts"` in index.ts puts it on the public
|
|
29
|
+
// `modelrig` surface for the InferWealth inference + a future MCP write path.
|
|
30
|
+
var manifest_1 = require("../prompts/manifest");
|
|
31
|
+
Object.defineProperty(exports, "validateManifest", { enumerable: true, get: function () { return manifest_1.validateManifest; } });
|
|
32
|
+
Object.defineProperty(exports, "assertValidManifest", { enumerable: true, get: function () { return manifest_1.assertValidManifest; } });
|
|
33
|
+
Object.defineProperty(exports, "parseAuthoringDoc", { enumerable: true, get: function () { return manifest_1.parseAuthoringDoc; } });
|
|
27
34
|
var content_uploader_2 = require("./content-uploader");
|
|
28
35
|
Object.defineProperty(exports, "uploadArtifactContent", { enumerable: true, get: function () { return content_uploader_2.uploadArtifactContent; } });
|
|
29
36
|
var scrub_1 = require("./scrub");
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evidence groundedness grader (call-QA F3 / research §4 G6, spec §5). A
|
|
3
|
+
* DETERMINISTIC bake-off tier that scores a scorecard OUTPUT by whether its
|
|
4
|
+
* evidence quotes are actually present in the replayed INPUT (the transcript /
|
|
5
|
+
* source). A scorer that invents justifications — quotes that never appear in
|
|
6
|
+
* the input — is penalized; this is what stops a model from fabricating
|
|
7
|
+
* evidence to defend a score.
|
|
8
|
+
*
|
|
9
|
+
* Unlike the golden tiers (exact / field-level / llm-judge in quality.ts), this
|
|
10
|
+
* grades the output against the INPUT, not a corrected golden, so it applies
|
|
11
|
+
* whether or not an eval-case `expected` value is present.
|
|
12
|
+
*
|
|
13
|
+
* The scorecard OUTPUT shape (call-QA F2 emit):
|
|
14
|
+
* { criteria: [{ id, score, evidence: [{ quote, turn_id? }], comment }], score, comment }
|
|
15
|
+
*
|
|
16
|
+
* Grounding is a normalized SUBSTRING match against the DECODED input values
|
|
17
|
+
* (inputHaystack): the string leaf values the model was actually shown — never
|
|
18
|
+
* the JSON envelope's escaping or its keys — so a truthful quote spanning escaped
|
|
19
|
+
* chars is grounded and a fabricated quote equal to a key is not. Trivially short
|
|
20
|
+
* quotes never ground (MIN_GROUNDED_QUOTE_LEN).
|
|
21
|
+
*
|
|
22
|
+
* SCOPE NOTE (F3 vs F5): `turn_id` on an evidence entry is carried but NOT
|
|
23
|
+
* hard-validated — turn-scoped matching against the diarized transcript-with-
|
|
24
|
+
* turns artifact is F5's job (spec §7). F3 is substring grounding only.
|
|
25
|
+
*
|
|
26
|
+
* Pure; no I/O. Unit-tested directly (groundedness.test.ts).
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* Pull every `criteria[].evidence[].quote` string from a scorecard output.
|
|
30
|
+
* Tolerant of missing / malformed shapes: a non-object output, a missing or
|
|
31
|
+
* non-array `criteria`, a criterion without an `evidence` array, or an evidence
|
|
32
|
+
* entry without a string `quote` are all skipped rather than throwing. Returns
|
|
33
|
+
* [] for anything it cannot read. Empty (whitespace-only) quotes are dropped —
|
|
34
|
+
* they carry no evidence to ground.
|
|
35
|
+
*/
|
|
36
|
+
export declare function extractEvidenceQuotes(output: unknown): string[];
|
|
37
|
+
/**
|
|
38
|
+
* The fraction in [0,1] of a scorecard's evidence quotes that are grounded in
|
|
39
|
+
* the input, or null when there is nothing to ground.
|
|
40
|
+
*
|
|
41
|
+
* A quote is GROUNDED when its whitespace-collapsed, case-insensitive form is a
|
|
42
|
+
* substring of the DECODED input's whitespace-collapsed, case-insensitive text
|
|
43
|
+
* ({@link inputHaystack} — the string leaf VALUES the model was shown, not the
|
|
44
|
+
* serialized envelope or its keys) AND it is at least {@link MIN_GROUNDED_QUOTE_LEN}
|
|
45
|
+
* chars long (a too-trivial quote is never grounded).
|
|
46
|
+
*
|
|
47
|
+
* Edge cases (explicit, tested):
|
|
48
|
+
* • ZERO evidence quotes across the whole scorecard (none present, or a
|
|
49
|
+
* malformed shape yields none) ⇒ null. There is nothing to ground, so the
|
|
50
|
+
* caller falls through to the next quality tier — NOT a spurious 1.0 or 0.0.
|
|
51
|
+
* • An empty / absent input WITH quotes ⇒ every quote is ungrounded ⇒ 0.
|
|
52
|
+
* • Quotes present but ALL below the min length ⇒ 0, NOT null — they were
|
|
53
|
+
* emitted, they just carry no evidentiary weight (null is only for ZERO
|
|
54
|
+
* emitted quotes).
|
|
55
|
+
* • All quotes grounded ⇒ 1; some grounded ⇒ grounded / total.
|
|
56
|
+
*/
|
|
57
|
+
export declare function groundednessScore(output: unknown, input: unknown): number | null;
|
|
58
|
+
//# sourceMappingURL=groundedness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"groundedness.d.ts","sourceRoot":"","sources":["../../src/bakeoff/groundedness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAoEH;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,EAAE,CAgB/D;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAYhF"}
|