modelrig 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +238 -0
- package/README.md +65 -3
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +36 -7
- package/dist/adapters/attempt-meta-map.d.ts +63 -0
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -0
- package/dist/adapters/attempt-meta-map.js +154 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +25 -2
- package/dist/adapters/gemini.d.ts +39 -0
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +82 -4
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +25 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +25 -2
- package/dist/adapters/openai-stream.d.ts +2 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +23 -3
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +36 -6
- package/dist/artifacts/artifact.d.ts +3 -2
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +35 -9
- package/dist/artifacts/classification.d.ts +27 -0
- package/dist/artifacts/classification.d.ts.map +1 -0
- package/dist/artifacts/classification.js +38 -0
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +10 -1
- package/dist/artifacts/run-context.d.ts +33 -2
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +95 -1
- package/dist/artifacts/scrub.d.ts +26 -0
- package/dist/artifacts/scrub.d.ts.map +1 -1
- package/dist/artifacts/scrub.js +53 -0
- package/dist/artifacts/step.d.ts +9 -0
- package/dist/artifacts/step.d.ts.map +1 -1
- package/dist/artifacts/step.js +9 -0
- package/dist/bakeoff/engine.d.ts +46 -4
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +269 -19
- package/dist/bakeoff/eval-cases.d.ts +5 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +3 -0
- package/dist/bakeoff/judge.d.ts +39 -0
- package/dist/bakeoff/judge.d.ts.map +1 -0
- package/dist/bakeoff/judge.js +102 -0
- package/dist/bakeoff/quality.d.ts +44 -0
- package/dist/bakeoff/quality.d.ts.map +1 -0
- package/dist/bakeoff/quality.js +126 -0
- package/dist/bakeoff/reference.d.ts +34 -0
- package/dist/bakeoff/reference.d.ts.map +1 -0
- package/dist/bakeoff/reference.js +68 -0
- package/dist/bakeoff/replay.d.ts +25 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +65 -2
- package/dist/bakeoff/report.d.ts +8 -1
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +83 -12
- package/dist/bakeoff/samples.d.ts +35 -2
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +67 -6
- package/dist/bakeoff/stats.d.ts +17 -1
- package/dist/bakeoff/stats.d.ts.map +1 -1
- package/dist/bakeoff/stats.js +67 -0
- package/dist/bakeoff/types.d.ts +76 -5
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/bakeoff/types.js +4 -3
- package/dist/cli-query.d.ts +14 -0
- package/dist/cli-query.d.ts.map +1 -0
- package/dist/cli-query.js +59 -0
- package/dist/cli.d.ts +2 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +47 -24
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +14 -0
- package/dist/detect/detector.d.ts +82 -0
- package/dist/detect/detector.d.ts.map +1 -0
- package/dist/detect/detector.js +23 -0
- package/dist/index.d.ts +34 -23
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +49 -12
- package/dist/query/analytics.d.ts +84 -0
- package/dist/query/analytics.d.ts.map +1 -0
- package/dist/query/analytics.js +280 -0
- package/dist/query/columns.d.ts +59 -0
- package/dist/query/columns.d.ts.map +1 -0
- package/dist/query/columns.js +89 -0
- package/dist/query/grammar.d.ts +74 -7
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +179 -71
- package/dist/query/saved.d.ts +26 -0
- package/dist/query/saved.d.ts.map +1 -0
- package/dist/query/saved.js +51 -0
- package/dist/registry/select.d.ts +59 -0
- package/dist/registry/select.d.ts.map +1 -1
- package/dist/registry/select.js +66 -0
- package/dist/routes/loader-warnings.d.ts +28 -0
- package/dist/routes/loader-warnings.d.ts.map +1 -0
- package/dist/routes/loader-warnings.js +56 -0
- package/dist/routes/loader.d.ts +1 -11
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +54 -31
- package/dist/routes/objective-loader.d.ts +35 -0
- package/dist/routes/objective-loader.d.ts.map +1 -0
- package/dist/routes/objective-loader.js +286 -0
- package/dist/run/accept.d.ts +43 -0
- package/dist/run/accept.d.ts.map +1 -0
- package/dist/run/accept.js +65 -0
- package/dist/run/cost.d.ts +13 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +27 -0
- package/dist/run/dispatch.d.ts +24 -1
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +19 -0
- package/dist/run/ladder-key.d.ts +4 -0
- package/dist/run/ladder-key.d.ts.map +1 -1
- package/dist/run/ladder-key.js +3 -0
- package/dist/run/quality-gate.d.ts +39 -0
- package/dist/run/quality-gate.d.ts.map +1 -0
- package/dist/run/quality-gate.js +70 -0
- package/dist/run/raw-defaults.d.ts +52 -0
- package/dist/run/raw-defaults.d.ts.map +1 -0
- package/dist/run/raw-defaults.js +123 -0
- package/dist/run/raw.d.ts +12 -0
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +107 -8
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +22 -1
- package/dist/run/resolve.d.ts +17 -1
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +20 -4
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +12 -5
- package/dist/run/run.d.ts +7 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +54 -69
- package/dist/run/variant.d.ts +43 -0
- package/dist/run/variant.d.ts.map +1 -0
- package/dist/run/variant.js +86 -0
- package/dist/savings/emit.d.ts +33 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +75 -3
- package/dist/settings/index.d.ts +1 -1
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +10 -0
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +17 -0
- package/dist/telemetry/attempt-meta.d.ts +178 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -0
- package/dist/telemetry/attempt-meta.js +319 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +6 -0
- package/dist/telemetry/ingest-protocol.d.ts +11 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +3 -0
- package/dist/telemetry/otel-map.d.ts +10 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -1
- package/dist/telemetry/otel-map.js +64 -0
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +15 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +25 -7
- package/dist/types/failures-teaching.d.ts.map +1 -1
- package/dist/types/failures-teaching.js +20 -0
- package/dist/types/failures.d.ts +1 -1
- package/dist/types/failures.d.ts.map +1 -1
- package/dist/types/failures.js +1 -0
- package/dist/types/provider.d.ts +9 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/quality.d.ts +41 -0
- package/dist/types/quality.d.ts.map +1 -0
- package/dist/types/quality.js +12 -0
- package/dist/types/route.d.ts +120 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +13 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +10 -0
- package/dist/watch/evaluate.d.ts +27 -1
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +13 -0
- package/dist/watch/proposals.d.ts +28 -0
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +51 -0
- package/package.json +1 -1
|
@@ -20,11 +20,60 @@
|
|
|
20
20
|
* v1 wire, zero new rows, zero behaviour change (spec §1.4).
|
|
21
21
|
*/
|
|
22
22
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.resetRunMetaWarningsForTest = resetRunMetaWarningsForTest;
|
|
23
24
|
exports.getActiveRunContext = getActiveRunContext;
|
|
25
|
+
exports.isValidStepKey = isValidStepKey;
|
|
26
|
+
exports.resolveStepKey = resolveStepKey;
|
|
27
|
+
exports.resetStepKeyWarningsForTest = resetStepKeyWarningsForTest;
|
|
24
28
|
exports.createRunNamespace = createRunNamespace;
|
|
25
29
|
const node_async_hooks_1 = require("node:async_hooks");
|
|
26
30
|
const node_crypto_1 = require("node:crypto");
|
|
27
31
|
const otel_map_1 = require("../telemetry/otel-map");
|
|
32
|
+
const attempt_meta_1 = require("../telemetry/attempt-meta");
|
|
33
|
+
const attempt_meta_2 = require("../telemetry/attempt-meta");
|
|
34
|
+
/** Once-per-process latch for the run-envelope sanitize warning (Ruling A,
|
|
35
|
+
* PR #262). Mirrors {@link warnedStepKeys}: a dropped field is named ONCE, so a
|
|
36
|
+
* mislabelled `actor` teaches without spamming the run loop. */
|
|
37
|
+
const warnedRunMetaFields = new Set();
|
|
38
|
+
/**
|
|
39
|
+
* Build a run's `RunMeta@v1` (spec §3): the caller's envelope with
|
|
40
|
+
* `pipeline_sha` auto-filled from git when unset, then SANITIZED at the same
|
|
41
|
+
* boundary the wire is (Ruling A, PR #262) — every string field must be an
|
|
42
|
+
* enum/identifier/hash (≤128 chars, no whitespace), so a free-text `actor`
|
|
43
|
+
* ("Jane Doe") is DROPPED, counted, and named in one once-per-process warning,
|
|
44
|
+
* never smuggled into `runs.meta`. Returns null only when the caller supplied no
|
|
45
|
+
* meta AND no sha resolves — a plain `run.start()` in a git tree still stamps the
|
|
46
|
+
* sha. NEVER throws (resolvePipelineSha + sanitizeMetaForWire are both safe); the
|
|
47
|
+
* recorded run row is byte-identical to the un-sanitized path for a clean envelope.
|
|
48
|
+
*/
|
|
49
|
+
function buildRunMeta(meta) {
|
|
50
|
+
const sha = meta?.pipeline_sha ?? (0, attempt_meta_2.resolvePipelineSha)();
|
|
51
|
+
if (meta === undefined && sha === undefined)
|
|
52
|
+
return null;
|
|
53
|
+
const merged = {
|
|
54
|
+
v: 1,
|
|
55
|
+
...(meta ?? {}),
|
|
56
|
+
...(sha !== undefined ? { pipeline_sha: sha } : {}),
|
|
57
|
+
};
|
|
58
|
+
const { meta: cleaned, dropped } = (0, attempt_meta_1.sanitizeMetaForWire)(merged);
|
|
59
|
+
if (dropped > 0) {
|
|
60
|
+
// RunMeta is flat, so a dropped string is a named top-level field (e.g.
|
|
61
|
+
// `actor`). Name each once — instrumentation teaches, never throws.
|
|
62
|
+
for (const key of Object.keys(merged)) {
|
|
63
|
+
if (typeof merged[key] === "string" && (cleaned === undefined || !(key in cleaned)) && !warnedRunMetaFields.has(key)) {
|
|
64
|
+
warnedRunMetaFields.add(key);
|
|
65
|
+
const tip = key === "actor" ? " `actor` must be an opaque identifier, not a display name." : "";
|
|
66
|
+
console.warn(`[modelrig run] RunMeta.${key} is not metadata-safe (an enum/identifier/hash: ≤128 chars, no ` +
|
|
67
|
+
`whitespace) — dropped from runs.meta.${tip}`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return (cleaned ?? { v: 1 });
|
|
72
|
+
}
|
|
73
|
+
/** Test-only: clear the once-per-process RunMeta-sanitize warning latch. */
|
|
74
|
+
function resetRunMetaWarningsForTest() {
|
|
75
|
+
warnedRunMetaFields.clear();
|
|
76
|
+
}
|
|
28
77
|
const storage = new node_async_hooks_1.AsyncLocalStorage();
|
|
29
78
|
/** The active run context, or null when none is set (namespace disabled, or no
|
|
30
79
|
* run.start on this async path). Read by the run loop's auto-step seam. */
|
|
@@ -32,6 +81,38 @@ function getActiveRunContext() {
|
|
|
32
81
|
const ctx = storage.getStore();
|
|
33
82
|
return ctx !== undefined && !ctx.ended ? ctx : null;
|
|
34
83
|
}
|
|
84
|
+
/** Valid ground-truth step key (G1 spec §3): a caller-supplied `stepKey` (either
|
|
85
|
+
* lane) or a `tags.step` used as one. Auto-minted keys (`label#seq`) may contain
|
|
86
|
+
* a `/` from a route slug or `provider/model` label and are NOT validated — the
|
|
87
|
+
* regex gates only what a customer types. */
|
|
88
|
+
const STEP_KEY_PATTERN = /^[A-Za-z0-9_.:#-]{1,128}$/;
|
|
89
|
+
/** True when `key` is a valid caller-supplied step key (spec §3). */
|
|
90
|
+
function isValidStepKey(key) {
|
|
91
|
+
return STEP_KEY_PATTERN.test(key);
|
|
92
|
+
}
|
|
93
|
+
const warnedStepKeys = new Set();
|
|
94
|
+
/** Resolve a ground-truth step key for either lane (spec §3): an explicit
|
|
95
|
+
* caller-supplied key wins, then the `tags.step` convention, else a freshly
|
|
96
|
+
* minted auto key. A caller key is validated `[A-Za-z0-9_.:#-]{1,128}`; an
|
|
97
|
+
* invalid one is warned (once per bad value) and DROPPED to the auto key —
|
|
98
|
+
* instrumentation never throws a telemetry-label error into customer code. */
|
|
99
|
+
function resolveStepKey(explicit, tagStep, auto) {
|
|
100
|
+
const candidate = explicit ?? tagStep;
|
|
101
|
+
if (candidate === undefined)
|
|
102
|
+
return auto();
|
|
103
|
+
if (isValidStepKey(candidate))
|
|
104
|
+
return candidate;
|
|
105
|
+
if (!warnedStepKeys.has(candidate)) {
|
|
106
|
+
warnedStepKeys.add(candidate);
|
|
107
|
+
console.warn(`[modelrig artifacts] invalid stepKey "${candidate}" (allowed [A-Za-z0-9_.:#-], 1-128 chars) — ` +
|
|
108
|
+
"grouped under an auto-generated key instead.");
|
|
109
|
+
}
|
|
110
|
+
return auto();
|
|
111
|
+
}
|
|
112
|
+
/** Test-only: clear the once-per-process invalid-stepKey warning latch. */
|
|
113
|
+
function resetStepKeyWarningsForTest() {
|
|
114
|
+
warnedStepKeys.clear();
|
|
115
|
+
}
|
|
35
116
|
function warnOnce(scope, err) {
|
|
36
117
|
console.warn(`[modelrig artifacts] ${scope} buffer write failed (dropped, run unaffected): ` +
|
|
37
118
|
`${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -67,6 +148,9 @@ function createRunNamespace(deps) {
|
|
|
67
148
|
const attemptByStep = new Map();
|
|
68
149
|
const loaded = new Set();
|
|
69
150
|
let stepSeq = 0;
|
|
151
|
+
// The ambient step key (G1): the last step minted or emitted in this run,
|
|
152
|
+
// which an artifact.save without an explicit stepKey attaches to.
|
|
153
|
+
let lastStepKey = null;
|
|
70
154
|
return {
|
|
71
155
|
runId,
|
|
72
156
|
traceId,
|
|
@@ -84,9 +168,16 @@ function createRunNamespace(deps) {
|
|
|
84
168
|
},
|
|
85
169
|
nextStepKey(label) {
|
|
86
170
|
stepSeq += 1;
|
|
87
|
-
|
|
171
|
+
lastStepKey = `${label}#${stepSeq}`;
|
|
172
|
+
return lastStepKey;
|
|
173
|
+
},
|
|
174
|
+
currentStepKey() {
|
|
175
|
+
return lastStepKey;
|
|
88
176
|
},
|
|
89
177
|
emitGroundTruthStep(input) {
|
|
178
|
+
// A caller-supplied key on the raw lane never minted through nextStepKey;
|
|
179
|
+
// record it as the ambient step so artifacts saved around this call join it.
|
|
180
|
+
lastStepKey = input.stepKey;
|
|
90
181
|
const attemptNo = (attemptByStep.get(input.stepKey) ?? 0) + 1;
|
|
91
182
|
attemptByStep.set(input.stepKey, attemptNo);
|
|
92
183
|
try {
|
|
@@ -122,6 +213,9 @@ function createRunNamespace(deps) {
|
|
|
122
213
|
startedAt: new Date().toISOString(),
|
|
123
214
|
endedAt: null,
|
|
124
215
|
status: "running",
|
|
216
|
+
// WI-1 (spec §3): the run envelope, git-sha auto-filled. endContext spreads
|
|
217
|
+
// ...ctx.startRow, so the terminal projection re-exports the same meta.
|
|
218
|
+
meta: buildRunMeta(opts.meta),
|
|
125
219
|
};
|
|
126
220
|
const traceId = opts.traceparent !== undefined
|
|
127
221
|
? ((0, otel_map_1.parseTraceparent)(opts.traceparent)?.traceId ?? null)
|
|
@@ -20,7 +20,17 @@
|
|
|
20
20
|
* `detect` result as a guarantee of no PII.
|
|
21
21
|
*
|
|
22
22
|
* NEVER logs a matched value. Findings are `{type, count}` only.
|
|
23
|
+
*
|
|
24
|
+
* ── SHARED DETECTOR INTERFACE (screening review A1, 2026-08-22) ─────────────
|
|
25
|
+
* The same patterns are ALSO exposed as span-producing `Detector`s
|
|
26
|
+
* (`detectSpans`, `scrubDetectors`) so the mask (CS-1) and screen (SC-1) lanes
|
|
27
|
+
* reuse this ONE corpus instead of twinning it (spec docs/ModelRig/screening §3;
|
|
28
|
+
* S1: extend, don't twin). `scrubContent`'s behaviour is unchanged — the span
|
|
29
|
+
* exports are additive and share the same DETECTORS table, so a matched value
|
|
30
|
+
* never leaves the process there either (a span is offsets + type + confidence,
|
|
31
|
+
* never a value).
|
|
23
32
|
*/
|
|
33
|
+
import type { DetectionSpan, Detector } from "../detect/detector";
|
|
24
34
|
export type ScrubMode = "off" | "detect" | "redact" | "block";
|
|
25
35
|
/** A finding is a COUNT by type — never a value (the load-bearing invariant). */
|
|
26
36
|
export interface ScrubFinding {
|
|
@@ -57,4 +67,20 @@ export declare function luhnValid(digits: string): boolean;
|
|
|
57
67
|
* Never returns or logs a matched value.
|
|
58
68
|
*/
|
|
59
69
|
export declare function scrubContent(text: string, mode: ScrubMode): ScrubResult;
|
|
70
|
+
/**
|
|
71
|
+
* Detect scrub v1 spans across ALL patterns, sorted by start offset. Deterministic
|
|
72
|
+
* (confidence 1). Offsets index into the exact `text` passed. Overlap resolution
|
|
73
|
+
* (e.g. PAN vs phone) is the CONSUMER's job (CS-1 §4 "earlier stages win") — this
|
|
74
|
+
* reports every pattern's raw hits, never a value.
|
|
75
|
+
*/
|
|
76
|
+
export declare function detectSpans(text: string): DetectionSpan[];
|
|
77
|
+
/**
|
|
78
|
+
* The scrub v1 patterns presented as the shared `Detector` contract
|
|
79
|
+
* (`src/detect/detector.ts`) — one detector per type, `kind: "deterministic"`,
|
|
80
|
+
* `confidence: 1`. The mask (CS-1) and screen (SC-1) lanes consume these so all
|
|
81
|
+
* three policies (`mask` / `scrub` / `screen`) run over ONE detection engine.
|
|
82
|
+
* The `source`/`tier` on the input are ignored by the deterministic PII patterns
|
|
83
|
+
* (they scan any tier); a tier-aware detector would branch on them.
|
|
84
|
+
*/
|
|
85
|
+
export declare function scrubDetectors(): Detector[];
|
|
60
86
|
//# sourceMappingURL=scrub.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrub.d.ts","sourceRoot":"","sources":["../../src/artifacts/scrub.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"scrub.d.ts","sourceRoot":"","sources":["../../src/artifacts/scrub.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAkB,aAAa,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAElF,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1E,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C;iEAC6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;0EAG0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;CAC5C;AAED;;kFAEkF;AAClF,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,GAAG,YAAY,CAEjF;AAaD,8EAA8E;AAC9E,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAejD;AA0CD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,WAAW,CA6CvE;AAqBD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CAIzD;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,IAAI,QAAQ,EAAE,CAS3C"}
|
package/dist/artifacts/scrub.js
CHANGED
|
@@ -21,11 +21,22 @@
|
|
|
21
21
|
* `detect` result as a guarantee of no PII.
|
|
22
22
|
*
|
|
23
23
|
* NEVER logs a matched value. Findings are `{type, count}` only.
|
|
24
|
+
*
|
|
25
|
+
* ── SHARED DETECTOR INTERFACE (screening review A1, 2026-08-22) ─────────────
|
|
26
|
+
* The same patterns are ALSO exposed as span-producing `Detector`s
|
|
27
|
+
* (`detectSpans`, `scrubDetectors`) so the mask (CS-1) and screen (SC-1) lanes
|
|
28
|
+
* reuse this ONE corpus instead of twinning it (spec docs/ModelRig/screening §3;
|
|
29
|
+
* S1: extend, don't twin). `scrubContent`'s behaviour is unchanged — the span
|
|
30
|
+
* exports are additive and share the same DETECTORS table, so a matched value
|
|
31
|
+
* never leaves the process there either (a span is offsets + type + confidence,
|
|
32
|
+
* never a value).
|
|
24
33
|
*/
|
|
25
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
35
|
exports.summarizeScrub = summarizeScrub;
|
|
27
36
|
exports.luhnValid = luhnValid;
|
|
28
37
|
exports.scrubContent = scrubContent;
|
|
38
|
+
exports.detectSpans = detectSpans;
|
|
39
|
+
exports.scrubDetectors = scrubDetectors;
|
|
29
40
|
/** Build the persisted summary from a scrub result. The only producer of the
|
|
30
41
|
* `artifacts.scrub` value, so the counts-not-values invariant lives in one
|
|
31
42
|
* place — it simply forwards `ScrubResult.findings`, which are already counts. */
|
|
@@ -137,3 +148,45 @@ function scrubContent(text, mode) {
|
|
|
137
148
|
// detect (or block with no findings): upload verbatim, report counts.
|
|
138
149
|
return { findings, text, blocked: false };
|
|
139
150
|
}
|
|
151
|
+
// ── Shared Detector interface (screening review A1) ─────────────────────────
|
|
152
|
+
// Span-producing views over the SAME `DETECTORS` table. Additive: `scrubContent`
|
|
153
|
+
// above is untouched, and these never surface a matched value (a span is offsets
|
|
154
|
+
// + type + confidence, per the load-bearing invariant).
|
|
155
|
+
/** Spans for one pattern in `text` — offsets + type + confidence 1. A fresh
|
|
156
|
+
* RegExp per call keeps the /g `lastIndex` from leaking across calls. */
|
|
157
|
+
function spansForPattern(det, text) {
|
|
158
|
+
const re = new RegExp(det.re.source, det.re.flags);
|
|
159
|
+
const spans = [];
|
|
160
|
+
for (const m of text.matchAll(re)) {
|
|
161
|
+
const value = m[0];
|
|
162
|
+
if (det.accept && !det.accept(value))
|
|
163
|
+
continue;
|
|
164
|
+
const start = m.index ?? 0;
|
|
165
|
+
spans.push({ start, end: start + value.length, type: det.type, confidence: 1 });
|
|
166
|
+
}
|
|
167
|
+
return spans;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Detect scrub v1 spans across ALL patterns, sorted by start offset. Deterministic
|
|
171
|
+
* (confidence 1). Offsets index into the exact `text` passed. Overlap resolution
|
|
172
|
+
* (e.g. PAN vs phone) is the CONSUMER's job (CS-1 §4 "earlier stages win") — this
|
|
173
|
+
* reports every pattern's raw hits, never a value.
|
|
174
|
+
*/
|
|
175
|
+
function detectSpans(text) {
|
|
176
|
+
return DETECTORS.flatMap((det) => spansForPattern(det, text)).sort((a, b) => a.start - b.start || a.end - b.end);
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* The scrub v1 patterns presented as the shared `Detector` contract
|
|
180
|
+
* (`src/detect/detector.ts`) — one detector per type, `kind: "deterministic"`,
|
|
181
|
+
* `confidence: 1`. The mask (CS-1) and screen (SC-1) lanes consume these so all
|
|
182
|
+
* three policies (`mask` / `scrub` / `screen`) run over ONE detection engine.
|
|
183
|
+
* The `source`/`tier` on the input are ignored by the deterministic PII patterns
|
|
184
|
+
* (they scan any tier); a tier-aware detector would branch on them.
|
|
185
|
+
*/
|
|
186
|
+
function scrubDetectors() {
|
|
187
|
+
return DETECTORS.map((det) => ({
|
|
188
|
+
id: `scrub:${det.type}@v1`,
|
|
189
|
+
kind: "deterministic",
|
|
190
|
+
detect: (input) => Promise.resolve(spansForPattern(det, input.text)),
|
|
191
|
+
}));
|
|
192
|
+
}
|
package/dist/artifacts/step.d.ts
CHANGED
|
@@ -14,6 +14,15 @@
|
|
|
14
14
|
* active run context (chunk B attributes artifact save/load lineage to it) and
|
|
15
15
|
* runs the body; it does not persist a row until it has a valid reference.
|
|
16
16
|
*
|
|
17
|
+
* ── AMBIENT STEP KEY (G1) ─────────────────────────────────────────────────
|
|
18
|
+
* Entering a `@step` / `step()` scope MINTS a unique key (`label#n`) and sets it
|
|
19
|
+
* as the run context's AMBIENT step. An `artifact.save` with no explicit
|
|
20
|
+
* `stepKey` attaches to that ambient key — not only for saves INSIDE the scope,
|
|
21
|
+
* but for every save AFTER it too, until the next step mints a new ambient key
|
|
22
|
+
* (the ambient key is a running "most recent step", not a lexical bracket). So a
|
|
23
|
+
* post-step save lands on the step just run; to detach it, open the next step or
|
|
24
|
+
* pass an explicit `stepKey`.
|
|
25
|
+
*
|
|
17
26
|
* Instrumentation never changes the wrapped function's return value, and never
|
|
18
27
|
* throws into it — a disabled or absent run context just runs the body.
|
|
19
28
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../../src/artifacts/step.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../../src/artifacts/step.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH;;;;GAIG;AACH,wBAAgB,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAOnD"}
|
package/dist/artifacts/step.js
CHANGED
|
@@ -15,6 +15,15 @@
|
|
|
15
15
|
* active run context (chunk B attributes artifact save/load lineage to it) and
|
|
16
16
|
* runs the body; it does not persist a row until it has a valid reference.
|
|
17
17
|
*
|
|
18
|
+
* ── AMBIENT STEP KEY (G1) ─────────────────────────────────────────────────
|
|
19
|
+
* Entering a `@step` / `step()` scope MINTS a unique key (`label#n`) and sets it
|
|
20
|
+
* as the run context's AMBIENT step. An `artifact.save` with no explicit
|
|
21
|
+
* `stepKey` attaches to that ambient key — not only for saves INSIDE the scope,
|
|
22
|
+
* but for every save AFTER it too, until the next step mints a new ambient key
|
|
23
|
+
* (the ambient key is a running "most recent step", not a lexical bracket). So a
|
|
24
|
+
* post-step save lands on the step just run; to detach it, open the next step or
|
|
25
|
+
* pass an explicit `stepKey`.
|
|
26
|
+
*
|
|
18
27
|
* Instrumentation never changes the wrapped function's return value, and never
|
|
19
28
|
* throws into it — a disabled or absent run context just runs the body.
|
|
20
29
|
*/
|
package/dist/bakeoff/engine.d.ts
CHANGED
|
@@ -12,8 +12,14 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import type { CaptureStore } from "../telemetry/capture";
|
|
14
14
|
import type { RunDeps } from "../run/run";
|
|
15
|
+
import type { CandidateSpec, ObjectiveMode, RouteBundle } from "../types/route";
|
|
16
|
+
import type { QualityGate } from "../types/quality";
|
|
15
17
|
import type { ArmSamples, KeepOutputsMode } from "./samples";
|
|
16
|
-
import type { BakeoffArtifact, VariantMetrics } from "./types";
|
|
18
|
+
import type { BakeoffArtifact, BakeoffObjective, VariantMetrics } from "./types";
|
|
19
|
+
/** The arm name carrying the reference (ceiling) metrics. Reserved — a user
|
|
20
|
+
* variant may not be named this (they cannot start a name with "__"; and it is
|
|
21
|
+
* never in `req.variants`). */
|
|
22
|
+
export declare const REFERENCE_ARM = "__reference__";
|
|
17
23
|
/** Sink for retained review samples (addendum A3). The engine assembles the
|
|
18
24
|
* arms to persist (incumbent + winner, or all); the caller — createRig with
|
|
19
25
|
* the service-role config — does the truncation + PostgREST insert. Keeps
|
|
@@ -34,11 +40,40 @@ export interface BakeoffRequest {
|
|
|
34
40
|
* winning challenger; "all" = every arm. Undefined = keep nothing (the
|
|
35
41
|
* default pure-metrics path). Requires a samplesSink to persist. */
|
|
36
42
|
readonly keepOutputs?: KeepOutputsMode;
|
|
43
|
+
/** RB-1: objective-mode override (CLI `--objective`). Absent ⇒ the route's
|
|
44
|
+
* `objective.mode`, else quality-first (spec §3 default). */
|
|
45
|
+
readonly objectiveMode?: ObjectiveMode;
|
|
46
|
+
/** RB-1: resolver for `reference: auto` → a concrete declared candidate. The
|
|
47
|
+
* rig injects a registry-backed resolver (bakeoff/reference.ts); absent ⇒ an
|
|
48
|
+
* `auto` reference cannot be resolved and no reference arm runs. */
|
|
49
|
+
readonly resolveReferenceAuto?: (route: RouteBundle) => CandidateSpec | null;
|
|
50
|
+
/** RB-1: eval-case expected values keyed by captureId (from-eval-cases path);
|
|
51
|
+
* feeds quality tier 1. */
|
|
52
|
+
readonly expected?: ReadonlyMap<string, {
|
|
53
|
+
readonly value: unknown;
|
|
54
|
+
}>;
|
|
55
|
+
/** Q1 quality gate (spec §4.2): scored on every replayed sample to populate
|
|
56
|
+
* valueAccuracy + the quality pass rate, unlocking computeProposal's cost-first
|
|
57
|
+
* third clause. A function, so it rides `rig.bakeoff` (programmatic) only — the
|
|
58
|
+
* `modelrig bakeoff` CLI cannot carry one until Q2's declared judge. When a
|
|
59
|
+
* route also declares a `bakeoff.judge`, the SDK gate wins the value tier (both
|
|
60
|
+
* are caller-quality signals; F5 deterministic checks still run first). */
|
|
61
|
+
readonly qualityGate?: QualityGate;
|
|
37
62
|
}
|
|
38
63
|
export declare function runBakeoff(deps: RunDeps, captures: CaptureStore, req: BakeoffRequest, samplesSink?: BakeoffSamplesSink): Promise<BakeoffArtifact>;
|
|
39
|
-
/**
|
|
40
|
-
*
|
|
41
|
-
export
|
|
64
|
+
/** Objective context for the gate (RB-1). Absent ⇒ the legacy cost-first path
|
|
65
|
+
* (byte-identical to pre-RB-1, so the existing goldens hold). */
|
|
66
|
+
export interface ProposalContext {
|
|
67
|
+
readonly objective?: BakeoffObjective;
|
|
68
|
+
/** The reference arm's name — excluded from the challenger set. */
|
|
69
|
+
readonly referenceVariant?: string;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Winner among gate-passing challengers; null when nobody beats the incumbent.
|
|
73
|
+
* Exported for direct unit testing. `ctx` absent OR `ctx.objective` absent ⇒ the
|
|
74
|
+
* legacy cost-first gate (goldens). Otherwise per the objective's mode.
|
|
75
|
+
*/
|
|
76
|
+
export declare function computeProposal(requested: readonly string[], metrics: readonly VariantMetrics[], ctx?: ProposalContext): BakeoffArtifact["proposal"];
|
|
42
77
|
export interface BakeoffMirrorConfig {
|
|
43
78
|
readonly supabaseUrl: string;
|
|
44
79
|
readonly serviceRoleKey: string;
|
|
@@ -49,4 +84,11 @@ export interface BakeoffMirrorConfig {
|
|
|
49
84
|
* Same PostgREST transport pattern as the telemetry exporter; captures have
|
|
50
85
|
* no cloud surface — ONLY aggregate metrics travel. */
|
|
51
86
|
export declare function mirrorBakeoff(artifact: BakeoffArtifact, config: BakeoffMirrorConfig): Promise<void>;
|
|
87
|
+
/**
|
|
88
|
+
* The proposal-evidence jsonb for a bake-off artifact (near-optimal spec §4.6) —
|
|
89
|
+
* the winning variant, the RB-1 objective/reference/regret/quality_metric, and a
|
|
90
|
+
* per-arm matrix (with per-arm regret). Built here so the CLI stays thin; the PR
|
|
91
|
+
* body reads these fields back off `proposal.evidence`. ZERO DDL — freeform jsonb.
|
|
92
|
+
*/
|
|
93
|
+
export declare function bakeoffEvidence(artifact: BakeoffArtifact, spentUsd: number): Record<string, unknown>;
|
|
52
94
|
//# sourceMappingURL=engine.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAM9F,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAmC,cAAc,EAAE,MAAM,SAAS,CAAC;AAElH;;+BAE+B;AAC/B,eAAO,MAAM,aAAa,kBAAkB,CAAC;AAE7C;;;gFAGgF;AAChF,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgBpB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC;iEAC6D;IAC7D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E;+BAC2B;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;;;;+EAK2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAsBD,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,cAAc,EACnB,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,eAAe,CAAC,CAiJ1B;AAsCD;iEACiE;AACjE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,mEAAmE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAoBD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,GAAG,CAAC,EAAE,eAAe,GACpB,eAAe,CAAC,UAAU,CAAC,CA4G7B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED;;uDAEuD;AACvD,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CA2BpG"}
|