modelrig 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +107 -0
- package/README.md +63 -0
- package/dist/bakeoff/disagreement.d.ts +93 -0
- package/dist/bakeoff/disagreement.d.ts.map +1 -0
- package/dist/bakeoff/disagreement.js +140 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +58 -2
- package/dist/bakeoff/pairwise.d.ts +102 -0
- package/dist/bakeoff/pairwise.d.ts.map +1 -0
- package/dist/bakeoff/pairwise.js +214 -0
- package/dist/bakeoff/replay.d.ts +6 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +6 -2
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +46 -0
- package/dist/bakeoff/types.d.ts +24 -0
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +9 -9
- package/dist/mask/apply.d.ts +80 -0
- package/dist/mask/apply.d.ts.map +1 -0
- package/dist/mask/apply.js +228 -0
- package/dist/mask/detect.d.ts +38 -0
- package/dist/mask/detect.d.ts.map +1 -0
- package/dist/mask/detect.js +167 -0
- package/dist/mask/index.d.ts +21 -0
- package/dist/mask/index.d.ts.map +1 -0
- package/dist/mask/index.js +54 -0
- package/dist/mask/mask-policy-cli.d.ts +11 -0
- package/dist/mask/mask-policy-cli.d.ts.map +1 -0
- package/dist/mask/mask-policy-cli.js +33 -0
- package/dist/mask/policy.d.ts +38 -0
- package/dist/mask/policy.d.ts.map +1 -0
- package/dist/mask/policy.js +187 -0
- package/dist/mask/proof.d.ts +54 -0
- package/dist/mask/proof.d.ts.map +1 -0
- package/dist/mask/proof.js +120 -0
- package/dist/mask/receipt.d.ts +30 -0
- package/dist/mask/receipt.d.ts.map +1 -0
- package/dist/mask/receipt.js +49 -0
- package/dist/mask/seam.d.ts +53 -0
- package/dist/mask/seam.d.ts.map +1 -0
- package/dist/mask/seam.js +120 -0
- package/dist/mask/types.d.ts +81 -0
- package/dist/mask/types.d.ts.map +1 -0
- package/dist/mask/types.js +10 -0
- package/dist/mask/vault.d.ts +50 -0
- package/dist/mask/vault.d.ts.map +1 -0
- package/dist/mask/vault.js +81 -0
- package/dist/receipts/types.d.ts +65 -0
- package/dist/receipts/types.d.ts.map +1 -0
- package/dist/receipts/types.js +134 -0
- package/dist/routes/loader-warnings.d.ts +8 -0
- package/dist/routes/loader-warnings.d.ts.map +1 -1
- package/dist/routes/loader-warnings.js +23 -0
- package/dist/routes/loader.d.ts +1 -1
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +24 -25
- package/dist/routes/mask-loader.d.ts +18 -0
- package/dist/routes/mask-loader.d.ts.map +1 -0
- package/dist/routes/mask-loader.js +39 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +51 -17
- package/dist/routes/screen-loader.d.ts +23 -0
- package/dist/routes/screen-loader.d.ts.map +1 -0
- package/dist/routes/screen-loader.js +79 -0
- package/dist/routes/template.d.ts +10 -0
- package/dist/routes/template.d.ts.map +1 -1
- package/dist/routes/template.js +10 -3
- package/dist/run/accept.d.ts +4 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +2 -0
- package/dist/run/dispatch.d.ts +35 -0
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +33 -0
- package/dist/run/grounding.d.ts +47 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +113 -6
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +6 -1
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +12 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +19 -16
- package/dist/screen/provenance.d.ts +59 -0
- package/dist/screen/provenance.d.ts.map +1 -0
- package/dist/screen/provenance.js +104 -0
- package/dist/screen/types.d.ts +71 -0
- package/dist/screen/types.d.ts.map +1 -0
- package/dist/screen/types.js +9 -0
- package/dist/status-cli.d.ts +11 -0
- package/dist/status-cli.d.ts.map +1 -0
- package/dist/status-cli.js +21 -0
- package/dist/status-runs.d.ts +38 -0
- package/dist/status-runs.d.ts.map +1 -0
- package/dist/status-runs.js +80 -0
- package/dist/status.d.ts +7 -2
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +11 -2
- package/dist/telemetry/ingest-protocol.d.ts +8 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +11 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +12 -3
- package/dist/telemetry/supabase-exporter.d.ts +64 -0
- package/dist/telemetry/supabase-exporter.d.ts.map +1 -0
- package/dist/telemetry/supabase-exporter.js +211 -0
- package/dist/types/provider.d.ts +11 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +9 -0
- package/dist/types/route.d.ts +55 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +7 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +13 -0
- package/dist/verify/types.d.ts +6 -0
- package/dist/verify/types.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +131 -0
- package/docs/how-it-fits.md +34 -0
- package/docs/instrumentation.md +221 -0
- package/docs/llms.txt +276 -0
- package/docs/migration-playbook.md +512 -0
- package/docs/quickstart.md +333 -0
- package/package.json +18 -2
- package/receipts.d.ts +12 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Async batch exporter: SQLite telemetry → ModelRig Supabase project.
|
|
4
|
+
* Spec §5 (cloud sink) + §6 criteria 8–9. Fire-and-forget from the run loop:
|
|
5
|
+
* the inference path NEVER blocks on, or fails because of, this sink.
|
|
6
|
+
* At-least-once delivery with idempotent ids; unexported rows queue locally in
|
|
7
|
+
* SQLite (the inferences table itself, behind a rowid watermark) and flush on
|
|
8
|
+
* recovery; watermark bookkeeping per source instance.
|
|
9
|
+
*
|
|
10
|
+
* Transport is raw PostgREST over fetch (dependency-light, service-role key).
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.createSupabaseExporter = createSupabaseExporter;
|
|
14
|
+
const constants_1 = require("../tenancy/constants");
|
|
15
|
+
const DEFAULT_BATCH_SIZE = 200;
|
|
16
|
+
const DEFAULT_INTERVAL_MS = 10000;
|
|
17
|
+
/** After a sink failure, skip this many intervals, doubling per consecutive
|
|
18
|
+
* failure (simple exponential backoff, capped). */
|
|
19
|
+
const BACKOFF_MAX_SKIPS = 32;
|
|
20
|
+
function toRemote(row, tenancy) {
|
|
21
|
+
return {
|
|
22
|
+
...tenancy,
|
|
23
|
+
id: row.id,
|
|
24
|
+
ts: row.ts,
|
|
25
|
+
route: row.route,
|
|
26
|
+
route_version: row.routeVersion,
|
|
27
|
+
provider: row.provider,
|
|
28
|
+
model: row.model,
|
|
29
|
+
requested_tier: row.requestedTier,
|
|
30
|
+
served_tier: row.servedTier,
|
|
31
|
+
tokens_in: row.tokensIn,
|
|
32
|
+
tokens_out: row.tokensOut,
|
|
33
|
+
tokens_cached: row.tokensCached,
|
|
34
|
+
cost_estimate_usd: row.costEstimateUsd,
|
|
35
|
+
latency_ms: row.latencyMs,
|
|
36
|
+
ttfb_ms: row.ttfbMs,
|
|
37
|
+
failure_class: row.failureClass,
|
|
38
|
+
attempts: JSON.parse(row.attemptsJson),
|
|
39
|
+
tags: JSON.parse(row.tagsJson),
|
|
40
|
+
fingerprint_cell: row.fingerprintCell,
|
|
41
|
+
served_variant: row.servedVariant,
|
|
42
|
+
repaired_by: row.repairedBy,
|
|
43
|
+
pricing_missing: row.pricingMissing,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function feedbackToRemote(row, tenancy) {
|
|
47
|
+
return {
|
|
48
|
+
org_id: tenancy.org_id,
|
|
49
|
+
project_id: tenancy.project_id,
|
|
50
|
+
id: row.id,
|
|
51
|
+
ts: row.ts,
|
|
52
|
+
inference_id: row.inferenceId,
|
|
53
|
+
episode_key: row.episodeKey,
|
|
54
|
+
verdict: row.verdict,
|
|
55
|
+
score: row.score,
|
|
56
|
+
note: row.note,
|
|
57
|
+
source: row.source,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function createSupabaseExporter(store, config) {
|
|
61
|
+
const batchSize = config.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
62
|
+
const intervalMs = config.intervalMs ?? DEFAULT_INTERVAL_MS;
|
|
63
|
+
const fetchFn = config.fetchFn ?? fetch;
|
|
64
|
+
const base = config.supabaseUrl.replace(/\/$/, "");
|
|
65
|
+
const tenancy = {
|
|
66
|
+
org_id: config.orgId ?? constants_1.HOUSE_ORG_ID,
|
|
67
|
+
project_id: config.projectId ?? null,
|
|
68
|
+
source_id: config.sourceId,
|
|
69
|
+
};
|
|
70
|
+
let timer = null;
|
|
71
|
+
let flushing = false;
|
|
72
|
+
let skipsRemaining = 0;
|
|
73
|
+
let consecutiveFailures = 0;
|
|
74
|
+
let routesMirrored = false;
|
|
75
|
+
let manifestMirrored = false;
|
|
76
|
+
async function post(table, body, upsert) {
|
|
77
|
+
const response = await fetchFn(`${base}/rest/v1/${table}`, {
|
|
78
|
+
method: "POST",
|
|
79
|
+
headers: {
|
|
80
|
+
"content-type": "application/json",
|
|
81
|
+
apikey: config.serviceRoleKey,
|
|
82
|
+
authorization: `Bearer ${config.serviceRoleKey}`,
|
|
83
|
+
prefer: upsert
|
|
84
|
+
? "resolution=merge-duplicates,return=minimal"
|
|
85
|
+
: "resolution=ignore-duplicates,return=minimal",
|
|
86
|
+
},
|
|
87
|
+
body: JSON.stringify(body),
|
|
88
|
+
});
|
|
89
|
+
if (!response.ok) {
|
|
90
|
+
throw new Error(`sink POST ${table} → ${response.status} ${await response.text().catch(() => "")}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async function flushOnce() {
|
|
94
|
+
// Routes mirror first (idempotent upsert; retried until it succeeds once).
|
|
95
|
+
if (!routesMirrored && config.routes !== undefined && config.routes.length > 0) {
|
|
96
|
+
await post("routes", config.routes.map((r) => ({
|
|
97
|
+
org_id: tenancy.org_id,
|
|
98
|
+
route: r.route,
|
|
99
|
+
version: r.version,
|
|
100
|
+
candidates: r.candidates,
|
|
101
|
+
capture: r.capture ?? false,
|
|
102
|
+
updated_at: new Date().toISOString(),
|
|
103
|
+
})), true);
|
|
104
|
+
routesMirrored = true;
|
|
105
|
+
}
|
|
106
|
+
// Manifest mirror (idempotent upsert, once per process — like routes).
|
|
107
|
+
if (!manifestMirrored && config.manifest !== undefined && config.manifest !== null) {
|
|
108
|
+
await post("rig_manifests", [
|
|
109
|
+
{
|
|
110
|
+
org_id: tenancy.org_id,
|
|
111
|
+
source_id: config.sourceId,
|
|
112
|
+
name: config.manifest.name,
|
|
113
|
+
dimensions: config.manifest.dimensions,
|
|
114
|
+
contribution: config.manifest.contribution,
|
|
115
|
+
// F1: names only — route, provider/model, and the env var to set.
|
|
116
|
+
serveability: (config.serveability ?? []).map((w) => ({
|
|
117
|
+
route: w.route,
|
|
118
|
+
inert: w.inert.map((c) => ({ provider: c.provider, model: c.model })),
|
|
119
|
+
serveable_count: w.serveableCount,
|
|
120
|
+
fix_hint: w.fixHint,
|
|
121
|
+
})),
|
|
122
|
+
updated_at: new Date().toISOString(),
|
|
123
|
+
},
|
|
124
|
+
], true);
|
|
125
|
+
manifestMirrored = true;
|
|
126
|
+
}
|
|
127
|
+
// Inferences past the watermark, in batches. Watermark advances ONLY
|
|
128
|
+
// after a confirmed write (at-least-once; remote id conflicts ignored).
|
|
129
|
+
for (;;) {
|
|
130
|
+
const watermark = store.getExportWatermark(config.sourceId);
|
|
131
|
+
const pending = store.readInferencesAfter(watermark, batchSize);
|
|
132
|
+
if (pending.length === 0)
|
|
133
|
+
break;
|
|
134
|
+
await post("inferences", pending.map(({ row }) => toRemote(row, tenancy)), false);
|
|
135
|
+
store.setExportWatermark(config.sourceId, pending[pending.length - 1].rowid);
|
|
136
|
+
await post("export_watermarks", [{ org_id: tenancy.org_id, source_id: config.sourceId, last_exported_ts: new Date().toISOString() }], true);
|
|
137
|
+
if (pending.length < batchSize)
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
// Feedback mirror (Phase 4 WS2): same at-least-once watermark dance as
|
|
141
|
+
// inferences, behind its own export_state key so the two streams never
|
|
142
|
+
// block each other.
|
|
143
|
+
const feedbackSource = `${config.sourceId}:feedback`;
|
|
144
|
+
for (;;) {
|
|
145
|
+
const watermark = store.getExportWatermark(feedbackSource);
|
|
146
|
+
const pending = store.readFeedbackAfter(watermark, batchSize);
|
|
147
|
+
if (pending.length === 0)
|
|
148
|
+
break;
|
|
149
|
+
await post("feedback", pending.map(({ row }) => feedbackToRemote(row, tenancy)), false);
|
|
150
|
+
store.setExportWatermark(feedbackSource, pending[pending.length - 1].rowid);
|
|
151
|
+
if (pending.length < batchSize)
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
// Envelopes: small table, full upsert each flush.
|
|
155
|
+
const envelopes = store.listEnvelopes();
|
|
156
|
+
if (envelopes.length > 0) {
|
|
157
|
+
await post("envelopes", envelopes.map((e) => ({
|
|
158
|
+
org_id: tenancy.org_id,
|
|
159
|
+
envelope_key: e.envelopeKey,
|
|
160
|
+
budget_usd: e.budgetUsd,
|
|
161
|
+
spent_usd: e.spentUsd,
|
|
162
|
+
status: e.status,
|
|
163
|
+
exported_at: new Date().toISOString(),
|
|
164
|
+
})), true);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
async function guardedFlush() {
|
|
168
|
+
if (flushing)
|
|
169
|
+
return;
|
|
170
|
+
flushing = true;
|
|
171
|
+
try {
|
|
172
|
+
await flushOnce();
|
|
173
|
+
consecutiveFailures = 0;
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
// Swallow-and-log, ALWAYS — sink failures must never reach the run loop.
|
|
177
|
+
consecutiveFailures += 1;
|
|
178
|
+
skipsRemaining = Math.min(2 ** consecutiveFailures, BACKOFF_MAX_SKIPS);
|
|
179
|
+
console.warn(`[modelrig exporter] sink flush failed (attempt ${consecutiveFailures}, backing off ` +
|
|
180
|
+
`${skipsRemaining} intervals): ${err instanceof Error ? err.message : String(err)}`);
|
|
181
|
+
}
|
|
182
|
+
finally {
|
|
183
|
+
flushing = false;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return {
|
|
187
|
+
start() {
|
|
188
|
+
if (timer !== null)
|
|
189
|
+
return;
|
|
190
|
+
timer = setInterval(() => {
|
|
191
|
+
if (skipsRemaining > 0) {
|
|
192
|
+
skipsRemaining -= 1;
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
void guardedFlush();
|
|
196
|
+
}, intervalMs);
|
|
197
|
+
// Never keep the process alive for telemetry export.
|
|
198
|
+
timer.unref?.();
|
|
199
|
+
},
|
|
200
|
+
async flush() {
|
|
201
|
+
await guardedFlush();
|
|
202
|
+
},
|
|
203
|
+
async stop() {
|
|
204
|
+
if (timer !== null) {
|
|
205
|
+
clearInterval(timer);
|
|
206
|
+
timer = null;
|
|
207
|
+
}
|
|
208
|
+
await guardedFlush();
|
|
209
|
+
},
|
|
210
|
+
};
|
|
211
|
+
}
|
package/dist/types/provider.d.ts
CHANGED
|
@@ -16,6 +16,17 @@ export type CapabilityFlag = "structured_native" | "json_mode" | "grounded_nativ
|
|
|
16
16
|
/** The canonical flag enumeration — the single list every static capability
|
|
17
17
|
* materialization iterates (index.ts, template validation, tests). */
|
|
18
18
|
export declare const ALL_CAPABILITY_FLAGS: readonly CapabilityFlag[];
|
|
19
|
+
/** Screening S1/S2 (SC-0 B3) — how an adapter prefers untrusted content to be
|
|
20
|
+
* delivered when the provenance assembler spotlights it out of the system slot.
|
|
21
|
+
* `tool_result`: the untrusted block is framed as a synthetic tool_result
|
|
22
|
+
* envelope (Anthropic guidance — untrusted content belongs in tool_result,
|
|
23
|
+
* JSON-encoded, source-labelled). `user_json`: a JSON block in the user turn
|
|
24
|
+
* (the default for providers without the capability). String-valued, so it is
|
|
25
|
+
* its own registry map rather than a boolean `CapabilityFlag`; additive. */
|
|
26
|
+
export type UntrustedBlockShape = "tool_result" | "user_json";
|
|
27
|
+
/** The untrusted-block shape for a provider; `user_json` for any provider that
|
|
28
|
+
* does not declare a preference. */
|
|
29
|
+
export declare function untrustedBlockFor(provider: ProviderId): UntrustedBlockShape;
|
|
19
30
|
declare const candidateBrand: unique symbol;
|
|
20
31
|
/**
|
|
21
32
|
* A (provider, model) pair validated against a route's candidate list.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/types/provider.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,MAAM,MAAM,UAAU,GAClB,QAAQ,GACR,QAAQ,GACR,UAAU,GACV,WAAW,GACX,MAAM,GAGN,WAAW,GACX,WAAW,CAAC;AAEhB;;;;iBAIiB;AACjB,eAAO,MAAM,YAAY,0FAQf,CAAC;AAYX,MAAM,MAAM,cAAc,GACtB,mBAAmB,GACnB,WAAW,GACX,iBAAiB,GACjB,wBAAwB,GACxB,kBAAkB,GAClB,uBAAuB,GACvB,WAAW,GACX,eAAe,GACf,gBAAgB,CAAC;AAErB;sEACsE;AACtE,eAAO,MAAM,oBAAoB,EAAE,SAAS,cAAc,EAUzD,CAAC;AAEF,OAAO,CAAC,MAAM,cAAc,EAAE,OAAO,MAAM,CAAC;AAE5C;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC;CACjC;AAED,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC;AAE3D;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,cAAc;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB;8EAC0E;IAC1E,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;2DAS2D;AAC3D,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEnE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;CAChC;AASD;yDACyD;AACzD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE;QACjB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAC9B,4CAA4C;QAC5C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;CACH;AAED;+EAC+E;AAC/E,MAAM,MAAM,UAAU,GAClB,MAAM,GACN,MAAM,GACN,UAAU,GACV;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEhF;;;gBAGgB;AAChB,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE;QACjB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,+EAA+E;IAC/E,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,2EAA2E;IAC3E,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC;;;;;;;;iCAQ6B;IAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;IACnC;;;;;;;;;OASG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;IACrC;;;;;;;;;;;8EAW0E;IAC1E,QAAQ,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC;IACxC;gFAC4E;IAC5E,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,mEAAmE;IACnE,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IAC3B;;;;;;;;OAQG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,iFAAiF;IACjF,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;;OAOG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;qEAEiE;IACjE,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC;IACjC;;;;gCAI4B;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,MAAM,aAAa,GACrB;IACE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B;;;uEAGmE;IACnE,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IACzC;;;;;wCAKoC;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;CACxC;AACH;;;8EAG8E;GAC5E;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE,CAAC;AAEhI;;;;4BAI4B;AAC5B,MAAM,MAAM,kBAAkB,GAC1B;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;CAAE,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1D;;;qFAGiF;IACjF,aAAa,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACvC,IAAI,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAClD;mFAC+E;IAC/E,MAAM,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAC;CACjE"}
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/types/provider.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,MAAM,MAAM,UAAU,GAClB,QAAQ,GACR,QAAQ,GACR,UAAU,GACV,WAAW,GACX,MAAM,GAGN,WAAW,GACX,WAAW,CAAC;AAEhB;;;;iBAIiB;AACjB,eAAO,MAAM,YAAY,0FAQf,CAAC;AAYX,MAAM,MAAM,cAAc,GACtB,mBAAmB,GACnB,WAAW,GACX,iBAAiB,GACjB,wBAAwB,GACxB,kBAAkB,GAClB,uBAAuB,GACvB,WAAW,GACX,eAAe,GACf,gBAAgB,CAAC;AAErB;sEACsE;AACtE,eAAO,MAAM,oBAAoB,EAAE,SAAS,cAAc,EAUzD,CAAC;AAEF;;;;;;4EAM4E;AAC5E,MAAM,MAAM,mBAAmB,GAAG,aAAa,GAAG,WAAW,CAAC;AAM9D;oCACoC;AACpC,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,UAAU,GAAG,mBAAmB,CAE3E;AAED,OAAO,CAAC,MAAM,cAAc,EAAE,OAAO,MAAM,CAAC;AAE5C;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC;CACjC;AAED,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,MAAM,GAAG,UAAU,CAAC;AAE3D;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,cAAc;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB;8EAC0E;IAC1E,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED;;;;;;;;;2DAS2D;AAC3D,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEnE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;CAChC;AASD;yDACyD;AACzD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE;QACjB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAC9B,4CAA4C;QAC5C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;CACH;AAED;+EAC+E;AAC/E,MAAM,MAAM,UAAU,GAClB,MAAM,GACN,MAAM,GACN,UAAU,GACV;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEhF;;;gBAGgB;AAChB,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE;QACjB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,+EAA+E;IAC/E,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,2EAA2E;IAC3E,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC;;;;;;;;iCAQ6B;IAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;IACnC;;;;;;;;;OASG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;IACrC;;;;;;;;;;;8EAW0E;IAC1E,QAAQ,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC;IACxC;gFAC4E;IAC5E,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,mEAAmE;IACnE,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IAC3B;;;;;;;;OAQG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,iFAAiF;IACjF,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;;OAOG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;qEAEiE;IACjE,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC;IACjC;;;;gCAI4B;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,MAAM,aAAa,GACrB;IACE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B;;;uEAGmE;IACnE,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IACzC;;;;;wCAKoC;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;CACxC;AACH;;;8EAG8E;GAC5E;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE,CAAC;AAEhI;;;;4BAI4B;AAC5B,MAAM,MAAM,kBAAkB,GAC1B;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;CAAE,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,UAAU,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1D;;;qFAGiF;IACjF,aAAa,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACvC,IAAI,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAClD;mFAC+E;IAC/E,MAAM,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,aAAa,CAAC,kBAAkB,CAAC,CAAC;CACjE"}
|
package/dist/types/provider.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
8
|
exports.ALL_CAPABILITY_FLAGS = exports.PROVIDER_IDS = void 0;
|
|
9
|
+
exports.untrustedBlockFor = untrustedBlockFor;
|
|
9
10
|
/** Runtime enumeration of every ProviderId — the single list a raw-model lane
|
|
10
11
|
* (`provider/model`) validates against (http gateway spec §3.3). Kept in sync
|
|
11
12
|
* with the `ProviderId` union by the two compile-time assertions below: adding
|
|
@@ -38,3 +39,11 @@ exports.ALL_CAPABILITY_FLAGS = [
|
|
|
38
39
|
"trace_visible",
|
|
39
40
|
"zero_retention",
|
|
40
41
|
];
|
|
42
|
+
const UNTRUSTED_BLOCK = {
|
|
43
|
+
anthropic: "tool_result",
|
|
44
|
+
};
|
|
45
|
+
/** The untrusted-block shape for a provider; `user_json` for any provider that
|
|
46
|
+
* does not declare a preference. */
|
|
47
|
+
function untrustedBlockFor(provider) {
|
|
48
|
+
return UNTRUSTED_BLOCK[provider] ?? "user_json";
|
|
49
|
+
}
|
package/dist/types/route.d.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import type { FailureClass } from "./failures";
|
|
6
6
|
import type { ProviderId, SamplingParams, ServiceTier, ToolCall, ToolChoice, ToolDefinition } from "./provider";
|
|
7
7
|
import type { QualityGate } from "./quality";
|
|
8
|
+
import type { MaskReceipt, ResolvedMaskPolicy } from "../mask/types";
|
|
8
9
|
export interface CandidateSpec {
|
|
9
10
|
readonly provider: ProviderId;
|
|
10
11
|
readonly model: string;
|
|
@@ -105,6 +106,15 @@ export interface RouteObjective {
|
|
|
105
106
|
readonly priceCeiling?: PriceCeiling;
|
|
106
107
|
/** `balanced` only — weight on normalized cost in `regret + lambda·cost_norm`. */
|
|
107
108
|
readonly lambda?: number;
|
|
109
|
+
/**
|
|
110
|
+
* N5 pairwise-aware gate (near-optimal brief §3.2 N5) — additive & optional,
|
|
111
|
+
* DEFAULT OFF. When true, both `quality-first` and `balanced` add one extra
|
|
112
|
+
* eligibility clause: a proposed arm must NOT be pairwise-worse than the
|
|
113
|
+
* reference within CI (its pairwise win-rate lower bound ≥ 0.5). When
|
|
114
|
+
* absent/false the gate is byte-identical to RB-1 — pairwise is a REPORTED
|
|
115
|
+
* signal only. Never gates cost-first (that path stays the pre-RB-1 gate).
|
|
116
|
+
*/
|
|
117
|
+
readonly pairwiseAware?: boolean;
|
|
108
118
|
}
|
|
109
119
|
/** A deterministic value assertion on a bake-off sample's output (RB-1 spec §3).
|
|
110
120
|
* `path` is a JSON pointer (RFC 6901) into the parsed output; `expect` is either
|
|
@@ -140,6 +150,22 @@ export interface BakeoffConfig {
|
|
|
140
150
|
readonly model: CandidateSpec;
|
|
141
151
|
readonly maxSamples: number;
|
|
142
152
|
};
|
|
153
|
+
/**
|
|
154
|
+
* N5 pairwise preference vs the reference arm (near-optimal brief §3.2 N5) —
|
|
155
|
+
* additive & optional. Deterministic-first: when both an arm and the reference
|
|
156
|
+
* carry a per-sample value-accuracy score, the pairwise verdict is free. This
|
|
157
|
+
* OPT-IN judge decides only the samples that cannot be scored deterministically
|
|
158
|
+
* (free-form outputs). The judge model MUST be a declared candidate (#223 F3);
|
|
159
|
+
* every call is a metered inference tagged `bakeoff_pairwise`, capped at
|
|
160
|
+
* `maxSamples` per arm. Absent ⇒ pairwise runs on the deterministic tier only
|
|
161
|
+
* (or not at all, when no side carries value accuracy).
|
|
162
|
+
*/
|
|
163
|
+
readonly pairwise?: {
|
|
164
|
+
readonly judge?: {
|
|
165
|
+
readonly model: CandidateSpec;
|
|
166
|
+
readonly maxSamples: number;
|
|
167
|
+
};
|
|
168
|
+
};
|
|
143
169
|
}
|
|
144
170
|
export interface RouteBundle {
|
|
145
171
|
readonly route: string;
|
|
@@ -181,6 +207,24 @@ export interface RouteBundle {
|
|
|
181
207
|
readonly bakeoff?: BakeoffConfig;
|
|
182
208
|
/** Deterministic value assertions for bake-off quality (RB-1). */
|
|
183
209
|
readonly valueChecks?: readonly ValueCheck[];
|
|
210
|
+
/** Screening S1 (SC-0 B3) — grounding assembly mode. `legacy` (default,
|
|
211
|
+
* absent ⇒ legacy) string-concatenates search results into the system prompt,
|
|
212
|
+
* byte-identical to pre-B3 behavior; `spotlight` routes them through the
|
|
213
|
+
* provenance assembler as an untrusted user-slot block. The flip from legacy
|
|
214
|
+
* to spotlight on a route is bake-off-gated (spec §2.4 / review R1); `validate`
|
|
215
|
+
* warns on `legacy` once spotlight is proven. Additive & optional. */
|
|
216
|
+
readonly groundingAssembly?: "legacy" | "spotlight";
|
|
217
|
+
/** Screening S2 (SC-0 B3) — per-variable trust annotation (`x-trust`). A
|
|
218
|
+
* variable marked `untrusted` is removed from the system slot and spotlighted
|
|
219
|
+
* into the untrusted user-slot block (spec §2.4). Absent ⇒ every variable is
|
|
220
|
+
* `semi` and stays in the system slot (byte-identical default). Additive. */
|
|
221
|
+
readonly variableTrust?: Readonly<Record<string, "untrusted">>;
|
|
222
|
+
/** CS-1 Proven Masking (spec §3) — the resolved `mask:` block: mode, detector
|
|
223
|
+
* types, loaded dictionary entries, field rules, and `policy_hash`. Absent ⇒
|
|
224
|
+
* masking off and BYTE-IDENTICAL behaviour (the golden test). Resolved at
|
|
225
|
+
* route load (mask-loader.ts + policy.ts) so an unreadable dictionary or an
|
|
226
|
+
* unknown type is a load error, never a runtime surprise. Additive & optional. */
|
|
227
|
+
readonly mask?: ResolvedMaskPolicy;
|
|
184
228
|
}
|
|
185
229
|
export interface RunTags {
|
|
186
230
|
readonly run_id?: string;
|
|
@@ -308,6 +352,12 @@ export interface RunMeta {
|
|
|
308
352
|
readonly url: string;
|
|
309
353
|
readonly title?: string;
|
|
310
354
|
}>;
|
|
355
|
+
/** Grounding hygiene counters (SC-0 B3 S5) — present when the grounding-inject
|
|
356
|
+
* rung served this run. `citations_dropped` counts model-output URLs the S5
|
|
357
|
+
* citation allowlist rejected (not among the injected sources). Additive. */
|
|
358
|
+
readonly grounding?: {
|
|
359
|
+
readonly citations_dropped: number;
|
|
360
|
+
};
|
|
311
361
|
/** Normalized tool calls the model emitted (G-1/G-5 Phase A, spec §4.4) —
|
|
312
362
|
* present only on a tool-call turn, where `validated` is false and
|
|
313
363
|
* `RunResult.output` is null. The caller runs the tool(s) and continues. */
|
|
@@ -322,6 +372,11 @@ export interface RunMeta {
|
|
|
322
372
|
readonly verdict: "pass" | "fail";
|
|
323
373
|
readonly judgedBy: string | null;
|
|
324
374
|
};
|
|
375
|
+
/** CS-1 Proven Masking receipt (spec §6) — counts-only summary of what this
|
|
376
|
+
* run's serving inference masked (policy hash, mode, findings by type, tokens
|
|
377
|
+
* issued/rehydrated/unresolved). Present only when the route has an active
|
|
378
|
+
* `mask:` policy; NEVER carries a matched value. Additive & optional. */
|
|
379
|
+
readonly mask?: MaskReceipt;
|
|
325
380
|
}
|
|
326
381
|
export interface RunResult {
|
|
327
382
|
readonly output: unknown;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"route.d.ts","sourceRoot":"","sources":["../../src/types/route.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAChH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"route.d.ts","sourceRoot":"","sources":["../../src/types/route.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAChH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAErE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,UAAU,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAChG,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,CAAC;AAE5C,MAAM,WAAW,WAAW;IAC1B,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAC;IACtC;;;;;;;mCAO+B;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;CACpC;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B;;;kCAG8B;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;oEACgE;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,aAAa,CAAC;CACtC;AAED;6EAC6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;wBAEoB;IACpB,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC/C;6EACyE;IACzE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;gCAC4B;IAC5B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B;4DACwD;IACxD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;;;+CAI+C;AAC/C,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG,UAAU,GAAG,YAAY,CAAC;AAExE;;;8BAG8B;AAC9B,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC;IACrC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;OAOG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC;AAED;;;+EAG+E;AAC/E,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;KAAE,GAAG;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7E;AAED;0DAC0D;AAC1D,MAAM,WAAW,aAAa;IAC5B;;wFAEoF;IACpF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG;QAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,2EAA2E;IAC3E,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAC1C,qEAAqE;IACrE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC;;;+BAG2B;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAChF;;;;;;;;;OASG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,CAAC,EAAE;YAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;YAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CACzG;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,gEAAgE;IAChE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,CAAC;IACvC,sEAAsE;IACtE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,qEAAqE;IACrE,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,sEAAsE;IACtE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;;;;;uCAMmC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB;;wEAEoE;IACpE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;kEAC8D;IAC9D,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;4BACwB;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC;IACpC,sDAAsD;IACtD,QAAQ,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC;IACjC,kEAAkE;IAClE,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C;;;;;0EAKsE;IACtE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;IACpD;;;iFAG6E;IAC7E,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAC/D;;;;sFAIkF;IAClF,QAAQ,CAAC,IAAI,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAED,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CAC5C;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB;;;;mFAI+E;IAC/E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE;QAChB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B;;;;kFAI0E;QAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF;qEACiE;IACjE,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;oCAMgC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IACjC;;;;wEAIoE;IACpE,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE;QACnB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;QACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;KACzC,CAAC;IACF;;;uDAGmD;IACnD,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE;QACf,iEAAiE;QACjE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB;;;;;;iCAMyB;QACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC9B;2EACmE;QACnE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;IACF;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,OAAO;IACtB;8BAC0B;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,2EAA2E;IAC3E,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;wBACoB;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;+EAG2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;gDAC4C;IAC5C,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAC;QACjC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;QAC9B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC,CAAC;IACH;;iFAE6E;IAC7E,QAAQ,CAAC,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D;;gFAE4E;IAC5E,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IACzC;;;;iFAI6E;IAC7E,QAAQ,CAAC,OAAO,CAAC,EAAE;QACjB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;QAClC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;KAClC,CAAC;IACF;;;6EAGyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC;CAC7B;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CACxB"}
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import type { FailureClass } from "./failures";
|
|
6
6
|
import type { RunTags } from "./route";
|
|
7
7
|
import type { AttemptMetaV1, RunMetaV1 } from "../telemetry/attempt-meta";
|
|
8
|
+
import type { MaskReceipt } from "../mask/types";
|
|
8
9
|
/**
|
|
9
10
|
* The rung a telemetry row occupied on the candidate ladder (routing-console R1
|
|
10
11
|
* review ruling F1). SDK-stamped into `tags.rung` on EVERY row so the dashboard
|
|
@@ -99,6 +100,12 @@ export interface TelemetryRow {
|
|
|
99
100
|
* null when the producer could not build it (byte-identical run result) and
|
|
100
101
|
* for rows recorded before the WI-1a migration. */
|
|
101
102
|
readonly meta?: AttemptMetaV1 | null;
|
|
103
|
+
/** CS-1 Proven Masking receipt (spec §6) — the counts-only mask summary for
|
|
104
|
+
* this attempt, persisted to the local store `inferences.mask_json` (v13) and
|
|
105
|
+
* exported to the hosted `inferences.mask` column (additive; older servers
|
|
106
|
+
* ignore the wire field). Optional/additive: null when the route does not
|
|
107
|
+
* mask or the row predates CS-1. NEVER carries a matched value. */
|
|
108
|
+
readonly mask?: MaskReceipt | null;
|
|
102
109
|
}
|
|
103
110
|
export interface EnvelopeRow {
|
|
104
111
|
readonly envelopeKey: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,kBAAkB,CAAC;AAEvB,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;wCAEoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;kFAM8E;IAC9E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;wFACoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;uDACmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;wEACoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;oDAGgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;oFACgF;IAChF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C;;6DAEyD;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;oDACgD;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;uDACmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;;;;+EAM2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;mEAC+D;IAC/D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;sDAGkD;IAClD,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;;;uDAKmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC;;;;uEAImE;IACnE,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC;CACvC;AAED;;6EAE6E;AAC7E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;gDAE4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;cAIc;AACd,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;IAClE;;;6EAGyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;CAClC;AAED;;;gDAGgD;AAChD,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,cAAc,GAAG,UAAU,CAAC;IACjD,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED;;mFAEmF;AACnF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;kEAG8D;IAC9D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC;CAC5C;AAED,4DAA4D;AAC5D,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;mDAC+C;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,eAAe,GAAG,aAAa,GAAG,OAAO,CAAC;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,6EAA6E;AAC7E,MAAM,MAAM,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,QAAQ,CAAC;AAE/E;;+EAE+E;AAC/E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,GAAG,EAAE,MAAM,OAAO,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC;IAC7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9C,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC;IAC1D,KAAK,IAAI,IAAI,CAAC;CACf"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;
|
|
1
|
+
{"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAUH,wBAAgB,cAAc,IAAI,MAAM,CAwEvC"}
|
package/dist/validate-cli.js
CHANGED
|
@@ -17,6 +17,7 @@ const rawonly_1 = require("./run/rawonly");
|
|
|
17
17
|
const loader_1 = require("./routes/loader");
|
|
18
18
|
const manifest_1 = require("./routes/manifest");
|
|
19
19
|
const attempt_meta_1 = require("./telemetry/attempt-meta");
|
|
20
|
+
const status_runs_1 = require("./status-runs");
|
|
20
21
|
function runValidateCli() {
|
|
21
22
|
const config = (0, config_1.loadConfigFromEnv)();
|
|
22
23
|
const routesDir = (0, rawonly_1.requireRoutesDir)(config, "modelrig validate");
|
|
@@ -66,6 +67,18 @@ function runValidateCli() {
|
|
|
66
67
|
console.error(`FAIL ${r.route ?? r.file}: ${r.problems.join("; ")}`);
|
|
67
68
|
}
|
|
68
69
|
}
|
|
70
|
+
// GS-1 teaching hint: a rig that is USED (has recorded inferences) but never
|
|
71
|
+
// wrapped an execution in run.start / run.scope records no runs, so /runs stays
|
|
72
|
+
// empty and /projects shows "(no run)". A WARN, never a failure — the config is
|
|
73
|
+
// valid; runs are the standard path (get-started v5). Best-effort + silent when
|
|
74
|
+
// there is no local buffer to read yet.
|
|
75
|
+
const adoption = (0, status_runs_1.readRunAdoption)(config.telemetryDbPath);
|
|
76
|
+
if (adoption !== null && adoption.inferences > 0 && adoption.runs === 0) {
|
|
77
|
+
console.warn(`WARN this rig has recorded ${adoption.inferences} call${adoption.inferences === 1 ? "" : "s"} ` +
|
|
78
|
+
"but has never started a run — wrap each pipeline execution in run.start()/run.scope() " +
|
|
79
|
+
"(one step() per model-call family, artifact.save for work products) so it appears on /runs. " +
|
|
80
|
+
"Runs are the standard path: https://modelrig.dev/instrumentation.html");
|
|
81
|
+
}
|
|
69
82
|
if (failures === 0) {
|
|
70
83
|
console.log(`\n${results.length} route(s) valid — no provider key required for this check.`);
|
|
71
84
|
return 0;
|
package/dist/verify/types.d.ts
CHANGED
|
@@ -75,6 +75,12 @@ export interface SampleReport extends SampleDiff {
|
|
|
75
75
|
export interface EquivalenceReport {
|
|
76
76
|
readonly schemaVersion: 1;
|
|
77
77
|
readonly kind: "shadow-equivalence";
|
|
78
|
+
/** What change this equivalence run proves, when it is not a plain model swap
|
|
79
|
+
* (the default, absent). `screen-assembly` marks a SC-0 B3 run proving an
|
|
80
|
+
* assembly change — grounding `legacy`→`spotlight`, or an `x-trust: untrusted`
|
|
81
|
+
* variable flip — is output-equivalent to the prior assembly on the pinned
|
|
82
|
+
* model (spec §5), so the flip ships through the bake-off like a swap. Additive. */
|
|
83
|
+
readonly subject?: "swap" | "mask-policy" | "screen-assembly";
|
|
78
84
|
readonly callSiteId: string;
|
|
79
85
|
readonly route: string;
|
|
80
86
|
readonly routeVersion: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/verify/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;;6BAE6B;AAC7B,MAAM,WAAW,aAAa;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;oEACoE;AACpE,MAAM,WAAW,oBAAoB;IACnC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;mEAC+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAC3C,4EAA4E;IAC5E,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C;;;sDAGkD;IAClD,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpD;AAED,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GACrB,YAAY,GACZ,kCAAkC,GAClC,WAAW,CAAC;AAEhB;kDACkD;AAClD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3D,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC;;wEAEoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;4EACwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;+CAC2C;IAC3C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;QAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;sEACkE;IAClE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/verify/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;;6BAE6B;AAC7B,MAAM,WAAW,aAAa;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;oEACoE;AACpE,MAAM,WAAW,oBAAoB;IACnC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;mEAC+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAC3C,4EAA4E;IAC5E,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C;;;sDAGkD;IAClD,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpD;AAED,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GACrB,YAAY,GACZ,kCAAkC,GAClC,WAAW,CAAC;AAEhB;kDACkD;AAClD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3D,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC;;wEAEoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;4EACwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;IACpC;;;;wFAIoF;IACpF,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,iBAAiB,CAAC;IAC9D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;+CAC2C;IAC3C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;QAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;sEACkE;IAClE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: modelrig
|
|
3
|
+
description: Use when working with LLM/model selection, cost, or reliability in a codebase — choosing a model, migrating direct provider SDK calls to routes, explaining what a model call actually costs, or debugging schema/grounding/caching failures. Answers from PROBED measurements rather than provider claims.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# ModelRig
|
|
7
|
+
|
|
8
|
+
**A pointer skill.** The canonical playbook lives at
|
|
9
|
+
<https://modelrig.dev/llms.txt> and is regenerated whenever the registry
|
|
10
|
+
is rebuilt. Fetch it rather than trusting anything cached here — model
|
|
11
|
+
capabilities change, and a stale answer about a model is worse than no answer.
|
|
12
|
+
|
|
13
|
+
One copy, every ecosystem: this file is the same in Claude Code, Cursor,
|
|
14
|
+
Codex, and anywhere else skills load. It deliberately holds almost no facts.
|
|
15
|
+
|
|
16
|
+
## Start here
|
|
17
|
+
|
|
18
|
+
1. **Fetch <https://modelrig.dev/llms.txt>.** It carries the current probed
|
|
19
|
+
facts, the model list with `as_of` stamps, and the tool advertisements.
|
|
20
|
+
2. **Prefer the MCP tools over parsing text.** If the ModelRig oracle is
|
|
21
|
+
registered, call it — the answers are typed, dated, and scoped:
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
query_registry which models satisfy a capability + price bound
|
|
25
|
+
get_leaderboard standings by effective cost of conformance
|
|
26
|
+
explain_pricing list price vs what conformance actually costs
|
|
27
|
+
get_call_notes per-model quirks that make requests fail
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Not registered? Add it — it needs no account for those four:
|
|
31
|
+
|
|
32
|
+
```json
|
|
33
|
+
{ "mcpServers": { "modelrig": { "command": "npx", "args": ["modelrig-oracle"] } } }
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
3. **Fall back to the raw registry** when neither is available:
|
|
37
|
+
<https://raw.githubusercontent.com/modelrig/modelrig/main/registry/registry.json>
|
|
38
|
+
|
|
39
|
+
## Rules that matter more than speed
|
|
40
|
+
|
|
41
|
+
- **Never state a model's capabilities from memory.** Look them up. Every
|
|
42
|
+
fact in the registry carries an `as_of` date and a sample count; quote both.
|
|
43
|
+
- **Absence is not a limit.** A model missing from an answer is one nobody has
|
|
44
|
+
probed — say that, rather than implying it can't do the thing.
|
|
45
|
+
- **Probed beats declared.** Where a provider's claim and our measurement
|
|
46
|
+
disagree, the entry's `discrepancies` field says so. Surface the
|
|
47
|
+
disagreement; it is usually the useful part.
|
|
48
|
+
- **Compare on effective cost of conformance,** not list price. A model at half
|
|
49
|
+
the price that conforms two-thirds as often is not cheaper.
|
|
50
|
+
|
|
51
|
+
## Just want to see your runs first? (observe, no migration)
|
|
52
|
+
|
|
53
|
+
Before migrating anything, you can just **watch** an existing pipeline. ModelRig
|
|
54
|
+
records each run as a run → step → artifact chain you click through in the
|
|
55
|
+
console — no routes moved, no model calls rerouted, and with the flag off the
|
|
56
|
+
pipeline behaves byte-for-byte as before. Scaffold it:
|
|
57
|
+
`npx modelrig observe [path] --scope <one pipeline>` writes a self-contained
|
|
58
|
+
`record-step` seam and an `OBSERVE.md` whose wiring is emitted as diffs you
|
|
59
|
+
apply. The instrumentation is a code-level SDK seam (no zero-code observe), but
|
|
60
|
+
you no longer hand-write it — see <https://modelrig.dev/instrumentation.html>.
|
|
61
|
+
The namespace is **on by default once a control plane is configured** (a
|
|
62
|
+
`rig_sk_` API key, or the self-host Supabase pair); a pure local-only rig stays
|
|
63
|
+
off — force it on there with `MODELRIG_ARTIFACTS=1`, and `MODELRIG_ARTIFACTS=0`
|
|
64
|
+
opts out anywhere. Observing the runs in the console needs a free
|
|
65
|
+
account + key from <https://app.modelrig.ai> — the one step a human has to do
|
|
66
|
+
(console login to mint the key), so plan the handoff rather than stalling on it.
|
|
67
|
+
|
|
68
|
+
## Migrating a repo — the T0–T2 playbook
|
|
69
|
+
|
|
70
|
+
When you import this skill to migrate a codebase, **fetch the canonical
|
|
71
|
+
playbook** and follow it: <https://modelrig.dev/migration-playbook.html>. It is
|
|
72
|
+
a four-tier autonomy ladder — T0 gateway-pointing and T1 instrumentation are
|
|
73
|
+
behaviour-preserving (apply after a plan review); T2 route extraction replaces a
|
|
74
|
+
direct provider call with `rig.run()` **always same-model**, and is
|
|
75
|
+
behaviour-affecting, so the agent *proposes* it, proves it with
|
|
76
|
+
`modelrig verify-swap` (shadow-equivalence), embeds the report in the PR, does
|
|
77
|
+
**one call site per PR**, and a **human merges**. Model optimization (T3) is
|
|
78
|
+
never the skill's job.
|
|
79
|
+
|
|
80
|
+
**Runs are the standard path, not an appendix.** Every pipeline execution should
|
|
81
|
+
be a **run**: wrap it in `rig.artifacts.run.start` / `run.scope`, name one step
|
|
82
|
+
per model-call family (pass `stepKey`), `artifact.save` the work products, and tag
|
|
83
|
+
`subject` / `feature`. Inside a run context the SDK auto-stamps `tags.run_id` for
|
|
84
|
+
you, so the run fills `/runs` **and** `/projects`; acceptance is the run appearing
|
|
85
|
+
on `/runs` with its steps. (Inside a run context the SDK also auto-stamps
|
|
86
|
+
`tags.step`; a value you pass always wins, so name the step with `stepKey` or
|
|
87
|
+
`tags.step` when you want a specific label.) This is Tier 1 and it is **not
|
|
88
|
+
optional for a raw-lane (Lane-B) seam** — a pipeline that reaches the model
|
|
89
|
+
through `rig.runRaw` without a `run.start` around it records inferences but no
|
|
90
|
+
run. The full API is the reference at
|
|
91
|
+
<https://modelrig.dev/instrumentation.html>.
|
|
92
|
+
|
|
93
|
+
The machinery underneath: `npx modelrig init [path] [--scope <glob>]` scans for
|
|
94
|
+
provider call sites and drafts a route bundle for each, plus a `MIGRATION.md`
|
|
95
|
+
mapping call site → route → the replacement `rig.run` line. It writes only under
|
|
96
|
+
`modelrig/routes/drafts/` and **never edits source** — applying the diff is the
|
|
97
|
+
human's decision, one call site at a time. The replaced call is **deleted** (not
|
|
98
|
+
commented out) with a one-line breadcrumb; rollback is `git revert`.
|
|
99
|
+
|
|
100
|
+
## Capabilities beyond migration (the full surface)
|
|
101
|
+
|
|
102
|
+
Shipped features a newbie migration won't stumble onto — one line each, with the
|
|
103
|
+
reference doc. The canonical detail is always <https://modelrig.dev/llms.txt>.
|
|
104
|
+
|
|
105
|
+
- **Runs & projects** — the run → step → artifact graph at `/runs`, grouped by tag
|
|
106
|
+
at `/projects`: <https://modelrig.dev/instrumentation.html>.
|
|
107
|
+
- **Tenants & statements** — a `subject` tag or project-scoped key becomes a
|
|
108
|
+
per-customer tenant with a derived statement: <https://modelrig.dev/tenants.html>.
|
|
109
|
+
- **Published receipts** — a signed, login-free receipt of a bake-off or route at
|
|
110
|
+
`/r/<id>`, verifiable at `/v1/verify/<id>`, content never leaves:
|
|
111
|
+
<https://modelrig.dev/receipts.html>.
|
|
112
|
+
- **Analytics query API** — self-describing, read-only, metadata-only SELECT over
|
|
113
|
+
your own telemetry (`GET /v1/analytics/meta`, `POST /v1/analytics/query`):
|
|
114
|
+
<https://modelrig.dev/optimization-loop.html>.
|
|
115
|
+
- **Provenance & trust** — provenance-first prompt assembly plus per-variable
|
|
116
|
+
`x-trust`: <https://modelrig.dev/screening.html>.
|
|
117
|
+
- **Bake-offs — pairwise, disagreement & regret** — replay your own traffic, judge
|
|
118
|
+
pairwise with disagreement shown, propose the lowest-regret arm under a quality
|
|
119
|
+
gate: <https://modelrig.dev/bakeoffs.html> and <https://modelrig.dev/run-verdicts.html>.
|
|
120
|
+
|
|
121
|
+
Content custody stays early access (metadata and hashes); no capability here is a
|
|
122
|
+
claim beyond what ships.
|
|
123
|
+
|
|
124
|
+
## What ModelRig is, in three sentences
|
|
125
|
+
|
|
126
|
+
A route is a task your code calls by name; the route's YAML declares which
|
|
127
|
+
models are allowed to serve it, the schema the output must match, and the
|
|
128
|
+
retry and budget policy. Your code calls the task, not the model, so swapping
|
|
129
|
+
models is a config change with evidence behind it rather than a code change
|
|
130
|
+
with hope behind it. Nothing switches what serves a route without a bake-off
|
|
131
|
+
on your own traffic and an explicit human approval.
|