modelrig 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +156 -0
- package/README.md +27 -2
- package/dist/artifacts/artifact.d.ts +12 -13
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +17 -23
- package/dist/artifacts/index.d.ts +13 -5
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +11 -4
- package/dist/artifacts/run-context.d.ts +3 -24
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +3 -39
- package/dist/bakeoff/engine.d.ts +29 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +47 -0
- package/dist/bakeoff/eval-cases-rig.d.ts +36 -0
- package/dist/bakeoff/eval-cases-rig.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases-rig.js +27 -0
- package/dist/bakeoff/eval-cases.d.ts +19 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +18 -1
- package/dist/bakeoff/golden-judge.d.ts +47 -0
- package/dist/bakeoff/golden-judge.d.ts.map +1 -0
- package/dist/bakeoff/golden-judge.js +49 -0
- package/dist/bakeoff/quality.d.ts +30 -0
- package/dist/bakeoff/quality.d.ts.map +1 -1
- package/dist/bakeoff/quality.js +21 -0
- package/dist/bakeoff/replay.d.ts +11 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +8 -0
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +26 -0
- package/dist/cli-judge.d.ts +12 -0
- package/dist/cli-judge.d.ts.map +1 -0
- package/dist/cli-judge.js +48 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +14 -42
- package/dist/config.d.ts +54 -29
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +86 -47
- package/dist/detect/detector.d.ts +4 -2
- package/dist/detect/detector.d.ts.map +1 -1
- package/dist/detect/safe-harbor.d.ts +57 -0
- package/dist/detect/safe-harbor.d.ts.map +1 -0
- package/dist/detect/safe-harbor.js +278 -0
- package/dist/feedback/coverage.d.ts +11 -8
- package/dist/feedback/coverage.d.ts.map +1 -1
- package/dist/feedback/coverage.js +29 -16
- package/dist/feedback/grade.d.ts +51 -0
- package/dist/feedback/grade.d.ts.map +1 -0
- package/dist/feedback/grade.js +63 -0
- package/dist/gateway/http-protocol.d.ts +1 -1
- package/dist/gateway/http-protocol.d.ts.map +1 -1
- package/dist/gateway/http-protocol.js +1 -1
- package/dist/index.d.ts +44 -27
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +110 -43
- package/dist/init/observe-cli.js +1 -1
- package/dist/init/observe.d.ts +2 -2
- package/dist/init/observe.d.ts.map +1 -1
- package/dist/init/observe.js +11 -7
- package/dist/mask/arm.d.ts +112 -0
- package/dist/mask/arm.d.ts.map +1 -0
- package/dist/mask/arm.js +179 -0
- package/dist/mask/index.d.ts +2 -0
- package/dist/mask/index.d.ts.map +1 -1
- package/dist/mask/index.js +7 -1
- package/dist/mask/mask-policy-cli.d.ts +12 -4
- package/dist/mask/mask-policy-cli.d.ts.map +1 -1
- package/dist/mask/mask-policy-cli.js +34 -9
- package/dist/query/grammar.d.ts +2 -2
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +3 -1
- package/dist/query/saved.d.ts +13 -0
- package/dist/query/saved.d.ts.map +1 -1
- package/dist/query/saved.js +67 -1
- package/dist/receipts/types.d.ts +1 -1
- package/dist/receipts/types.d.ts.map +1 -1
- package/dist/receipts/types.js +1 -0
- package/dist/registry/build.d.ts +43 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +52 -0
- package/dist/registry/pricing-shape.d.ts +29 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +10 -1
- package/dist/registry/rows.d.ts.map +1 -1
- package/dist/registry/rows.js +1 -0
- package/dist/registry/select.d.ts +6 -1
- package/dist/registry/select.d.ts.map +1 -1
- package/dist/registry/select.js +11 -0
- package/dist/routes/call-config.d.ts +199 -0
- package/dist/routes/call-config.d.ts.map +1 -0
- package/dist/routes/call-config.js +343 -0
- package/dist/routes/loader.js +5 -3
- package/dist/routes/objective-loader.d.ts +12 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +82 -0
- package/dist/routes/resolved-bundle.d.ts +50 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +52 -0
- package/dist/routes/screen-loader.d.ts +28 -0
- package/dist/routes/screen-loader.d.ts.map +1 -1
- package/dist/routes/screen-loader.js +51 -0
- package/dist/run/accept.d.ts +2 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +1 -0
- package/dist/run/dispatch.d.ts +42 -13
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +107 -25
- package/dist/run/grounding.d.ts +28 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +54 -10
- package/dist/run/record-attempt.d.ts +4 -0
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +4 -0
- package/dist/run/routing-docs.js +1 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +21 -26
- package/dist/savings/odometer.d.ts +4 -0
- package/dist/savings/odometer.d.ts.map +1 -1
- package/dist/savings/odometer.js +20 -1
- package/dist/savings/swap-execute.d.ts +7 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +30 -0
- package/dist/savings/swaps-client.d.ts +10 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/screen/fixtures/probe-corpora.d.ts +23 -0
- package/dist/screen/fixtures/probe-corpora.d.ts.map +1 -0
- package/dist/screen/fixtures/probe-corpora.js +69 -0
- package/dist/screen/index.d.ts +21 -0
- package/dist/screen/index.d.ts.map +1 -0
- package/dist/screen/index.js +64 -0
- package/dist/screen/model-screen.d.ts +111 -0
- package/dist/screen/model-screen.d.ts.map +1 -0
- package/dist/screen/model-screen.js +178 -0
- package/dist/screen/patterns.d.ts +51 -0
- package/dist/screen/patterns.d.ts.map +1 -0
- package/dist/screen/patterns.js +151 -0
- package/dist/screen/phi-gate.d.ts +52 -0
- package/dist/screen/phi-gate.d.ts.map +1 -0
- package/dist/screen/phi-gate.js +86 -0
- package/dist/screen/policy.d.ts +71 -0
- package/dist/screen/policy.d.ts.map +1 -0
- package/dist/screen/policy.js +334 -0
- package/dist/screen/probe.d.ts +118 -0
- package/dist/screen/probe.d.ts.map +1 -0
- package/dist/screen/probe.js +132 -0
- package/dist/screen/screen-cli.d.ts +16 -0
- package/dist/screen/screen-cli.d.ts.map +1 -0
- package/dist/screen/screen-cli.js +178 -0
- package/dist/screen/screen.d.ts +99 -0
- package/dist/screen/screen.d.ts.map +1 -0
- package/dist/screen/screen.js +0 -0
- package/dist/screen/sidecar.d.ts +69 -0
- package/dist/screen/sidecar.d.ts.map +1 -0
- package/dist/screen/sidecar.js +145 -0
- package/dist/screen/types.d.ts +43 -2
- package/dist/screen/types.d.ts.map +1 -1
- package/dist/settings/api-fetcher.d.ts +14 -3
- package/dist/settings/api-fetcher.d.ts.map +1 -1
- package/dist/settings/api-fetcher.js +61 -10
- package/dist/settings/grading-guide.d.ts +76 -0
- package/dist/settings/grading-guide.d.ts.map +1 -0
- package/dist/settings/grading-guide.js +23 -0
- package/dist/settings/index.d.ts +3 -2
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +44 -7
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +53 -12
- package/dist/status.d.ts +26 -20
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +47 -29
- package/dist/telemetry/artifact-store.d.ts +5 -6
- package/dist/telemetry/artifact-store.d.ts.map +1 -1
- package/dist/telemetry/artifact-store.js +10 -24
- package/dist/telemetry/attempt-meta.d.ts +21 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +9 -0
- package/dist/telemetry/exporter.d.ts +2 -2
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +22 -41
- package/dist/telemetry/grade-mappers.d.ts +22 -0
- package/dist/telemetry/grade-mappers.d.ts.map +1 -0
- package/dist/telemetry/grade-mappers.js +58 -0
- package/dist/telemetry/grades-sql.d.ts +53 -0
- package/dist/telemetry/grades-sql.d.ts.map +1 -0
- package/dist/telemetry/grades-sql.js +133 -0
- package/dist/telemetry/ingest-protocol.d.ts +29 -46
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +16 -54
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +51 -0
- package/dist/telemetry/store.d.ts +9 -6
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +8 -50
- package/dist/tenancy/api-key-format.d.ts +30 -1
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +33 -1
- package/dist/types/provider.d.ts +3 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +25 -0
- package/dist/types/route.d.ts +78 -2
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +43 -46
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +32 -0
- package/dist/verify/verify.d.ts +5 -0
- package/dist/verify/verify.d.ts.map +1 -1
- package/dist/verify/verify.js +2 -0
- package/dist/watch/evaluate.d.ts +84 -1
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +28 -0
- package/dist/watch/monitor.d.ts +8 -2
- package/dist/watch/monitor.d.ts.map +1 -1
- package/dist/watch/monitor.js +13 -5
- package/dist/watch/regret-monitor-cli.d.ts +16 -0
- package/dist/watch/regret-monitor-cli.d.ts.map +1 -0
- package/dist/watch/regret-monitor-cli.js +137 -0
- package/dist/watch/regret-monitor.d.ts +168 -0
- package/dist/watch/regret-monitor.d.ts.map +1 -0
- package/dist/watch/regret-monitor.js +304 -0
- package/dist/watch/upgrade.d.ts +62 -0
- package/dist/watch/upgrade.d.ts.map +1 -0
- package/dist/watch/upgrade.js +57 -0
- package/docs/agent-skill/SKILL.md +109 -5
- package/docs/how-it-fits.md +3 -2
- package/docs/instrumentation.md +149 -21
- package/docs/llms.txt +64 -20
- package/docs/mcp-oracle.md +249 -0
- package/docs/quickstart.md +103 -9
- package/package.json +16 -3
- package/registry/leaderboard.json +74 -0
- package/registry/registry.json +150 -112
- package/screen.d.ts +16 -0
- package/templates/call-disposition-qa.json +77 -0
- package/templates/catalog-cleansing.json +77 -0
- package/templates/compliance-review.json +76 -0
- package/templates/cs-next-action.json +76 -0
- package/templates/document-extraction.json +78 -0
- package/templates/financial-classification.json +75 -0
- package/templates/lead-qualification.json +76 -0
- package/templates/medical-classification.json +71 -0
- package/templates/ticket-triage.json +76 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
package/dist/watch/evaluate.d.ts
CHANGED
|
@@ -56,7 +56,28 @@ export interface WatchConfig {
|
|
|
56
56
|
* Analyst's triggers ride the widened kinds (`insight`/`promote-eval`) or reuse
|
|
57
57
|
* `bakeoff` for an Analyst-initiated bake-off — all through the SAME proposals
|
|
58
58
|
* machinery and the SAME `submitDrafts` seam (trajectory F1/F4, one emission path). */
|
|
59
|
-
export type ProposalTrigger = EconomicsTrigger | ConformDegradationTrigger | AnalystClusterTrigger | AnalystPromoteEvalTrigger | AnalystBakeoffTrigger | ManualChallengerTrigger;
|
|
59
|
+
export type ProposalTrigger = EconomicsTrigger | ConformDegradationTrigger | AnalystClusterTrigger | AnalystPromoteEvalTrigger | AnalystBakeoffTrigger | ManualChallengerTrigger | UpgradeOpportunityTrigger | RegretBreachTrigger | OptimizerImprovementTrigger;
|
|
60
|
+
/**
|
|
61
|
+
* IP-2a-2: an EXPLICIT optimizer run raised this (the /graders/[route] console
|
|
62
|
+
* action or the `run_optimizer` MCP tool — spec-ip2 §2; no cron, no sweep).
|
|
63
|
+
* `kind` stays `"swap"`: the winning arm is a variant swap in the wide sense
|
|
64
|
+
* (a model pin, a prompt override, or both), delivered through the SAME
|
|
65
|
+
* never-silent machinery — the change_spec on the evidence is the
|
|
66
|
+
* machine-readable half. `bakeoffId` (the eval run that PROVED the winner) is
|
|
67
|
+
* deliberately INSIDE the hash: each explicit run is its own decision record,
|
|
68
|
+
* never deduped against an earlier run's.
|
|
69
|
+
*/
|
|
70
|
+
export interface OptimizerImprovementTrigger {
|
|
71
|
+
readonly type: "optimizer_improvement";
|
|
72
|
+
readonly route: string;
|
|
73
|
+
/** Which inputs the engine ran on: prompt bytes (G1 open) or config-only. */
|
|
74
|
+
readonly mode: "prompt" | "config";
|
|
75
|
+
/** The bake-off (eval run) this run scored candidates through. */
|
|
76
|
+
readonly bakeoffId: string;
|
|
77
|
+
/** Honest input counts at trigger time. */
|
|
78
|
+
readonly evalCasesN: number;
|
|
79
|
+
readonly gradesN: number;
|
|
80
|
+
}
|
|
60
81
|
/**
|
|
61
82
|
* A human-initiated declare-first challenger (routing-console spec §2.4 / §3, R3)
|
|
62
83
|
* — the console picker (or the oracle) proposes trialing a specific model on a
|
|
@@ -149,6 +170,68 @@ export interface EconomicsTrigger {
|
|
|
149
170
|
readonly newRates?: TokenRates;
|
|
150
171
|
readonly pricing?: TokenRates | null;
|
|
151
172
|
}
|
|
173
|
+
/** An aggregate regret band over one window — the quality twin of the cost
|
|
174
|
+
* odometer's savings band (spec §2). Carries ONLY aggregate regret numbers +
|
|
175
|
+
* the paired n (content-custody: no capture content, ever). `regretCi95` is a
|
|
176
|
+
* seeded paired bootstrap; `nPaired` is the window's paired-sample count. */
|
|
177
|
+
export interface RegretBandSummary {
|
|
178
|
+
/** Window bounds — inclusive-from / exclusive-to ISO days. */
|
|
179
|
+
readonly from: string;
|
|
180
|
+
readonly to: string;
|
|
181
|
+
readonly regretMean: number;
|
|
182
|
+
readonly regretCi95: readonly [number, number];
|
|
183
|
+
readonly nPaired: number;
|
|
184
|
+
}
|
|
185
|
+
/** N4 upgrade trigger (spec §4.1): the route's reference/ceiling ROSE — a new
|
|
186
|
+
* registry model, a new snapshot of the reference, or a Q1 regret breach — so a
|
|
187
|
+
* bake-off is proposed to measure the opportunity. The mirror image of
|
|
188
|
+
* `price_change`/`new_model`. Fires ONLY for a route that resolves a reference
|
|
189
|
+
* (invariant `upswap-opt-in-only`). */
|
|
190
|
+
export interface UpgradeOpportunityTrigger {
|
|
191
|
+
readonly type: "upgrade_opportunity";
|
|
192
|
+
readonly route: string;
|
|
193
|
+
/** The model that raised the ceiling (new registry entry, or a new snapshot). */
|
|
194
|
+
readonly reference: CandidateEconomics;
|
|
195
|
+
/** Why it fired — console/audit context; the idempotency hash keys on the ref
|
|
196
|
+
* identity (+ snapshot) + cause, NOT the whole trigger. */
|
|
197
|
+
readonly cause: "new_reference" | "reference_snapshot" | "regret_breach";
|
|
198
|
+
/** Present when the fire came from Q1 (cause=regret_breach): the band that
|
|
199
|
+
* tripped ε — so the console shows WHY without a second read. */
|
|
200
|
+
readonly band?: RegretBandSummary;
|
|
201
|
+
}
|
|
202
|
+
/** Q1 regret-breach trigger (spec §5.2): the recent window's regret upper-CI
|
|
203
|
+
* exceeded ε, OR drifted up from baseline by ≥ the ruled margin, with min-n on
|
|
204
|
+
* BOTH sides. Emits `kind: "bakeoff"` — the sampled monitor number is the ALARM,
|
|
205
|
+
* a fresh bake-off is the EVIDENCE (and, on an opted-in route, where an up-swap
|
|
206
|
+
* winner is found → Q1 feeds N4). */
|
|
207
|
+
export interface RegretBreachTrigger {
|
|
208
|
+
readonly type: "regret_breach";
|
|
209
|
+
readonly route: string;
|
|
210
|
+
/** What we measured against. */
|
|
211
|
+
readonly reference: CandidateEconomics;
|
|
212
|
+
readonly recent: RegretBandSummary;
|
|
213
|
+
readonly baseline: RegretBandSummary;
|
|
214
|
+
/** The route's `max_regret` (null ⇒ drift-only breach). */
|
|
215
|
+
readonly epsilon: number | null;
|
|
216
|
+
readonly breachKind: "over_epsilon" | "drift_up";
|
|
217
|
+
readonly minN: number;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Idempotency key for an `upgrade_opportunity` — per (route, reference, cause,
|
|
221
|
+
* snapshot) ONLY (spec §4.1), so the `band`/audit context never splits a dedupe.
|
|
222
|
+
* A re-run over the SAME ceiling is a no-op via the `(org_id, route,
|
|
223
|
+
* trigger_hash)` unique index; a NEWER snapshot (distinct `snapshotId`) is a
|
|
224
|
+
* genuinely new opportunity and a distinct hash.
|
|
225
|
+
*/
|
|
226
|
+
export declare function upgradeTriggerHash(route: string, reference: CandidateEconomics, cause: UpgradeOpportunityTrigger["cause"], snapshotId?: string): string;
|
|
227
|
+
/**
|
|
228
|
+
* Idempotency key for a `regret_breach` — per (route, reference, window bounds,
|
|
229
|
+
* breachKind) ONLY (spec §5.2), so the sampled band's noisy per-run regret
|
|
230
|
+
* numbers never split a dedupe. The window bounds are day-quantized off the
|
|
231
|
+
* injected `now`, so a re-run the SAME day is a no-op via the `(org_id, route,
|
|
232
|
+
* trigger_hash)` unique index.
|
|
233
|
+
*/
|
|
234
|
+
export declare function regretBreachTriggerHash(route: string, reference: CandidateEconomics, windowFrom: string, windowTo: string, breachKind: RegretBreachTrigger["breachKind"]): string;
|
|
152
235
|
/** One side of the degradation comparison (E4 chunk B) — a window's conform
|
|
153
236
|
* rate over its evaluations. Dates are inclusive-from / exclusive-to ISO days. */
|
|
154
237
|
export interface ConformWindow {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/watch/evaluate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAYH,OAAO,KAAK,EACV,qBAAqB,EACrB,YAAY,EACZ,cAAc,EACd,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AAEpC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,SAAS,kBAAkB,EAAE,CAAC;CACpD;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,MAAM,UAAU,GAClB;IACE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IACzB,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;CAC1B,GACD;IACE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAAC;CACrC,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,8DAA8D;IAC9D,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;qFAIiF;IACjF,QAAQ,CAAC,eAAe,CAAC,EAAE,qBAAqB,CAAC;CAClD;AAED;;;;;;uFAMuF;AACvF,MAAM,MAAM,eAAe,GACvB,gBAAgB,GAChB,yBAAyB,GACzB,qBAAqB,GACrB,yBAAyB,GACzB,qBAAqB,GACrB,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/watch/evaluate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAYH,OAAO,KAAK,EACV,qBAAqB,EACrB,YAAY,EACZ,cAAc,EACd,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AAEpC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,SAAS,kBAAkB,EAAE,CAAC;CACpD;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,MAAM,UAAU,GAClB;IACE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IACzB,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;CAC1B,GACD;IACE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAAC;CACrC,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,8DAA8D;IAC9D,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;qFAIiF;IACjF,QAAQ,CAAC,eAAe,CAAC,EAAE,qBAAqB,CAAC;CAClD;AAED;;;;;;uFAMuF;AACvF,MAAM,MAAM,eAAe,GACvB,gBAAgB,GAChB,yBAAyB,GACzB,qBAAqB,GACrB,yBAAyB,GACzB,qBAAqB,GACrB,uBAAuB,GACvB,yBAAyB,GACzB,mBAAmB,GACnB,2BAA2B,CAAC;AAEhC;;;;;;;;;GASG;AACH,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,IAAI,EAAE,uBAAuB,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IACnC,kEAAkE;IAClE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,2CAA2C;IAC3C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAC;IACxC,wEAAwE;IACxE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,sEAAsE;IACtE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAQD,8EAA8E;AAC9E,MAAM,WAAW,mBAAmB;IAClC,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,qEAAqE;IACrE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,6EAA6E;IAC7E,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9C;AAED;;4EAE4E;AAC5E,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,mBAAmB,CAAC;IACpC,mEAAmE;IACnE,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5C;AAED;;sFAEsF;AACtF,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,SAAS,uBAAuB,EAAE,CAAC;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;kBAGkB;AAClB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,yEAAyE;IACzE,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAC;IACxC,sEAAsE;IACtE,QAAQ,CAAC,UAAU,EAAE,SAAS,kBAAkB,EAAE,CAAC;CACpD;AAED,8EAA8E;AAC9E,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,cAAc,GAAG,WAAW,CAAC;IAC5C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,uDAAuD;IACvD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC;IAC/B,QAAQ,CAAC,OAAO,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;CACtC;AAOD;;;6EAG6E;AAC7E,MAAM,WAAW,iBAAiB;IAChC,8DAA8D;IAC9D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;uCAIuC;AACvC,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,IAAI,EAAE,qBAAqB,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,iFAAiF;IACjF,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;IACvC;+DAC2D;IAC3D,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,oBAAoB,GAAG,eAAe,CAAC;IACzE;qEACiE;IACjE,QAAQ,CAAC,IAAI,CAAC,EAAE,iBAAiB,CAAC;CACnC;AAED;;;;qCAIqC;AACrC,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,gCAAgC;IAChC,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;IACvC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAC;IACrC,2DAA2D;IAC3D,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,cAAc,GAAG,UAAU,CAAC;IACjD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,kBAAkB,EAC7B,KAAK,EAAE,yBAAyB,CAAC,OAAO,CAAC,EACzC,UAAU,CAAC,EAAE,MAAM,GAClB,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,kBAAkB,EAC7B,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,mBAAmB,CAAC,YAAY,CAAC,GAC5C,MAAM,CAOR;AAED;kFACkF;AAClF,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;iFAEiF;AACjF,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,IAAI,EAAE,qBAAqB,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,sEAAsE;IACtE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;0CAE0C;AAC1C,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,cAAc,CAAC;AAE3E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;uDAGmD;IACnD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAYpC,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAE3E;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,kBAAkB,GAC7B,MAAM,CAKR;AAMD,wBAAgB,cAAc,CAC5B,MAAM,EAAE,SAAS,cAAc,EAAE,EACjC,MAAM,EAAE,SAAS,UAAU,EAAE,EAC7B,MAAM,EAAE,WAAW,GAClB,aAAa,EAAE,CAoDjB;AAUD;;oDAEoD;AACpD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,+BAA+B,EAAE,MAAM,CAAC;IACjD,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,0BAA0B,EAAE,MAAM,CAAC;IAC5C,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;CAChC;AAED;;;;kCAIkC;AAClC,MAAM,WAAW,iBAAiB;IAChC,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,6CAA6C;IAC7C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,8CAA8C;IAC9C,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC,iFAAiF;IACjF,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,kEAAkE;IAClE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,0EAA0E;IAC1E,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;2EACuE;IACvE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAID;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,iBAAiB,EACxB,MAAM,GAAE,qBAAgD,GACvD,YAAY,CA8Cd;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,YAAY,EACjB,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,GACtB,MAAM,CAaR"}
|
package/dist/watch/evaluate.js
CHANGED
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
*/
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.PROPOSAL_TTL_DAYS = void 0;
|
|
20
|
+
exports.upgradeTriggerHash = upgradeTriggerHash;
|
|
21
|
+
exports.regretBreachTriggerHash = regretBreachTriggerHash;
|
|
20
22
|
exports.triggerHash = triggerHash;
|
|
21
23
|
exports.manualChallengerTriggerHash = manualChallengerTriggerHash;
|
|
22
24
|
exports.evaluateEvents = evaluateEvents;
|
|
@@ -24,6 +26,32 @@ exports.buildCacheContext = buildCacheContext;
|
|
|
24
26
|
exports.explainCacheContext = explainCacheContext;
|
|
25
27
|
const node_crypto_1 = require("node:crypto");
|
|
26
28
|
const effective_rate_1 = require("../registry/effective-rate");
|
|
29
|
+
/**
|
|
30
|
+
* Idempotency key for an `upgrade_opportunity` — per (route, reference, cause,
|
|
31
|
+
* snapshot) ONLY (spec §4.1), so the `band`/audit context never splits a dedupe.
|
|
32
|
+
* A re-run over the SAME ceiling is a no-op via the `(org_id, route,
|
|
33
|
+
* trigger_hash)` unique index; a NEWER snapshot (distinct `snapshotId`) is a
|
|
34
|
+
* genuinely new opportunity and a distinct hash.
|
|
35
|
+
*/
|
|
36
|
+
function upgradeTriggerHash(route, reference, cause, snapshotId) {
|
|
37
|
+
return (0, node_crypto_1.createHash)("sha256")
|
|
38
|
+
.update(`upgrade_opportunity\n${route}\n${reference.provider}/${reference.model}\n${cause}\n${snapshotId ?? ""}`)
|
|
39
|
+
.digest("hex")
|
|
40
|
+
.slice(0, 32);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Idempotency key for a `regret_breach` — per (route, reference, window bounds,
|
|
44
|
+
* breachKind) ONLY (spec §5.2), so the sampled band's noisy per-run regret
|
|
45
|
+
* numbers never split a dedupe. The window bounds are day-quantized off the
|
|
46
|
+
* injected `now`, so a re-run the SAME day is a no-op via the `(org_id, route,
|
|
47
|
+
* trigger_hash)` unique index.
|
|
48
|
+
*/
|
|
49
|
+
function regretBreachTriggerHash(route, reference, windowFrom, windowTo, breachKind) {
|
|
50
|
+
return (0, node_crypto_1.createHash)("sha256")
|
|
51
|
+
.update(`regret_breach\n${route}\n${reference.provider}/${reference.model}\n${windowFrom}\n${windowTo}\n${breachKind}`)
|
|
52
|
+
.digest("hex")
|
|
53
|
+
.slice(0, 32);
|
|
54
|
+
}
|
|
27
55
|
exports.PROPOSAL_TTL_DAYS = 30;
|
|
28
56
|
/** Reference token mix for blending input/output rates into one comparable
|
|
29
57
|
* number: 3 input tokens per output token — a conservative, route-agnostic
|
package/dist/watch/monitor.d.ts
CHANGED
|
@@ -110,7 +110,13 @@ export interface MonitorReadsConfig {
|
|
|
110
110
|
}
|
|
111
111
|
/** PostgREST reads for a Monitor cycle (service role; mirrors the proposals
|
|
112
112
|
* client). Two org-scoped queries, joined in-SDK by `runMonitorCycle` — no DDL,
|
|
113
|
-
* no view (spec §3.1).
|
|
114
|
-
*
|
|
113
|
+
* no view (spec §3.1). The deterministic-eval read addresses `grades` directly
|
|
114
|
+
* (grade primitive, GP-2): `artifact_evaluations` was a compat view dropped by
|
|
115
|
+
* 00071, so this reads artifact-SUBJECT deterministic grades (`subject_kind=
|
|
116
|
+
* eq.artifact&kind=eq.deterministic`) and derives `passed = score >= 0.5`. This
|
|
117
|
+
* is the exact set the old read effectively used: run-subject deterministic rows
|
|
118
|
+
* carried a NULL `artifact_id`, so they never joined a task in `rollupConformRate`
|
|
119
|
+
* and were always dropped downstream. `taskFor` batches ids ≤100 (T9 — PostgREST
|
|
120
|
+
* URLs fail silently around 8KB). */
|
|
115
121
|
export declare function createMonitorReads(config: MonitorReadsConfig): MonitorReads;
|
|
116
122
|
//# sourceMappingURL=monitor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"monitor.d.ts","sourceRoot":"","sources":["../../src/watch/monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAA4C,aAAa,EAAE,MAAM,YAAY,CAAC;AAG1F;gEACgE;AAChE,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC;;;+EAG+E;AAC/E,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC;;oEAEoE;AACpE,eAAO,MAAM,kBAAkB,KAAK,CAAC;AACrC;uDACuD;AACvD,eAAO,MAAM,aAAa,KAAK,CAAC;AAEhC,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,sBAAsB,EAAE,aAKpC,CAAC;AAEF,wEAAwE;AACxE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC;AAED,gFAAgF;AAChF,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAYD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,SAAS,aAAa,EAAE,EAC9B,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAC1C,iBAAiB,EAAE,CA0BrB;AAeD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,SAAS,iBAAiB,EAAE,EACpC,GAAG,EAAE,IAAI,EACT,MAAM,GAAE,aAAsC,GAC7C,aAAa,EAAE,CAgCjB;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,wBAAwB,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,aAAa,EAAE,CAAC,CAAC;IAC3F;;4EAEwE;IACxE,QAAQ,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC5F;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IACjC,iFAAiF;IACjF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,EACnE,GAAG,EAAE,IAAI,EACT,MAAM,GAAE,aAAsC,GAC7C,OAAO,CAAC,kBAAkB,CAAC,CAU7B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;mFAC+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAYD
|
|
1
|
+
{"version":3,"file":"monitor.d.ts","sourceRoot":"","sources":["../../src/watch/monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAA4C,aAAa,EAAE,MAAM,YAAY,CAAC;AAG1F;gEACgE;AAChE,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC;;;+EAG+E;AAC/E,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC;;oEAEoE;AACpE,eAAO,MAAM,kBAAkB,KAAK,CAAC;AACrC;uDACuD;AACvD,eAAO,MAAM,aAAa,KAAK,CAAC;AAEhC,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,sBAAsB,EAAE,aAKpC,CAAC;AAEF,wEAAwE;AACxE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC;AAED,gFAAgF;AAChF,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAYD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,SAAS,aAAa,EAAE,EAC9B,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAC1C,iBAAiB,EAAE,CA0BrB;AAeD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,SAAS,iBAAiB,EAAE,EACpC,GAAG,EAAE,IAAI,EACT,MAAM,GAAE,aAAsC,GAC7C,aAAa,EAAE,CAgCjB;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,wBAAwB,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,aAAa,EAAE,CAAC,CAAC;IAC3F;;4EAEwE;IACxE,QAAQ,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC5F;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IACjC,iFAAiF;IACjF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,EACnE,GAAG,EAAE,IAAI,EACT,MAAM,GAAE,aAAsC,GAC7C,OAAO,CAAC,kBAAkB,CAAC,CAU7B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;mFAC+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAYD;;;;;;;;;qCASqC;AACrC,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,YAAY,CA+B3E"}
|
package/dist/watch/monitor.js
CHANGED
|
@@ -164,8 +164,14 @@ async function runMonitorCycle(reads, submitDrafts, now, config = exports.DEFAUL
|
|
|
164
164
|
}
|
|
165
165
|
/** PostgREST reads for a Monitor cycle (service role; mirrors the proposals
|
|
166
166
|
* client). Two org-scoped queries, joined in-SDK by `runMonitorCycle` — no DDL,
|
|
167
|
-
* no view (spec §3.1).
|
|
168
|
-
*
|
|
167
|
+
* no view (spec §3.1). The deterministic-eval read addresses `grades` directly
|
|
168
|
+
* (grade primitive, GP-2): `artifact_evaluations` was a compat view dropped by
|
|
169
|
+
* 00071, so this reads artifact-SUBJECT deterministic grades (`subject_kind=
|
|
170
|
+
* eq.artifact&kind=eq.deterministic`) and derives `passed = score >= 0.5`. This
|
|
171
|
+
* is the exact set the old read effectively used: run-subject deterministic rows
|
|
172
|
+
* carried a NULL `artifact_id`, so they never joined a task in `rollupConformRate`
|
|
173
|
+
* and were always dropped downstream. `taskFor` batches ids ≤100 (T9 — PostgREST
|
|
174
|
+
* URLs fail silently around 8KB). */
|
|
169
175
|
function createMonitorReads(config) {
|
|
170
176
|
const base = config.supabaseUrl.replace(/\/$/, "");
|
|
171
177
|
const fetchFn = config.fetchFn ?? fetch;
|
|
@@ -180,9 +186,11 @@ function createMonitorReads(config) {
|
|
|
180
186
|
}
|
|
181
187
|
return {
|
|
182
188
|
async recentDeterministicEvals(sinceIso) {
|
|
183
|
-
const rows = await get(`
|
|
184
|
-
`&
|
|
185
|
-
|
|
189
|
+
const rows = await get(`grades?select=subject_id,score,created_at&org_id=eq.${org}` +
|
|
190
|
+
`&subject_kind=eq.artifact&kind=eq.deterministic&created_at=gte.${encodeURIComponent(sinceIso)}`);
|
|
191
|
+
// Grade primitive: score is the NOT-NULL primitive; the pass/fail verdict is
|
|
192
|
+
// DERIVED (passed = score >= 0.5), never stored.
|
|
193
|
+
return rows.map((r) => ({ artifactId: r.subject_id, passed: r.score >= 0.5, createdAt: r.created_at }));
|
|
186
194
|
},
|
|
187
195
|
async taskFor(artifactIds) {
|
|
188
196
|
const out = new Map();
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `modelrig regret-monitor --route <route> [--dry-run] [--json]` — run ONE Q1
|
|
3
|
+
* regret-monitor cycle for a route locally (near-optimal spec §6). Prints the
|
|
4
|
+
* band (regret ± CI, n, the ε line, the 7d-vs-28d drift arrow, last-measured) and
|
|
5
|
+
* any `regret_breach` proposal it would file. `--dry-run` computes the band +
|
|
6
|
+
* draft WITHOUT persisting the cycle or submitting the proposal.
|
|
7
|
+
*
|
|
8
|
+
* This is the LOCAL/self-host proof of the cycle; the multi-tenant hosted run is
|
|
9
|
+
* `regret-monitor-main` (a POST-DEPLOY gate — keyed reference replay + Matt's
|
|
10
|
+
* per-route reference envelope). Reuses `runRegretMonitorCycle` unchanged (N2 — no
|
|
11
|
+
* second implementation). T12: reads/writes ride the `bakeoffs` mirror through the
|
|
12
|
+
* control-plane client, never a bespoke DB path.
|
|
13
|
+
*/
|
|
14
|
+
import type { Flags } from "../cli";
|
|
15
|
+
export declare function cmdRegretMonitor(flags: Flags): Promise<number>;
|
|
16
|
+
//# sourceMappingURL=regret-monitor-cli.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"regret-monitor-cli.d.ts","sourceRoot":"","sources":["../../src/watch/regret-monitor-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAiBH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AA8BpC,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CA0FpE"}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `modelrig regret-monitor --route <route> [--dry-run] [--json]` — run ONE Q1
|
|
4
|
+
* regret-monitor cycle for a route locally (near-optimal spec §6). Prints the
|
|
5
|
+
* band (regret ± CI, n, the ε line, the 7d-vs-28d drift arrow, last-measured) and
|
|
6
|
+
* any `regret_breach` proposal it would file. `--dry-run` computes the band +
|
|
7
|
+
* draft WITHOUT persisting the cycle or submitting the proposal.
|
|
8
|
+
*
|
|
9
|
+
* This is the LOCAL/self-host proof of the cycle; the multi-tenant hosted run is
|
|
10
|
+
* `regret-monitor-main` (a POST-DEPLOY gate — keyed reference replay + Matt's
|
|
11
|
+
* per-route reference envelope). Reuses `runRegretMonitorCycle` unchanged (N2 — no
|
|
12
|
+
* second implementation). T12: reads/writes ride the `bakeoffs` mirror through the
|
|
13
|
+
* control-plane client, never a bespoke DB path.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.cmdRegretMonitor = cmdRegretMonitor;
|
|
17
|
+
const config_1 = require("../config");
|
|
18
|
+
const loader_1 = require("../routes/loader");
|
|
19
|
+
const rawonly_1 = require("../run/rawonly");
|
|
20
|
+
const index_1 = require("../index");
|
|
21
|
+
const node_os_1 = require("node:os");
|
|
22
|
+
const proposals_1 = require("./proposals");
|
|
23
|
+
const regret_monitor_1 = require("./regret-monitor");
|
|
24
|
+
/** Format a band as "regret 1.4 pts (CI 0.6…2.3, n=180)" — every regret number
|
|
25
|
+
* carries n + CI (regret-claim-ceiling). Null band ⇒ "no cycles yet". */
|
|
26
|
+
function fmtBand(band) {
|
|
27
|
+
if (band === null)
|
|
28
|
+
return "no cycles in window yet";
|
|
29
|
+
const pts = (v) => (v * 100).toFixed(1);
|
|
30
|
+
return `regret ${pts(band.regretMean)} pts (CI ${pts(band.regretCi95[0])}…${pts(band.regretCi95[1])}, n=${band.nPaired})`;
|
|
31
|
+
}
|
|
32
|
+
/** The 7d-vs-28d drift arrow (⬆ worse / ⬇ better / → flat), or "—" when a side
|
|
33
|
+
* is missing. */
|
|
34
|
+
function driftArrow(recent, baseline) {
|
|
35
|
+
if (recent === null || baseline === null)
|
|
36
|
+
return "—";
|
|
37
|
+
const delta = recent.regretMean - baseline.regretMean;
|
|
38
|
+
if (Math.abs(delta) < 1e-9)
|
|
39
|
+
return "→ flat";
|
|
40
|
+
return delta > 0 ? `⬆ +${(delta * 100).toFixed(1)} pts vs 28d` : `⬇ ${(delta * 100).toFixed(1)} pts vs 28d`;
|
|
41
|
+
}
|
|
42
|
+
/** Resolve the reference identity (provider/model) for the trigger + dedupe hash
|
|
43
|
+
* from a resolved bake-off artifact — handles `reference: auto` (the concrete
|
|
44
|
+
* model is only known after the engine resolves it). */
|
|
45
|
+
function referenceOf(artifact) {
|
|
46
|
+
const model = artifact.reference?.model;
|
|
47
|
+
if (typeof model !== "string")
|
|
48
|
+
return null;
|
|
49
|
+
const slash = model.indexOf("/");
|
|
50
|
+
if (slash <= 0)
|
|
51
|
+
return null;
|
|
52
|
+
return { provider: model.slice(0, slash), model: model.slice(slash + 1) };
|
|
53
|
+
}
|
|
54
|
+
async function cmdRegretMonitor(flags) {
|
|
55
|
+
const route = flags.named.get("route");
|
|
56
|
+
if (route === undefined || route === "") {
|
|
57
|
+
throw new Error("usage: modelrig regret-monitor --route <route> [--dry-run] [--json]");
|
|
58
|
+
}
|
|
59
|
+
const dryRun = flags.bare.has("dry-run");
|
|
60
|
+
const asJson = flags.bare.has("json");
|
|
61
|
+
const config = (0, config_1.loadConfigFromEnv)();
|
|
62
|
+
const routes = (0, loader_1.loadRoutes)((0, rawonly_1.requireRoutesDir)(config, "modelrig regret-monitor"));
|
|
63
|
+
const bundle = routes.get(route);
|
|
64
|
+
if (bundle.bakeoff?.reference === undefined) {
|
|
65
|
+
throw new Error(`route "${route}" declares no bakeoff.reference — there is no ceiling to measure regret against. ` +
|
|
66
|
+
"Add bakeoff.reference (auto or provider/model) and, optionally, bakeoff.regret_monitor.");
|
|
67
|
+
}
|
|
68
|
+
const rm = bundle.bakeoff.regretMonitor;
|
|
69
|
+
const sampleN = rm?.sampleN ?? 200;
|
|
70
|
+
const referenceEnvelope = bundle.bakeoff.referenceEnvelopeUsd;
|
|
71
|
+
const rig = (0, index_1.createRig)(config);
|
|
72
|
+
try {
|
|
73
|
+
// Run the sampled 2-arm bake-off ONCE (default vs the route's reference — the
|
|
74
|
+
// reference arm runs automatically because the route declares it). Memoize it
|
|
75
|
+
// so `runRegretMonitorCycle`'s injected collaborator returns the same artifact
|
|
76
|
+
// and we can resolve the (possibly `auto`) reference identity from it first.
|
|
77
|
+
const artifact = await rig.bakeoff(route, {
|
|
78
|
+
variants: ["default"],
|
|
79
|
+
replayLastN: sampleN,
|
|
80
|
+
...(referenceEnvelope !== undefined ? { envelopeUsdPerVariant: referenceEnvelope } : {}),
|
|
81
|
+
});
|
|
82
|
+
const reference = referenceOf(artifact);
|
|
83
|
+
if (reference === null) {
|
|
84
|
+
throw new Error(`route "${route}" resolved no reference arm on this cycle (no reference model, or nothing to replay) — ` +
|
|
85
|
+
"cannot form a regret band. Check the reference declaration and that captures exist.");
|
|
86
|
+
}
|
|
87
|
+
const cfg = (0, regret_monitor_1.regretMonitorConfig)({
|
|
88
|
+
route,
|
|
89
|
+
reference,
|
|
90
|
+
epsilon: bundle.objective?.maxRegret ?? null,
|
|
91
|
+
...(rm?.windowDays !== undefined ? { windowDays: rm.windowDays } : {}),
|
|
92
|
+
...(rm?.baselineDays !== undefined ? { baselineDays: rm.baselineDays } : {}),
|
|
93
|
+
});
|
|
94
|
+
// Reads: the real bakeoffs-mirror when a direct control plane is configured;
|
|
95
|
+
// an in-memory stub for a keyless local proof (no prior cycles, no persist).
|
|
96
|
+
let reads;
|
|
97
|
+
let submitDrafts;
|
|
98
|
+
if (config.export?.mode === "direct" && !dryRun) {
|
|
99
|
+
const exp = (0, config_1.directExportOf)(config, "modelrig regret-monitor");
|
|
100
|
+
reads = (0, regret_monitor_1.createRegretMonitorReads)({
|
|
101
|
+
supabaseUrl: exp.supabaseUrl,
|
|
102
|
+
serviceRoleKey: exp.serviceRoleKey,
|
|
103
|
+
route,
|
|
104
|
+
sourceId: exp.sourceId ?? (0, node_os_1.hostname)(),
|
|
105
|
+
});
|
|
106
|
+
const client = (0, proposals_1.createProposalsClient)({ supabaseUrl: exp.supabaseUrl, serviceRoleKey: exp.serviceRoleKey });
|
|
107
|
+
submitDrafts = (drafts) => client.submitDrafts(drafts);
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
// Local/dry proof: band from THIS cycle only, no persistence, no submit.
|
|
111
|
+
reads = { recentCycles: async () => [], persistCycle: async () => { } };
|
|
112
|
+
submitDrafts = async () => 0;
|
|
113
|
+
}
|
|
114
|
+
const result = await (0, regret_monitor_1.runRegretMonitorCycle)(reads, async () => artifact, submitDrafts, new Date(), cfg);
|
|
115
|
+
if (asJson) {
|
|
116
|
+
console.log(JSON.stringify({ route, band: result.band, baseline: result.baseline, drafts: result.drafts, inserted: result.inserted, dryRun }, null, 2));
|
|
117
|
+
return 0;
|
|
118
|
+
}
|
|
119
|
+
const epsLine = cfg.epsilon === null ? "ε: none declared (drift-only)" : `ε (max_regret): ${(cfg.epsilon * 100).toFixed(1)} pts`;
|
|
120
|
+
console.log(`regret-monitor ${route} vs reference ${reference.provider}/${reference.model}`);
|
|
121
|
+
console.log(` recent (7d): ${fmtBand(result.band)}`);
|
|
122
|
+
console.log(` baseline(28d): ${fmtBand(result.baseline)}`);
|
|
123
|
+
console.log(` ${epsLine} · drift ${driftArrow(result.band, result.baseline)} · last measured ${artifact.ranAt}`);
|
|
124
|
+
if (result.drafts.length > 0) {
|
|
125
|
+
const t = result.drafts[0].trigger;
|
|
126
|
+
console.log(` BREACH (${t.breachKind ?? "?"}) → ${dryRun ? "would file" : `filed ${result.inserted}`} ` +
|
|
127
|
+
"one regret_breach bake-off proposal. Approval stays human: run the bake-off, then swap execute --approve.");
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
console.log(" within bounds — no proposal (measured-only; below min-n both sides ⇒ also no proposal).");
|
|
131
|
+
}
|
|
132
|
+
return 0;
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
await rig.close();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Q1 regret monitor — a standing measurement of how far a served route sits from
|
|
3
|
+
* its reference/ceiling over time (near-optimal spec §5). The quality twin of the
|
|
4
|
+
* cost odometer: the watch loop reacts to PRICE, the conform Monitor
|
|
5
|
+
* (`watch/monitor.ts`) reacts to shape, and THIS reacts to REGRET drifting up.
|
|
6
|
+
*
|
|
7
|
+
* A cycle is, precisely, a 2-arm SAMPLED bake-off — `default` vs the route's
|
|
8
|
+
* `__reference__` — under `reference_envelope_usd`. It therefore REUSES the
|
|
9
|
+
* bake-off engine (`runBakeoff`) wholesale via the injected `runSampledBakeoff`
|
|
10
|
+
* collaborator: the reference arm, `replayVariant`, `pairedRegret`, partial-result
|
|
11
|
+
* honesty and metering all come for free. This file adds NO regret estimator — it
|
|
12
|
+
* is the Monitor's STRUCTURE (a per-route band, a drift rule, a cron) wrapped
|
|
13
|
+
* around the number the engine already produces (`metrics.regret`).
|
|
14
|
+
*
|
|
15
|
+
* ── MEASURED-ONLY DOCTRINE (mirrors the conform Monitor) ───────────────────
|
|
16
|
+
* A breach is proposed only when BOTH the recent window and the trailing baseline
|
|
17
|
+
* clear `REGRET_MONITOR_MIN_N` PAIRED samples. Below that the band is too noisy to
|
|
18
|
+
* act on — the house rule is to refuse rather than propose on thin data. The rule
|
|
19
|
+
* is deterministic over an injected `now`, so the fixture test is reproducible.
|
|
20
|
+
*
|
|
21
|
+
* ZERO DDL (spec §10): a cycle mirrors as a `bakeoffs` row MARKED a monitor cycle
|
|
22
|
+
* inside the existing matrix jsonb (`regret-band-provenance`) so the band reads
|
|
23
|
+
* back over time, never double-proposes, and never seeds the odometer.
|
|
24
|
+
*/
|
|
25
|
+
import type { BakeoffArtifact } from "../bakeoff/types";
|
|
26
|
+
import type { CandidateEconomics, ProposalDraft, RegretBandSummary } from "./evaluate";
|
|
27
|
+
/** The recent window whose regret we judge — a week, mirroring the conform
|
|
28
|
+
* Monitor (smooths daily noise, stays responsive to a just-landed regression). */
|
|
29
|
+
export declare const REGRET_MONITOR_WINDOW_DAYS = 7;
|
|
30
|
+
/** The trailing baseline the window is compared against — the 28 days before it
|
|
31
|
+
* (non-overlapping), mirroring the conform Monitor + the proposal TTL cadence. */
|
|
32
|
+
export declare const REGRET_MONITOR_BASELINE_DAYS = 28;
|
|
33
|
+
/** Minimum PAIRED samples required in EACH window before the band is trustworthy
|
|
34
|
+
* enough to propose on (measured-only; mirrors `MONITOR_MIN_N`). */
|
|
35
|
+
export declare const REGRET_MONITOR_MIN_N = 20;
|
|
36
|
+
/** Drift-up margin in QUALITY POINTS (0..1 — the same scale as `regretMean` and
|
|
37
|
+
* ε / `max_regret`). A recent window whose mean regret sits this far ABOVE the
|
|
38
|
+
* baseline is materially worse even while still under ε, and files a drift_up
|
|
39
|
+
* breach. Distinct from ε (the absolute bound); this is the relative move. */
|
|
40
|
+
export declare const REGRET_MONITOR_MARGIN = 0.02;
|
|
41
|
+
export interface RegretMonitorConfig {
|
|
42
|
+
/** The route this cycle monitors. */
|
|
43
|
+
readonly route: string;
|
|
44
|
+
/** The reference (ceiling) it measures against — recorded on the trigger. */
|
|
45
|
+
readonly reference: CandidateEconomics;
|
|
46
|
+
/** The route's `objective.max_regret` (ε). null ⇒ drift-only (no over_epsilon
|
|
47
|
+
* breach; a drift_up breach can still fire). */
|
|
48
|
+
readonly epsilon: number | null;
|
|
49
|
+
readonly windowDays: number;
|
|
50
|
+
readonly baselineDays: number;
|
|
51
|
+
/** Drift-up margin, quality points (0..1). */
|
|
52
|
+
readonly margin: number;
|
|
53
|
+
readonly minN: number;
|
|
54
|
+
}
|
|
55
|
+
/** Build a config from the route's declared knobs, defaulting to the house
|
|
56
|
+
* constants (mirrors `DEFAULT_MONITOR_CONFIG`). */
|
|
57
|
+
export declare function regretMonitorConfig(input: {
|
|
58
|
+
readonly route: string;
|
|
59
|
+
readonly reference: CandidateEconomics;
|
|
60
|
+
readonly epsilon: number | null;
|
|
61
|
+
readonly windowDays?: number;
|
|
62
|
+
readonly baselineDays?: number;
|
|
63
|
+
readonly margin?: number;
|
|
64
|
+
readonly minN?: number;
|
|
65
|
+
}): RegretMonitorConfig;
|
|
66
|
+
/** One persisted monitor cycle's regret point (the `default` arm's regret vs the
|
|
67
|
+
* reference on that cycle's paired sample). Read back to form the band. */
|
|
68
|
+
export interface RegretCyclePoint {
|
|
69
|
+
readonly date: string;
|
|
70
|
+
readonly regretMean: number;
|
|
71
|
+
readonly regretCi95: readonly [number, number];
|
|
72
|
+
readonly nPaired: number;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Aggregate the cycle points whose date is in [from, to) into one band. The
|
|
76
|
+
* pooled mean + CI bounds are nPaired-WEIGHTED across the window's cycles (each
|
|
77
|
+
* cycle already carries its own seeded paired-bootstrap CI from `pairedRegret`);
|
|
78
|
+
* `nPaired` is the window's paired total — the min-n gate input. Honest-limits:
|
|
79
|
+
* this pools per-cycle summaries, not per-sample data, so the CI is a weighted
|
|
80
|
+
* blend of the cycles' intervals, not a fresh cross-cycle bootstrap. Null when
|
|
81
|
+
* the window has no cycles.
|
|
82
|
+
*/
|
|
83
|
+
export declare function aggregateRegretBand(points: readonly RegretCyclePoint[], from: string, to: string): RegretBandSummary | null;
|
|
84
|
+
/**
|
|
85
|
+
* The regret drift rule (measured-only) — mirrors `evaluateConformDegradation`.
|
|
86
|
+
* Compares the recent window band to the trailing baseline band; emits ONE
|
|
87
|
+
* `regret_breach` bake-off draft when either
|
|
88
|
+
* • recent.regretCi95[1] > ε (over_epsilon), or
|
|
89
|
+
* • recent.regretMean − baseline.regretMean ≥ margin (drift_up),
|
|
90
|
+
* with `nPaired ≥ MIN_N` on BOTH windows; else refuse. Deterministic over `now`.
|
|
91
|
+
* The trigger_hash keys on (route, reference, window bounds, breachKind) so a
|
|
92
|
+
* same-day re-run dedupes.
|
|
93
|
+
*/
|
|
94
|
+
export declare function evaluateRegretDrift(series: readonly RegretCyclePoint[], now: Date, config: RegretMonitorConfig): ProposalDraft[];
|
|
95
|
+
/**
|
|
96
|
+
* Extract this cycle's regret point from a freshly-run 2-arm sampled bake-off —
|
|
97
|
+
* the `default` arm's regret vs the reference. Null when the artifact carries no
|
|
98
|
+
* reference/regret (no ceiling resolved ⇒ nothing to band). Pure.
|
|
99
|
+
*/
|
|
100
|
+
export declare function cyclePointFromArtifact(artifact: BakeoffArtifact, now: Date): RegretCyclePoint | null;
|
|
101
|
+
/** A cycle to persist as a monitor-marked `bakeoffs` artifact (spec §10). Carries
|
|
102
|
+
* the full artifact (the mirror row) + the monitor marker + the epsilon it was
|
|
103
|
+
* judged against, so the band reads back with provenance. */
|
|
104
|
+
export interface PersistedRegretCycle {
|
|
105
|
+
readonly route: string;
|
|
106
|
+
readonly artifact: BakeoffArtifact;
|
|
107
|
+
readonly point: RegretCyclePoint;
|
|
108
|
+
readonly epsilon: number | null;
|
|
109
|
+
/** Always true — the provenance marker (`regret-band-provenance`). */
|
|
110
|
+
readonly monitorCycle: true;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* The reads a regret-monitor cycle needs, injected so the orchestrator is
|
|
114
|
+
* testable without a database (mirrors `MonitorReads`). Both are scoped to ONE
|
|
115
|
+
* (org, route): a hosted cron iterates them.
|
|
116
|
+
*/
|
|
117
|
+
export interface RegretMonitorReads {
|
|
118
|
+
/** Prior monitor-cycle regret points for this route, since `sinceIso` — the
|
|
119
|
+
* `bakeoffs` mirror filtered to MONITOR cycles (regret-band-provenance). MUST
|
|
120
|
+
* exclude non-monitor (hand-run) bake-offs so the band is provenance-clean. */
|
|
121
|
+
readonly recentCycles: (sinceIso: string) => Promise<readonly RegretCyclePoint[]>;
|
|
122
|
+
/** Persist a freshly-run cycle as a monitor-marked `bakeoffs` artifact. */
|
|
123
|
+
readonly persistCycle: (cycle: PersistedRegretCycle) => Promise<void>;
|
|
124
|
+
}
|
|
125
|
+
export interface RegretMonitorCycleResult {
|
|
126
|
+
/** The recent-window band for the surface (regret ± CI, n) — null when the
|
|
127
|
+
* window has no cycles yet. */
|
|
128
|
+
readonly band: RegretBandSummary | null;
|
|
129
|
+
/** The trailing baseline band — for the drift arrow on the surface. */
|
|
130
|
+
readonly baseline: RegretBandSummary | null;
|
|
131
|
+
/** This cycle's fresh point (null when no reference/regret resolved). */
|
|
132
|
+
readonly point: RegretCyclePoint | null;
|
|
133
|
+
readonly drafts: ProposalDraft[];
|
|
134
|
+
/** How many proposals the control plane actually created (0 on duplicates). */
|
|
135
|
+
readonly inserted: number;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* One regret-monitor cycle (mirror of `runMonitorCycle`): read the prior monitor
|
|
139
|
+
* cycles, run ONE sampled bake-off (default vs the reference) via the injected
|
|
140
|
+
* collaborator, persist it as a monitor-marked artifact, then apply the drift
|
|
141
|
+
* rule over [prior + this cycle] and submit any breach draft through the EXISTING
|
|
142
|
+
* proposals client (idempotent on (route, trigger_hash)). The prior series is
|
|
143
|
+
* read BEFORE persisting this cycle, so the fresh point is counted exactly once.
|
|
144
|
+
* Never actuates — submitting a proposal is the whole of its authority.
|
|
145
|
+
*/
|
|
146
|
+
export declare function runRegretMonitorCycle(reads: RegretMonitorReads, runSampledBakeoff: () => Promise<BakeoffArtifact>, submitDrafts: (drafts: readonly ProposalDraft[]) => Promise<number>, now: Date, config: RegretMonitorConfig): Promise<RegretMonitorCycleResult>;
|
|
147
|
+
/** The sentinel arm name that marks a `bakeoffs` matrix as a Q1 monitor cycle.
|
|
148
|
+
* A user variant can never be named this (names cannot start with `__`). */
|
|
149
|
+
export declare const MONITOR_CYCLE_MARKER = "__monitor_cycle__";
|
|
150
|
+
/** The `source_id` prefix that distinguishes a monitor cycle from a hand-run
|
|
151
|
+
* bake-off in the mirror. */
|
|
152
|
+
export declare const MONITOR_SOURCE_PREFIX = "regret-monitor:";
|
|
153
|
+
export interface RegretMonitorReadsConfig {
|
|
154
|
+
readonly supabaseUrl: string;
|
|
155
|
+
readonly serviceRoleKey: string;
|
|
156
|
+
/** The route this reads/writes cycles for (scopes both queries). */
|
|
157
|
+
readonly route: string;
|
|
158
|
+
/** The org this cycle runs for — reads/writes are org-scoped so tenants never
|
|
159
|
+
* merge. A hosted cron iterates orgs; a self-host has exactly one. */
|
|
160
|
+
readonly orgId?: string;
|
|
161
|
+
/** The host that ran the cycle — recorded after the monitor prefix. */
|
|
162
|
+
readonly sourceId: string;
|
|
163
|
+
readonly fetchFn?: typeof fetch;
|
|
164
|
+
}
|
|
165
|
+
/** PostgREST reads for a regret-monitor cycle (service role; mirrors
|
|
166
|
+
* createMonitorReads). ZERO DDL — reuses the `bakeoffs` mirror. */
|
|
167
|
+
export declare function createRegretMonitorReads(config: RegretMonitorReadsConfig): RegretMonitorReads;
|
|
168
|
+
//# sourceMappingURL=regret-monitor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"regret-monitor.d.ts","sourceRoot":"","sources":["../../src/watch/regret-monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EACV,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EAElB,MAAM,YAAY,CAAC;AAGpB;kFACkF;AAClF,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAC5C;kFACkF;AAClF,eAAO,MAAM,4BAA4B,KAAK,CAAC;AAC/C;oEACoE;AACpE,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC;;;8EAG8E;AAC9E,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAI1C,MAAM,WAAW,mBAAmB;IAClC,qCAAqC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;IACvC;oDACgD;IAChD,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,8CAA8C;IAC9C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;mDACmD;AACnD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB,GAAG,mBAAmB,CAUtB;AAED;2EAC2E;AAC3E,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAgBD;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,SAAS,gBAAgB,EAAE,EACnC,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM,GACT,iBAAiB,GAAG,IAAI,CA6B1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,SAAS,gBAAgB,EAAE,EACnC,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,mBAAmB,GAC1B,aAAa,EAAE,CAoCjB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,EAAE,IAAI,GAAG,gBAAgB,GAAG,IAAI,CAWpG;AAED;;6DAE6D;AAC7D,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC;IACjC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,sEAAsE;IACtE,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC;;mFAE+E;IAC/E,QAAQ,CAAC,YAAY,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,gBAAgB,EAAE,CAAC,CAAC;IAClF,2EAA2E;IAC3E,QAAQ,CAAC,YAAY,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvE;AAED,MAAM,WAAW,wBAAwB;IACvC;mCAC+B;IAC/B,QAAQ,CAAC,IAAI,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACxC,uEAAuE;IACvE,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAC5C,yEAAyE;IACzE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IACjC,+EAA+E;IAC/E,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,kBAAkB,EACzB,iBAAiB,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC,EACjD,YAAY,EAAE,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,EACnE,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,wBAAwB,CAAC,CAwBnC;AAaD;4EAC4E;AAC5E,eAAO,MAAM,oBAAoB,sBAAsB,CAAC;AACxD;6BAC6B;AAC7B,eAAO,MAAM,qBAAqB,oBAAoB,CAAC;AAEvD,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;0EACsE;IACtE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AA+BD;mEACmE;AACnE,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,wBAAwB,GAAG,kBAAkB,CAqD7F"}
|