modelrig 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +41 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +46 -6
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +105 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +37 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +138 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +122 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
package/dist/watch/evaluate.d.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* the same facts produces the same hashes, and the control-plane unique
|
|
15
15
|
* index (route, trigger_hash) makes duplicates a no-op.
|
|
16
16
|
*/
|
|
17
|
+
import type { CacheHysteresisConfig, CacheVerdict, EffectiveRates, ObservedCacheRates } from "../registry/effective-rate";
|
|
17
18
|
export interface CandidateEconomics {
|
|
18
19
|
readonly provider: string;
|
|
19
20
|
readonly model: string;
|
|
@@ -41,9 +42,85 @@ export type WatchEvent = {
|
|
|
41
42
|
export interface WatchConfig {
|
|
42
43
|
/** Material-shift threshold in percent (spec default ±10). */
|
|
43
44
|
readonly thresholdPct: number;
|
|
45
|
+
/** Cache-aware hysteresis (cache-aware-routing spec §4.3) — the absolute
|
|
46
|
+
* per-call floor and relative window-spend threshold a cache-adjusted swap
|
|
47
|
+
* must clear to be `cache_positive`. OPTIONAL & additive: absent ⇒
|
|
48
|
+
* `DEFAULT_CACHE_HYSTERESIS`. Same home as `thresholdPct`, distinct numbers —
|
|
49
|
+
* the price-move threshold and the cache-swap hysteresis are different gates. */
|
|
50
|
+
readonly cacheHysteresis?: CacheHysteresisConfig;
|
|
44
51
|
}
|
|
45
|
-
/** Trigger payload persisted on the proposal row (jsonb).
|
|
46
|
-
|
|
52
|
+
/** Trigger payload persisted on the proposal row (jsonb). A discriminated union
|
|
53
|
+
* on `type`: the watch loop's price/model economics triggers, the Monitor's
|
|
54
|
+
* conform-rate degradation trigger (E4 chunk B), and the Rig Analyst's cluster
|
|
55
|
+
* findings (E6 chunk A). The watch/monitor triggers keep KIND `bakeoff`; the
|
|
56
|
+
* Analyst's triggers ride the widened kinds (`insight`/`promote-eval`) or reuse
|
|
57
|
+
* `bakeoff` for an Analyst-initiated bake-off — all through the SAME proposals
|
|
58
|
+
* machinery and the SAME `submitDrafts` seam (trajectory F1/F4, one emission path). */
|
|
59
|
+
export type ProposalTrigger = EconomicsTrigger | ConformDegradationTrigger | AnalystClusterTrigger | AnalystPromoteEvalTrigger | AnalystBakeoffTrigger;
|
|
60
|
+
/** The measured statistics behind one Analyst cluster — all deterministic. */
|
|
61
|
+
export interface AnalystClusterStats {
|
|
62
|
+
/** Runs in this (task, cluster) over the window. */
|
|
63
|
+
readonly runs: number;
|
|
64
|
+
/** This cluster's fail rate, 0..1 (run-outcome@v1 verdicts). */
|
|
65
|
+
readonly failRate: number;
|
|
66
|
+
/** The org's window-wide fail rate, 0..1 — the salience baseline. */
|
|
67
|
+
readonly windowFailRate: number;
|
|
68
|
+
/** failRate − windowFailRate, in percentage POINTS (the salience margin). */
|
|
69
|
+
readonly failPtsOverWindow: number;
|
|
70
|
+
/** Feedback down-rate for the task, 0..1 (downs / (ups+downs)); 0 when none. */
|
|
71
|
+
readonly downRate: number;
|
|
72
|
+
/** Feedback signals (ups+downs) behind `downRate` — the min-n gate input. */
|
|
73
|
+
readonly feedbackN: number;
|
|
74
|
+
/** The dominant run-outcome failure_class label for the cluster, or null. */
|
|
75
|
+
readonly dominantFailureClass: string | null;
|
|
76
|
+
}
|
|
77
|
+
/** One reporting cluster → the `insight` proposal's trigger. `clusterKey` is the
|
|
78
|
+
* within-task discriminator (`fail:<class>` | `feedback` | `ok`); the window END
|
|
79
|
+
* (quantized to day) is what the trigger_hash keys on so re-runs dedupe. */
|
|
80
|
+
export interface AnalystClusterTrigger {
|
|
81
|
+
readonly type: "analyst_cluster";
|
|
82
|
+
readonly task: string;
|
|
83
|
+
readonly clusterKey: string;
|
|
84
|
+
/** Trailing window bounds — inclusive-from / exclusive-to ISO days. */
|
|
85
|
+
readonly windowFrom: string;
|
|
86
|
+
readonly windowTo: string;
|
|
87
|
+
readonly stats: AnalystClusterStats;
|
|
88
|
+
/** Exemplar run ids (≤10) so a human can open the E2 run pages. */
|
|
89
|
+
readonly exemplarRunIds: readonly string[];
|
|
90
|
+
}
|
|
91
|
+
/** A failing cluster's promotable failing artifacts → the `promote-eval`
|
|
92
|
+
* proposal's trigger. Approving iterates `candidates` through the EXISTING
|
|
93
|
+
* promote path (≤ ANALYST_PROMOTE_MAX items). `task` is what each promote records. */
|
|
94
|
+
export interface AnalystPromoteEvalTrigger {
|
|
95
|
+
readonly type: "analyst_promote_eval";
|
|
96
|
+
readonly task: string;
|
|
97
|
+
readonly clusterKey: string;
|
|
98
|
+
readonly windowFrom: string;
|
|
99
|
+
readonly windowTo: string;
|
|
100
|
+
readonly candidates: readonly AnalystPromoteCandidate[];
|
|
101
|
+
}
|
|
102
|
+
export interface AnalystPromoteCandidate {
|
|
103
|
+
readonly runId: string;
|
|
104
|
+
readonly artifactId: string;
|
|
105
|
+
}
|
|
106
|
+
/** A cluster on a route with a cheaper NON-serving candidate → an Analyst-
|
|
107
|
+
* initiated `bakeoff` proposal's trigger (the wedge motion). Only candidates the
|
|
108
|
+
* route already DECLARES may appear (STOP list: no novel challenger generation,
|
|
109
|
+
* that is E8). */
|
|
110
|
+
export interface AnalystBakeoffTrigger {
|
|
111
|
+
readonly type: "analyst_bakeoff";
|
|
112
|
+
readonly task: string;
|
|
113
|
+
readonly route: string;
|
|
114
|
+
readonly clusterKey: string;
|
|
115
|
+
readonly windowFrom: string;
|
|
116
|
+
readonly windowTo: string;
|
|
117
|
+
/** The cheaper non-serving candidate the Analyst proposes baking off. */
|
|
118
|
+
readonly challenger: CandidateEconomics;
|
|
119
|
+
/** The route's current serving candidates (context for the human). */
|
|
120
|
+
readonly incumbents: readonly CandidateEconomics[];
|
|
121
|
+
}
|
|
122
|
+
/** Watch-loop trigger: a candidate's price moved, or a new model appeared. */
|
|
123
|
+
export interface EconomicsTrigger {
|
|
47
124
|
readonly type: "price_change" | "new_model";
|
|
48
125
|
readonly provider: string;
|
|
49
126
|
readonly model: string;
|
|
@@ -53,14 +130,96 @@ export interface ProposalTrigger {
|
|
|
53
130
|
readonly newRates?: TokenRates;
|
|
54
131
|
readonly pricing?: TokenRates | null;
|
|
55
132
|
}
|
|
133
|
+
/** One side of the degradation comparison (E4 chunk B) — a window's conform
|
|
134
|
+
* rate over its evaluations. Dates are inclusive-from / exclusive-to ISO days. */
|
|
135
|
+
export interface ConformWindow {
|
|
136
|
+
readonly from: string;
|
|
137
|
+
readonly to: string;
|
|
138
|
+
readonly n: number;
|
|
139
|
+
readonly conformRate: number;
|
|
140
|
+
}
|
|
141
|
+
/** Monitor trigger (E4 chunk B): the recent window's deterministic conform rate
|
|
142
|
+
* fell below the trailing baseline by ≥ the ruled margin, with min-n met on both
|
|
143
|
+
* sides. Records BOTH windows + their ns so the evidence is self-explanatory. */
|
|
144
|
+
export interface ConformDegradationTrigger {
|
|
145
|
+
readonly type: "conform_degradation";
|
|
146
|
+
readonly task: string;
|
|
147
|
+
readonly window: ConformWindow;
|
|
148
|
+
readonly baseline: ConformWindow;
|
|
149
|
+
/** The drop that flagged it, in percentage POINTS of conform rate. */
|
|
150
|
+
readonly dropPts: number;
|
|
151
|
+
readonly marginPts: number;
|
|
152
|
+
readonly minN: number;
|
|
153
|
+
}
|
|
154
|
+
/** The proposal-kind vocabulary. `bakeoff`/`swap` are the shipped watch/monitor
|
|
155
|
+
* kinds (00006); `insight`/`promote-eval` join for the Rig Analyst (E6, migration
|
|
156
|
+
* 00052 widens the DB CHECK to match). */
|
|
157
|
+
export type ProposalKind = "bakeoff" | "swap" | "insight" | "promote-eval";
|
|
56
158
|
export interface ProposalDraft {
|
|
57
159
|
readonly route: string;
|
|
58
|
-
readonly kind:
|
|
160
|
+
readonly kind: ProposalKind;
|
|
59
161
|
readonly trigger: ProposalTrigger;
|
|
60
162
|
readonly triggerHash: string;
|
|
61
163
|
readonly expiresAt: string;
|
|
164
|
+
/** Evidence seeded AT INSERT (optional). The Rig Analyst embeds a new insight
|
|
165
|
+
* cluster's summary here (Chunk B) so the summary rides the same idempotent
|
|
166
|
+
* insert as the row — a duplicate (already-seen trigger_hash) is a no-op and
|
|
167
|
+
* never overwrites. Watch/monitor never set it. */
|
|
168
|
+
readonly evidence?: Record<string, unknown>;
|
|
62
169
|
}
|
|
63
170
|
export declare const PROPOSAL_TTL_DAYS = 30;
|
|
64
171
|
export declare function triggerHash(route: string, trigger: ProposalTrigger): string;
|
|
65
172
|
export declare function evaluateEvents(routes: readonly RouteEconomics[], events: readonly WatchEvent[], config: WatchConfig): ProposalDraft[];
|
|
173
|
+
/** The §5.5 evidence block — every field the rendered explanation and the
|
|
174
|
+
* verdict need, in one self-describing object. Rates are per-Mtok (the surface
|
|
175
|
+
* unit); switch/option/spend quantities are USD. */
|
|
176
|
+
export interface CacheContext {
|
|
177
|
+
readonly observed_hit_rate: number;
|
|
178
|
+
readonly observed_write_rate: number;
|
|
179
|
+
readonly window_days: number;
|
|
180
|
+
readonly incumbent_effective_in_per_mtok: number;
|
|
181
|
+
readonly challenger_cold_in_per_mtok: number;
|
|
182
|
+
readonly switch_cost_usd: number;
|
|
183
|
+
readonly cache_option_value_usd: number;
|
|
184
|
+
readonly breakeven_conformant_calls: number;
|
|
185
|
+
readonly verdict: CacheVerdict;
|
|
186
|
+
}
|
|
187
|
+
/** The inputs `buildCacheContext` needs — the incumbent's observed cache
|
|
188
|
+
* behavior + both candidates' rates + the switch geometry. Kept as explicit
|
|
189
|
+
* separate inputs (NOT a widened `watch/evaluate` local `TokenRates`) per the
|
|
190
|
+
* QA name-collision note: the effective blend takes the rollup and the rate
|
|
191
|
+
* cards as distinct arguments. */
|
|
192
|
+
export interface CacheContextInput {
|
|
193
|
+
/** Incumbent's trailing-window observed cache rates (from `rollupCacheStats`). */
|
|
194
|
+
readonly observed: ObservedCacheRates;
|
|
195
|
+
readonly windowDays: number;
|
|
196
|
+
/** Incumbent (warm) rate card, USD/token. */
|
|
197
|
+
readonly incumbentRates: EffectiveRates;
|
|
198
|
+
/** Challenger (cold) rate card, USD/token. */
|
|
199
|
+
readonly challengerRates: EffectiveRates;
|
|
200
|
+
/** Warm prefix size the incumbent carries (tokens) — from the derived ledger. */
|
|
201
|
+
readonly incumbentPrefixTokens: number;
|
|
202
|
+
/** Expected remaining calls over which the decision amortizes. */
|
|
203
|
+
readonly remainingCalls: number;
|
|
204
|
+
/** Trailing-window spend on the route, USD — base of the relative bar. */
|
|
205
|
+
readonly windowSpendUsd: number;
|
|
206
|
+
/** True when the window carried ANY cache observation (read or write). When
|
|
207
|
+
* false the verdict is `no_cache_signal` regardless of the numbers. */
|
|
208
|
+
readonly hasCacheSignal: boolean;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Assemble the `cache_context` evidence block + verdict for a proposal (spec
|
|
212
|
+
* §5.5), pure. The verdict compares the incumbent's cache-ADJUSTED effective
|
|
213
|
+
* input rate against the challenger's COLD list rate, amortized over the switch
|
|
214
|
+
* cost, and gates on the hysteresis config. The advantage is measured PER
|
|
215
|
+
* CACHED PREFIX (the tokens the decision actually turns on), so a per-call
|
|
216
|
+
* dollar figure is directly comparable to the floor.
|
|
217
|
+
*/
|
|
218
|
+
export declare function buildCacheContext(input: CacheContextInput, config?: CacheHysteresisConfig): CacheContext;
|
|
219
|
+
/**
|
|
220
|
+
* The [Briefing §30] rendered explanation — one prose line a human reads on the
|
|
221
|
+
* console and the `swap execute` approval prompt. Single-sourced here so the
|
|
222
|
+
* console, the CLI, and the OTLP span never phrase the same decision three ways.
|
|
223
|
+
*/
|
|
224
|
+
export declare function explainCacheContext(ctx: CacheContext, incumbentLabel: string, challengerLabel: string): string;
|
|
66
225
|
//# sourceMappingURL=evaluate.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/watch/evaluate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;
|
|
1
|
+
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/watch/evaluate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAYH,OAAO,KAAK,EACV,qBAAqB,EACrB,YAAY,EACZ,cAAc,EACd,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AAEpC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,SAAS,kBAAkB,EAAE,CAAC;CACpD;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,MAAM,UAAU,GAClB;IACE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IACzB,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;CAC1B,GACD;IACE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAAC;CACrC,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,8DAA8D;IAC9D,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;qFAIiF;IACjF,QAAQ,CAAC,eAAe,CAAC,EAAE,qBAAqB,CAAC;CAClD;AAED;;;;;;uFAMuF;AACvF,MAAM,MAAM,eAAe,GACvB,gBAAgB,GAChB,yBAAyB,GACzB,qBAAqB,GACrB,yBAAyB,GACzB,qBAAqB,CAAC;AAQ1B,8EAA8E;AAC9E,MAAM,WAAW,mBAAmB;IAClC,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,qEAAqE;IACrE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,6EAA6E;IAC7E,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9C;AAED;;4EAE4E;AAC5E,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,mBAAmB,CAAC;IACpC,mEAAmE;IACnE,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5C;AAED;;sFAEsF;AACtF,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,SAAS,uBAAuB,EAAE,CAAC;CACzD;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;kBAGkB;AAClB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,yEAAyE;IACzE,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAC;IACxC,sEAAsE;IACtE,QAAQ,CAAC,UAAU,EAAE,SAAS,kBAAkB,EAAE,CAAC;CACpD;AAED,8EAA8E;AAC9E,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,cAAc,GAAG,WAAW,CAAC;IAC5C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,uDAAuD;IACvD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC;IAC/B,QAAQ,CAAC,OAAO,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;CACtC;AAED;kFACkF;AAClF,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;iFAEiF;AACjF,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,IAAI,EAAE,qBAAqB,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,sEAAsE;IACtE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;0CAE0C;AAC1C,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,cAAc,CAAC;AAE3E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;uDAGmD;IACnD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAYpC,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAE3E;AAMD,wBAAgB,cAAc,CAC5B,MAAM,EAAE,SAAS,cAAc,EAAE,EACjC,MAAM,EAAE,SAAS,UAAU,EAAE,EAC7B,MAAM,EAAE,WAAW,GAClB,aAAa,EAAE,CAoDjB;AAUD;;oDAEoD;AACpD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,+BAA+B,EAAE,MAAM,CAAC;IACjD,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,0BAA0B,EAAE,MAAM,CAAC;IAC5C,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;CAChC;AAED;;;;kCAIkC;AAClC,MAAM,WAAW,iBAAiB;IAChC,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,6CAA6C;IAC7C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,8CAA8C;IAC9C,QAAQ,CAAC,eAAe,EAAE,cAAc,CAAC;IACzC,iFAAiF;IACjF,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,kEAAkE;IAClE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,0EAA0E;IAC1E,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;2EACuE;IACvE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAID;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,iBAAiB,EACxB,MAAM,GAAE,qBAAgD,GACvD,YAAY,CA8Cd;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,YAAY,EACjB,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,GACtB,MAAM,CAaR"}
|
package/dist/watch/evaluate.js
CHANGED
|
@@ -19,7 +19,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
19
19
|
exports.PROPOSAL_TTL_DAYS = void 0;
|
|
20
20
|
exports.triggerHash = triggerHash;
|
|
21
21
|
exports.evaluateEvents = evaluateEvents;
|
|
22
|
+
exports.buildCacheContext = buildCacheContext;
|
|
23
|
+
exports.explainCacheContext = explainCacheContext;
|
|
22
24
|
const node_crypto_1 = require("node:crypto");
|
|
25
|
+
const effective_rate_1 = require("../registry/effective-rate");
|
|
23
26
|
exports.PROPOSAL_TTL_DAYS = 30;
|
|
24
27
|
/** Reference token mix for blending input/output rates into one comparable
|
|
25
28
|
* number: 3 input tokens per output token — a conservative, route-agnostic
|
|
@@ -92,3 +95,61 @@ function evaluateEvents(routes, events, config) {
|
|
|
92
95
|
}
|
|
93
96
|
return drafts;
|
|
94
97
|
}
|
|
98
|
+
const PER_MTOK = 1e6;
|
|
99
|
+
/**
|
|
100
|
+
* Assemble the `cache_context` evidence block + verdict for a proposal (spec
|
|
101
|
+
* §5.5), pure. The verdict compares the incumbent's cache-ADJUSTED effective
|
|
102
|
+
* input rate against the challenger's COLD list rate, amortized over the switch
|
|
103
|
+
* cost, and gates on the hysteresis config. The advantage is measured PER
|
|
104
|
+
* CACHED PREFIX (the tokens the decision actually turns on), so a per-call
|
|
105
|
+
* dollar figure is directly comparable to the floor.
|
|
106
|
+
*/
|
|
107
|
+
function buildCacheContext(input, config = effective_rate_1.DEFAULT_CACHE_HYSTERESIS) {
|
|
108
|
+
const incumbentEff = (0, effective_rate_1.effectiveInputRate)(input.observed, input.incumbentRates);
|
|
109
|
+
const challengerCold = input.challengerRates.input;
|
|
110
|
+
// Switch cost of leaving the warm incumbent for the cold challenger, and the
|
|
111
|
+
// per-call saving the incumbent's warmth is worth (spec §4.2–4.3).
|
|
112
|
+
const switch_cost_usd = (0, effective_rate_1.switchCostUsd)(input.incumbentPrefixTokens, input.challengerRates, input.incumbentRates);
|
|
113
|
+
const perCallSaving = (0, effective_rate_1.perCallCacheSavingsUsd)(input.incumbentPrefixTokens, input.observed.h, input.incumbentRates);
|
|
114
|
+
const cache_option_value_usd = (0, effective_rate_1.cacheOptionValueUsd)(input.remainingCalls, perCallSaving);
|
|
115
|
+
const breakeven_conformant_calls = (0, effective_rate_1.breakevenCalls)(switch_cost_usd, perCallSaving);
|
|
116
|
+
// The cache-adjusted per-call advantage of the challenger over the incumbent,
|
|
117
|
+
// on the cached prefix: (incumbent effective − challenger cold) per prefix
|
|
118
|
+
// token, times the prefix. Positive ⇒ the cold challenger is still cheaper.
|
|
119
|
+
const perCallAdvantageUsd = (incumbentEff - challengerCold) * input.incumbentPrefixTokens;
|
|
120
|
+
const totalAdvantageUsd = perCallAdvantageUsd * Math.max(input.remainingCalls, 0);
|
|
121
|
+
const verdict = (0, effective_rate_1.classifyCacheVerdict)({
|
|
122
|
+
hasCacheSignal: input.hasCacheSignal,
|
|
123
|
+
perCallAdvantageUsd,
|
|
124
|
+
totalAdvantageUsd,
|
|
125
|
+
windowSpendUsd: input.windowSpendUsd,
|
|
126
|
+
}, config);
|
|
127
|
+
return {
|
|
128
|
+
observed_hit_rate: input.observed.h,
|
|
129
|
+
observed_write_rate: input.observed.w,
|
|
130
|
+
window_days: input.windowDays,
|
|
131
|
+
incumbent_effective_in_per_mtok: incumbentEff * PER_MTOK,
|
|
132
|
+
challenger_cold_in_per_mtok: challengerCold * PER_MTOK,
|
|
133
|
+
switch_cost_usd,
|
|
134
|
+
cache_option_value_usd,
|
|
135
|
+
breakeven_conformant_calls,
|
|
136
|
+
verdict,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* The [Briefing §30] rendered explanation — one prose line a human reads on the
|
|
141
|
+
* console and the `swap execute` approval prompt. Single-sourced here so the
|
|
142
|
+
* console, the CLI, and the OTLP span never phrase the same decision three ways.
|
|
143
|
+
*/
|
|
144
|
+
function explainCacheContext(ctx, incumbentLabel, challengerLabel) {
|
|
145
|
+
const warmMtok = ctx.incumbent_effective_in_per_mtok.toFixed(2);
|
|
146
|
+
const coldMtok = ctx.challenger_cold_in_per_mtok.toFixed(2);
|
|
147
|
+
const hit = `${(ctx.observed_hit_rate * 100).toFixed(0)}%`;
|
|
148
|
+
const breakeven = Number.isFinite(ctx.breakeven_conformant_calls)
|
|
149
|
+
? `breaks even after ${Math.ceil(ctx.breakeven_conformant_calls)} calls`
|
|
150
|
+
: "never breaks even";
|
|
151
|
+
return (`${incumbentLabel} is warm (${hit} hit rate, effective $${warmMtok}/Mtok in); ` +
|
|
152
|
+
`${challengerLabel} is nominally cheaper but cold ($${coldMtok}/Mtok in). ` +
|
|
153
|
+
`Switching costs $${ctx.switch_cost_usd.toFixed(4)} and ${breakeven}. ` +
|
|
154
|
+
`Verdict: ${ctx.verdict}.`);
|
|
155
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Monitor v1 — conform-rate degradation → bake-off proposal (Expansion E4 chunk B).
|
|
3
|
+
*
|
|
4
|
+
* The watch loop reacts to PRICE moves; the Monitor reacts to QUALITY moves. It
|
|
5
|
+
* rolls up the deterministic-eval conform rate per (org, task) into a daily
|
|
6
|
+
* series, and when a recent window falls below the trailing baseline by a ruled
|
|
7
|
+
* margin — with enough evaluations on BOTH sides to trust the number — it emits
|
|
8
|
+
* ONE `proposals` row (kind='bakeoff') asking for a re-evaluation. It reuses the
|
|
9
|
+
* ENTIRE existing proposals machinery: same table, same kind, same
|
|
10
|
+
* (route, trigger_hash) idempotency, same human-approval gate. No new proposal
|
|
11
|
+
* kind, no auto-actuation — the swap gate is untouched (spec §3.2).
|
|
12
|
+
*
|
|
13
|
+
* ── MEASURED-ONLY DOCTRINE ────────────────────────────────────────────────
|
|
14
|
+
* A degradation is proposed only from DETERMINISTIC evaluations (evaluator_kind
|
|
15
|
+
* = 'deterministic' — no model-judge softness), and only when BOTH windows clear
|
|
16
|
+
* `MONITOR_MIN_N`. Below that the rate is too noisy to act on: two failures can
|
|
17
|
+
* swing a small sample past the margin, and the house rule is to refuse rather
|
|
18
|
+
* than propose on thin data.
|
|
19
|
+
*/
|
|
20
|
+
import type { ProposalDraft } from "./evaluate";
|
|
21
|
+
/** The recent window whose conformance we judge. A week smooths daily noise
|
|
22
|
+
* while staying responsive to a regression that just landed. */
|
|
23
|
+
export declare const MONITOR_WINDOW_DAYS = 7;
|
|
24
|
+
/** The trailing baseline the window is compared against — the 28 days
|
|
25
|
+
* immediately BEFORE the window (non-overlapping). Four weeks is a stable
|
|
26
|
+
* reference that predates a week-old regression; it matches the proposal TTL
|
|
27
|
+
* cadence, so a baseline never silently includes an unresolved degradation. */
|
|
28
|
+
export declare const MONITOR_BASELINE_DAYS = 28;
|
|
29
|
+
/** The drop, in percentage POINTS of conform rate, that flags a degradation.
|
|
30
|
+
* Mirrors the watch loop's ±10% material-shift threshold — one house number for
|
|
31
|
+
* "materially worse" — so the two loops speak the same language. */
|
|
32
|
+
export declare const MONITOR_MARGIN_PTS = 10;
|
|
33
|
+
/** Minimum deterministic evaluations required in EACH window before the rate is
|
|
34
|
+
* trustworthy enough to propose on (measured-only). */
|
|
35
|
+
export declare const MONITOR_MIN_N = 20;
|
|
36
|
+
export interface MonitorConfig {
|
|
37
|
+
readonly windowDays: number;
|
|
38
|
+
readonly baselineDays: number;
|
|
39
|
+
readonly marginPts: number;
|
|
40
|
+
readonly minN: number;
|
|
41
|
+
}
|
|
42
|
+
export declare const DEFAULT_MONITOR_CONFIG: MonitorConfig;
|
|
43
|
+
/** One day's deterministic-eval counts for a task. `conformN` ≤ `n`. */
|
|
44
|
+
export interface ConformDay {
|
|
45
|
+
readonly date: string;
|
|
46
|
+
readonly n: number;
|
|
47
|
+
readonly conformN: number;
|
|
48
|
+
}
|
|
49
|
+
export interface TaskConformSeries {
|
|
50
|
+
readonly task: string;
|
|
51
|
+
readonly days: readonly ConformDay[];
|
|
52
|
+
}
|
|
53
|
+
/** A single deterministic evaluation row for the rollup (server read shape). */
|
|
54
|
+
export interface EvalRollupRow {
|
|
55
|
+
readonly artifactId: string;
|
|
56
|
+
readonly passed: boolean | null;
|
|
57
|
+
readonly createdAt: string;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Roll deterministic-eval rows up into per-(task) daily conform counts. The
|
|
61
|
+
* task comes from `taskByArtifact` (the caller joins artifacts→task locally,
|
|
62
|
+
* since artifact_evaluations carries only artifact_id). Rows whose artifact has
|
|
63
|
+
* no known task, or whose `passed` is null (not a pass/fail eval), are dropped.
|
|
64
|
+
*/
|
|
65
|
+
export declare function rollupConformRate(rows: readonly EvalRollupRow[], taskByArtifact: ReadonlyMap<string, string>): TaskConformSeries[];
|
|
66
|
+
/**
|
|
67
|
+
* The degradation rule (measured-only). For each task, compare the recent
|
|
68
|
+
* window to the trailing baseline; emit a bake-off draft when the drop clears
|
|
69
|
+
* the margin AND both windows clear min-n. Deterministic over `now` (passed in,
|
|
70
|
+
* never read from the clock) so the fixture test is reproducible. The
|
|
71
|
+
* trigger_hash keys on the task + both window bounds, so re-running the SAME
|
|
72
|
+
* day is a no-op (the (route, trigger_hash) unique index dedupes it).
|
|
73
|
+
*/
|
|
74
|
+
export declare function evaluateConformDegradation(series: readonly TaskConformSeries[], now: Date, config?: MonitorConfig): ProposalDraft[];
|
|
75
|
+
/**
|
|
76
|
+
* The reads a Monitor cycle needs, injected so the orchestrator is testable
|
|
77
|
+
* without a database. BOTH must be scoped to ONE org (the rollup groups by task
|
|
78
|
+
* within that scope — cross-org tasks sharing a name must never merge): a
|
|
79
|
+
* hosted cron iterates orgs, a self-host has exactly one.
|
|
80
|
+
*/
|
|
81
|
+
export interface MonitorReads {
|
|
82
|
+
/** Deterministic artifact evaluations created at/after `sinceIso`. */
|
|
83
|
+
readonly recentDeterministicEvals: (sinceIso: string) => Promise<readonly EvalRollupRow[]>;
|
|
84
|
+
/** task for each of the given artifact ids (the rollup join — artifact_evaluations
|
|
85
|
+
* carries only artifact_id). Only the ids that actually have evals are asked
|
|
86
|
+
* for, so the read stays bounded regardless of total artifact count. */
|
|
87
|
+
readonly taskFor: (artifactIds: readonly string[]) => Promise<ReadonlyMap<string, string>>;
|
|
88
|
+
}
|
|
89
|
+
export interface MonitorCycleResult {
|
|
90
|
+
readonly series: TaskConformSeries[];
|
|
91
|
+
readonly drafts: ProposalDraft[];
|
|
92
|
+
/** How many proposals the control plane actually created (0 when duplicates). */
|
|
93
|
+
readonly inserted: number;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* One Monitor cycle: read the recent deterministic evals + task map, roll up,
|
|
97
|
+
* apply the degradation rule, and submit any drafts through the EXISTING
|
|
98
|
+
* proposals client (idempotent on (route, trigger_hash)). Returns the series +
|
|
99
|
+
* drafts + inserted count so a caller/CLI can report without a second query.
|
|
100
|
+
* Never actuates — submitting a proposal is the whole of its authority (§3.2).
|
|
101
|
+
*/
|
|
102
|
+
export declare function runMonitorCycle(reads: MonitorReads, submitDrafts: (drafts: readonly ProposalDraft[]) => Promise<number>, now: Date, config?: MonitorConfig): Promise<MonitorCycleResult>;
|
|
103
|
+
export interface MonitorReadsConfig {
|
|
104
|
+
readonly supabaseUrl: string;
|
|
105
|
+
readonly serviceRoleKey: string;
|
|
106
|
+
/** The org this cycle runs for — reads are org-scoped so tasks never merge
|
|
107
|
+
* across tenants. A hosted cron iterates orgs; a self-host has exactly one. */
|
|
108
|
+
readonly orgId: string;
|
|
109
|
+
readonly fetchFn?: typeof fetch;
|
|
110
|
+
}
|
|
111
|
+
/** PostgREST reads for a Monitor cycle (service role; mirrors the proposals
|
|
112
|
+
* client). Two org-scoped queries, joined in-SDK by `runMonitorCycle` — no DDL,
|
|
113
|
+
* no view (spec §3.1). `taskFor` batches ids ≤100 (T9 — PostgREST URLs fail
|
|
114
|
+
* silently around 8KB). */
|
|
115
|
+
export declare function createMonitorReads(config: MonitorReadsConfig): MonitorReads;
|
|
116
|
+
//# sourceMappingURL=monitor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"monitor.d.ts","sourceRoot":"","sources":["../../src/watch/monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAA4C,aAAa,EAAE,MAAM,YAAY,CAAC;AAG1F;gEACgE;AAChE,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC;;;+EAG+E;AAC/E,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC;;oEAEoE;AACpE,eAAO,MAAM,kBAAkB,KAAK,CAAC;AACrC;uDACuD;AACvD,eAAO,MAAM,aAAa,KAAK,CAAC;AAEhC,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,sBAAsB,EAAE,aAKpC,CAAC;AAEF,wEAAwE;AACxE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC;AAED,gFAAgF;AAChF,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAYD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,SAAS,aAAa,EAAE,EAC9B,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAC1C,iBAAiB,EAAE,CA0BrB;AAeD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,SAAS,iBAAiB,EAAE,EACpC,GAAG,EAAE,IAAI,EACT,MAAM,GAAE,aAAsC,GAC7C,aAAa,EAAE,CAgCjB;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,wBAAwB,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,aAAa,EAAE,CAAC,CAAC;IAC3F;;4EAEwE;IACxE,QAAQ,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC5F;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IACjC,iFAAiF;IACjF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,EACnE,GAAG,EAAE,IAAI,EACT,MAAM,GAAE,aAAsC,GAC7C,OAAO,CAAC,kBAAkB,CAAC,CAU7B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;mFAC+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAYD;;;2BAG2B;AAC3B,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,YAAY,CA6B3E"}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Monitor v1 — conform-rate degradation → bake-off proposal (Expansion E4 chunk B).
|
|
4
|
+
*
|
|
5
|
+
* The watch loop reacts to PRICE moves; the Monitor reacts to QUALITY moves. It
|
|
6
|
+
* rolls up the deterministic-eval conform rate per (org, task) into a daily
|
|
7
|
+
* series, and when a recent window falls below the trailing baseline by a ruled
|
|
8
|
+
* margin — with enough evaluations on BOTH sides to trust the number — it emits
|
|
9
|
+
* ONE `proposals` row (kind='bakeoff') asking for a re-evaluation. It reuses the
|
|
10
|
+
* ENTIRE existing proposals machinery: same table, same kind, same
|
|
11
|
+
* (route, trigger_hash) idempotency, same human-approval gate. No new proposal
|
|
12
|
+
* kind, no auto-actuation — the swap gate is untouched (spec §3.2).
|
|
13
|
+
*
|
|
14
|
+
* ── MEASURED-ONLY DOCTRINE ────────────────────────────────────────────────
|
|
15
|
+
* A degradation is proposed only from DETERMINISTIC evaluations (evaluator_kind
|
|
16
|
+
* = 'deterministic' — no model-judge softness), and only when BOTH windows clear
|
|
17
|
+
* `MONITOR_MIN_N`. Below that the rate is too noisy to act on: two failures can
|
|
18
|
+
* swing a small sample past the margin, and the house rule is to refuse rather
|
|
19
|
+
* than propose on thin data.
|
|
20
|
+
*/
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.DEFAULT_MONITOR_CONFIG = exports.MONITOR_MIN_N = exports.MONITOR_MARGIN_PTS = exports.MONITOR_BASELINE_DAYS = exports.MONITOR_WINDOW_DAYS = void 0;
|
|
23
|
+
exports.rollupConformRate = rollupConformRate;
|
|
24
|
+
exports.evaluateConformDegradation = evaluateConformDegradation;
|
|
25
|
+
exports.runMonitorCycle = runMonitorCycle;
|
|
26
|
+
exports.createMonitorReads = createMonitorReads;
|
|
27
|
+
const evaluate_1 = require("./evaluate");
|
|
28
|
+
/** The recent window whose conformance we judge. A week smooths daily noise
|
|
29
|
+
* while staying responsive to a regression that just landed. */
|
|
30
|
+
exports.MONITOR_WINDOW_DAYS = 7;
|
|
31
|
+
/** The trailing baseline the window is compared against — the 28 days
|
|
32
|
+
* immediately BEFORE the window (non-overlapping). Four weeks is a stable
|
|
33
|
+
* reference that predates a week-old regression; it matches the proposal TTL
|
|
34
|
+
* cadence, so a baseline never silently includes an unresolved degradation. */
|
|
35
|
+
exports.MONITOR_BASELINE_DAYS = 28;
|
|
36
|
+
/** The drop, in percentage POINTS of conform rate, that flags a degradation.
|
|
37
|
+
* Mirrors the watch loop's ±10% material-shift threshold — one house number for
|
|
38
|
+
* "materially worse" — so the two loops speak the same language. */
|
|
39
|
+
exports.MONITOR_MARGIN_PTS = 10;
|
|
40
|
+
/** Minimum deterministic evaluations required in EACH window before the rate is
|
|
41
|
+
* trustworthy enough to propose on (measured-only). */
|
|
42
|
+
exports.MONITOR_MIN_N = 20;
|
|
43
|
+
exports.DEFAULT_MONITOR_CONFIG = {
|
|
44
|
+
windowDays: exports.MONITOR_WINDOW_DAYS,
|
|
45
|
+
baselineDays: exports.MONITOR_BASELINE_DAYS,
|
|
46
|
+
marginPts: exports.MONITOR_MARGIN_PTS,
|
|
47
|
+
minN: exports.MONITOR_MIN_N,
|
|
48
|
+
};
|
|
49
|
+
/** ISO day (YYYY-MM-DD) of a timestamp. */
|
|
50
|
+
function dayOf(iso) {
|
|
51
|
+
return iso.slice(0, 10);
|
|
52
|
+
}
|
|
53
|
+
/** N days before `d`, as a YYYY-MM-DD day string. */
|
|
54
|
+
function dayMinus(d, days) {
|
|
55
|
+
return new Date(d.getTime() - days * 86400000).toISOString().slice(0, 10);
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Roll deterministic-eval rows up into per-(task) daily conform counts. The
|
|
59
|
+
* task comes from `taskByArtifact` (the caller joins artifacts→task locally,
|
|
60
|
+
* since artifact_evaluations carries only artifact_id). Rows whose artifact has
|
|
61
|
+
* no known task, or whose `passed` is null (not a pass/fail eval), are dropped.
|
|
62
|
+
*/
|
|
63
|
+
function rollupConformRate(rows, taskByArtifact) {
|
|
64
|
+
// task → day → {n, conformN}
|
|
65
|
+
const byTask = new Map();
|
|
66
|
+
for (const r of rows) {
|
|
67
|
+
if (r.passed === null)
|
|
68
|
+
continue;
|
|
69
|
+
const task = taskByArtifact.get(r.artifactId);
|
|
70
|
+
if (task === undefined)
|
|
71
|
+
continue;
|
|
72
|
+
const day = dayOf(r.createdAt);
|
|
73
|
+
let days = byTask.get(task);
|
|
74
|
+
if (days === undefined) {
|
|
75
|
+
days = new Map();
|
|
76
|
+
byTask.set(task, days);
|
|
77
|
+
}
|
|
78
|
+
const cell = days.get(day) ?? { n: 0, conformN: 0 };
|
|
79
|
+
cell.n += 1;
|
|
80
|
+
if (r.passed)
|
|
81
|
+
cell.conformN += 1;
|
|
82
|
+
days.set(day, cell);
|
|
83
|
+
}
|
|
84
|
+
return [...byTask.entries()]
|
|
85
|
+
.map(([task, days]) => ({
|
|
86
|
+
task,
|
|
87
|
+
days: [...days.entries()]
|
|
88
|
+
.map(([date, c]) => ({ date, n: c.n, conformN: c.conformN }))
|
|
89
|
+
.sort((a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0)),
|
|
90
|
+
}))
|
|
91
|
+
.sort((a, b) => (a.task < b.task ? -1 : a.task > b.task ? 1 : 0));
|
|
92
|
+
}
|
|
93
|
+
/** Aggregate the days whose date is in [from, to) into one window. */
|
|
94
|
+
function windowOf(days, from, to) {
|
|
95
|
+
let n = 0;
|
|
96
|
+
let conformN = 0;
|
|
97
|
+
for (const d of days) {
|
|
98
|
+
if (d.date >= from && d.date < to) {
|
|
99
|
+
n += d.n;
|
|
100
|
+
conformN += d.conformN;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return { from, to, n, conformRate: n > 0 ? conformN / n : 0 };
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The degradation rule (measured-only). For each task, compare the recent
|
|
107
|
+
* window to the trailing baseline; emit a bake-off draft when the drop clears
|
|
108
|
+
* the margin AND both windows clear min-n. Deterministic over `now` (passed in,
|
|
109
|
+
* never read from the clock) so the fixture test is reproducible. The
|
|
110
|
+
* trigger_hash keys on the task + both window bounds, so re-running the SAME
|
|
111
|
+
* day is a no-op (the (route, trigger_hash) unique index dedupes it).
|
|
112
|
+
*/
|
|
113
|
+
function evaluateConformDegradation(series, now, config = exports.DEFAULT_MONITOR_CONFIG) {
|
|
114
|
+
const windowFrom = dayMinus(now, config.windowDays);
|
|
115
|
+
const windowTo = dayMinus(now, 0);
|
|
116
|
+
const baselineFrom = dayMinus(now, config.windowDays + config.baselineDays);
|
|
117
|
+
const baselineTo = windowFrom;
|
|
118
|
+
const expiresAt = new Date(now.getTime() + evaluate_1.PROPOSAL_TTL_DAYS * 86400000).toISOString();
|
|
119
|
+
const drafts = [];
|
|
120
|
+
for (const s of series) {
|
|
121
|
+
const window = windowOf(s.days, windowFrom, windowTo);
|
|
122
|
+
const baseline = windowOf(s.days, baselineFrom, baselineTo);
|
|
123
|
+
if (window.n < config.minN || baseline.n < config.minN)
|
|
124
|
+
continue; // thin data → refuse
|
|
125
|
+
const dropPts = (baseline.conformRate - window.conformRate) * 100;
|
|
126
|
+
if (dropPts < config.marginPts)
|
|
127
|
+
continue; // not materially worse
|
|
128
|
+
const trigger = {
|
|
129
|
+
type: "conform_degradation",
|
|
130
|
+
task: s.task,
|
|
131
|
+
window,
|
|
132
|
+
baseline,
|
|
133
|
+
dropPts,
|
|
134
|
+
marginPts: config.marginPts,
|
|
135
|
+
minN: config.minN,
|
|
136
|
+
};
|
|
137
|
+
drafts.push({
|
|
138
|
+
route: s.task,
|
|
139
|
+
kind: "bakeoff",
|
|
140
|
+
trigger,
|
|
141
|
+
triggerHash: (0, evaluate_1.triggerHash)(s.task, trigger),
|
|
142
|
+
expiresAt,
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
return drafts;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* One Monitor cycle: read the recent deterministic evals + task map, roll up,
|
|
149
|
+
* apply the degradation rule, and submit any drafts through the EXISTING
|
|
150
|
+
* proposals client (idempotent on (route, trigger_hash)). Returns the series +
|
|
151
|
+
* drafts + inserted count so a caller/CLI can report without a second query.
|
|
152
|
+
* Never actuates — submitting a proposal is the whole of its authority (§3.2).
|
|
153
|
+
*/
|
|
154
|
+
async function runMonitorCycle(reads, submitDrafts, now, config = exports.DEFAULT_MONITOR_CONFIG) {
|
|
155
|
+
const sinceMs = now.getTime() - (config.windowDays + config.baselineDays) * 86400000;
|
|
156
|
+
const sinceIso = new Date(sinceMs).toISOString();
|
|
157
|
+
const rows = await reads.recentDeterministicEvals(sinceIso);
|
|
158
|
+
const ids = [...new Set(rows.map((r) => r.artifactId))];
|
|
159
|
+
const tasks = ids.length > 0 ? await reads.taskFor(ids) : new Map();
|
|
160
|
+
const series = rollupConformRate(rows, tasks);
|
|
161
|
+
const drafts = evaluateConformDegradation(series, now, config);
|
|
162
|
+
const inserted = drafts.length > 0 ? await submitDrafts(drafts) : 0;
|
|
163
|
+
return { series, drafts, inserted };
|
|
164
|
+
}
|
|
165
|
+
/** PostgREST reads for a Monitor cycle (service role; mirrors the proposals
|
|
166
|
+
* client). Two org-scoped queries, joined in-SDK by `runMonitorCycle` — no DDL,
|
|
167
|
+
* no view (spec §3.1). `taskFor` batches ids ≤100 (T9 — PostgREST URLs fail
|
|
168
|
+
* silently around 8KB). */
|
|
169
|
+
function createMonitorReads(config) {
|
|
170
|
+
const base = config.supabaseUrl.replace(/\/$/, "");
|
|
171
|
+
const fetchFn = config.fetchFn ?? fetch;
|
|
172
|
+
const org = encodeURIComponent(config.orgId);
|
|
173
|
+
async function get(path) {
|
|
174
|
+
const res = await fetchFn(`${base}/rest/v1/${path}`, {
|
|
175
|
+
headers: { apikey: config.serviceRoleKey, authorization: `Bearer ${config.serviceRoleKey}` },
|
|
176
|
+
});
|
|
177
|
+
if (!res.ok)
|
|
178
|
+
throw new Error(`monitor read ${path} → ${res.status} ${await res.text().catch(() => "")}`);
|
|
179
|
+
return (await res.json().catch(() => []));
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
async recentDeterministicEvals(sinceIso) {
|
|
183
|
+
const rows = await get(`artifact_evaluations?select=artifact_id,passed,created_at&org_id=eq.${org}` +
|
|
184
|
+
`&evaluator_kind=eq.deterministic&created_at=gte.${encodeURIComponent(sinceIso)}`);
|
|
185
|
+
return rows.map((r) => ({ artifactId: r.artifact_id, passed: r.passed, createdAt: r.created_at }));
|
|
186
|
+
},
|
|
187
|
+
async taskFor(artifactIds) {
|
|
188
|
+
const out = new Map();
|
|
189
|
+
for (let i = 0; i < artifactIds.length; i += 100) {
|
|
190
|
+
const batch = artifactIds.slice(i, i + 100).map(encodeURIComponent).join(",");
|
|
191
|
+
const rows = await get(`artifacts?select=id,task&org_id=eq.${org}&id=in.(${batch})`);
|
|
192
|
+
for (const r of rows)
|
|
193
|
+
if (r.task !== null)
|
|
194
|
+
out.set(r.id, r.task);
|
|
195
|
+
}
|
|
196
|
+
return out;
|
|
197
|
+
},
|
|
198
|
+
};
|
|
199
|
+
}
|
|
@@ -10,13 +10,13 @@
|
|
|
10
10
|
* Transport is PostgREST with the service-role key — proposals are
|
|
11
11
|
* control-plane-only state (cross-run truth, not per-inference telemetry).
|
|
12
12
|
*/
|
|
13
|
-
import type { ProposalDraft, ProposalTrigger } from "./evaluate";
|
|
13
|
+
import type { ProposalDraft, ProposalKind, ProposalTrigger } from "./evaluate";
|
|
14
14
|
export type ProposalStatus = "suggested" | "evidenced" | "approved" | "rejected" | "executed" | "expired";
|
|
15
15
|
export declare function assertTransition(from: ProposalStatus, to: ProposalStatus): void;
|
|
16
16
|
export interface ProposalRecord {
|
|
17
17
|
readonly id: string;
|
|
18
18
|
readonly route: string;
|
|
19
|
-
readonly kind:
|
|
19
|
+
readonly kind: ProposalKind;
|
|
20
20
|
readonly trigger: ProposalTrigger;
|
|
21
21
|
readonly triggerHash: string;
|
|
22
22
|
readonly evidence: Record<string, unknown> | null;
|
|
@@ -32,6 +32,18 @@ export interface ProposalRecord {
|
|
|
32
32
|
export interface ProposalsClientConfig {
|
|
33
33
|
readonly supabaseUrl: string;
|
|
34
34
|
readonly serviceRoleKey: string;
|
|
35
|
+
/**
|
|
36
|
+
* The tenant this client writes for. When set (the multi-tenant hosted cron —
|
|
37
|
+
* managed-loop §2), `submitDrafts` STAMPS every row with this org and dedupes
|
|
38
|
+
* against the real `(org_id, route, trigger_hash)` unique index, and
|
|
39
|
+
* `expireOverdue` sweeps only THIS org's rows. When omitted (the CLI /
|
|
40
|
+
* self-host lane) behaviour is unchanged: `org_id` falls to its house-org
|
|
41
|
+
* column default and one deployment sweeps its own single tenant. Omitting it
|
|
42
|
+
* on a hosted cron would file every org's proposals under the house org — the
|
|
43
|
+
* whole reason this field exists. Reads keyed by id (`get`/`orgOf`/`decide`)
|
|
44
|
+
* are org-agnostic and unaffected; `list` already takes its org as a param.
|
|
45
|
+
*/
|
|
46
|
+
readonly orgId?: string;
|
|
35
47
|
readonly fetchFn?: typeof fetch;
|
|
36
48
|
}
|
|
37
49
|
export interface ProposalsClient {
|
|
@@ -50,8 +62,12 @@ export interface ProposalsClient {
|
|
|
50
62
|
*/
|
|
51
63
|
list(orgId: string | null, status?: ProposalStatus): Promise<ProposalRecord[]>;
|
|
52
64
|
get(id: string): Promise<ProposalRecord | null>;
|
|
53
|
-
/** Bake-off evidence + the suggested→evidenced flip, one PATCH.
|
|
54
|
-
|
|
65
|
+
/** Bake-off evidence + the suggested→evidenced flip, one status-scoped PATCH.
|
|
66
|
+
* Only a proposal still OPEN to evidence (suggested, or evidenced being
|
|
67
|
+
* re-run) matches — a terminal proposal (approved/executed/rejected/expired)
|
|
68
|
+
* matches zero rows, so evidence can never resurrect it. Returns false when
|
|
69
|
+
* nothing matched (already moved on), like `decide`. */
|
|
70
|
+
attachEvidence(id: string, evidence: Record<string, unknown>): Promise<boolean>;
|
|
55
71
|
/** Validated status change (assertTransition is the caller's guard for
|
|
56
72
|
* reads-before-writes; the PATCH is still status-scoped server-side). */
|
|
57
73
|
setStatus(id: string, from: ProposalStatus, to: ProposalStatus): Promise<void>;
|