modelrig 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -0
- package/README.md +53 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts +10 -0
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +78 -7
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +106 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +48 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +167 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +130 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +363 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +147 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
package/dist/status.js
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Rig posture — the boot banner + `/status` report (settings-plane §2).
|
|
4
|
+
*
|
|
5
|
+
* WS1's lesson, made a rule: "a default nobody can observe gets asked about
|
|
6
|
+
* instead of answered." The artifact namespace now defaults ON when a control
|
|
7
|
+
* plane is configured (§2 flip), so a user must be able to SEE whether it is on
|
|
8
|
+
* and WHY, without reading source. This module derives that posture from the
|
|
9
|
+
* resolved config (plus, for `/status`, the settings resolver's snapshot) and
|
|
10
|
+
* shapes it for a one-line boot banner and a fuller `/status` report.
|
|
11
|
+
*
|
|
12
|
+
* Pure formatters (no env, no network) except `emitBootBanner`, which writes the
|
|
13
|
+
* one banner line to stderr — kept here so index.ts calls it in a single line.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.cachePosture = cachePosture;
|
|
17
|
+
exports.exportModeOf = exportModeOf;
|
|
18
|
+
exports.artifactsPosture = artifactsPosture;
|
|
19
|
+
exports.buildRigStatus = buildRigStatus;
|
|
20
|
+
exports.formatBootBanner = formatBootBanner;
|
|
21
|
+
exports.emitBootBanner = emitBootBanner;
|
|
22
|
+
exports.formatStatusReport = formatStatusReport;
|
|
23
|
+
const cache_policy_1 = require("./registry/cache-policy");
|
|
24
|
+
/** The cache posture from the policy registry alone (WS-A) — which providers
|
|
25
|
+
* ModelRig accounts caching for, and the policy version. No telemetry: live
|
|
26
|
+
* hit rates live behind the control-plane rollup (WS-B / the console). */
|
|
27
|
+
function cachePosture() {
|
|
28
|
+
const eligible = (0, cache_policy_1.seededCachePolicyProviders)().filter((p) => (0, cache_policy_1.getCachePolicy)(p).cache_supported);
|
|
29
|
+
const policyVersion = eligible.length > 0 ? (0, cache_policy_1.getCachePolicy)(eligible[0]).policy_version : "n/a";
|
|
30
|
+
return { policyVersion, eligibleProviders: eligible };
|
|
31
|
+
}
|
|
32
|
+
function exportModeOf(config) {
|
|
33
|
+
return config.export?.mode ?? "local-only";
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The artifact-namespace posture as a boolean + a reason, derivable from config
|
|
37
|
+
* ALONE (so both the banner and `/status` agree). The four cases the §2 flip
|
|
38
|
+
* creates, each named honestly:
|
|
39
|
+
* on + control plane → the default is now on (a sink exists to ship to)
|
|
40
|
+
* on + local-only → an explicit MODELRIG_ARTIFACTS opt-in
|
|
41
|
+
* off + control plane → an explicit MODELRIG_ARTIFACTS=0 opt-out
|
|
42
|
+
* off + local-only → nothing to ship to; off is the local-only default
|
|
43
|
+
*/
|
|
44
|
+
function artifactsPosture(config) {
|
|
45
|
+
const enabled = config.artifacts?.enabled === true;
|
|
46
|
+
const localOnly = exportModeOf(config) === "local-only";
|
|
47
|
+
if (enabled) {
|
|
48
|
+
return {
|
|
49
|
+
enabled,
|
|
50
|
+
reason: localOnly ? "MODELRIG_ARTIFACTS opt-in (local-only rig)" : "control plane configured",
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
enabled,
|
|
55
|
+
reason: localOnly ? "local-only rig — no control plane to ship to" : "MODELRIG_ARTIFACTS=0 opt-out",
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/** Build the full status snapshot from config + the resolver's per-task cache. */
|
|
59
|
+
function buildRigStatus(config, tasks = new Map()) {
|
|
60
|
+
return {
|
|
61
|
+
exportMode: exportModeOf(config),
|
|
62
|
+
artifacts: artifactsPosture(config),
|
|
63
|
+
contentCustody: config.artifacts?.content !== undefined,
|
|
64
|
+
otlp: config.otlp ? "configured" : "disabled",
|
|
65
|
+
cache: cachePosture(),
|
|
66
|
+
tasks: [...tasks].map(([task, s]) => ({ task, ...s })),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/** The one-line boot banner (§2). Concise on purpose — mode, artifacts (+why),
|
|
70
|
+
* content, otlp. */
|
|
71
|
+
function formatBootBanner(config) {
|
|
72
|
+
const a = artifactsPosture(config);
|
|
73
|
+
const content = config.artifacts?.content !== undefined ? "on" : "off";
|
|
74
|
+
const otlp = config.otlp ? "on" : "off";
|
|
75
|
+
return (`[modelrig] telemetry: ${exportModeOf(config)} · ` +
|
|
76
|
+
`artifacts: ${a.enabled ? "on" : "off"} (${a.reason}) · ` +
|
|
77
|
+
`content: ${content} · otlp: ${otlp}`);
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Emit the boot banner to stderr — ONCE at rig construction, and ONLY when a
|
|
81
|
+
* control plane is configured (the case where the §2 default flip is in effect
|
|
82
|
+
* and the posture is non-obvious). A pure local-only rig with artifacts off is
|
|
83
|
+
* the boring, expected default and stays silent (no banner noise in tests or
|
|
84
|
+
* embedders). Stderr, so it never pollutes a command's stdout.
|
|
85
|
+
*/
|
|
86
|
+
function emitBootBanner(config) {
|
|
87
|
+
if (config.export === undefined)
|
|
88
|
+
return;
|
|
89
|
+
console.warn(formatBootBanner(config));
|
|
90
|
+
}
|
|
91
|
+
/** Render a `/status` report as multi-line text for the CLI. */
|
|
92
|
+
function formatStatusReport(status) {
|
|
93
|
+
const lines = [
|
|
94
|
+
"modelrig status",
|
|
95
|
+
` telemetry export : ${status.exportMode}`,
|
|
96
|
+
` artifacts : ${status.artifacts.enabled ? "enabled" : "disabled"} (${status.artifacts.reason})`,
|
|
97
|
+
` content custody : ${status.contentCustody ? "enabled" : "disabled"}`,
|
|
98
|
+
` otlp span export : ${status.otlp}`,
|
|
99
|
+
` cache accounting : on (policy ${status.cache.policyVersion}) — cache-eligible: ${status.cache.eligibleProviders.length > 0 ? status.cache.eligibleProviders.join(", ") : "none"}`,
|
|
100
|
+
" (live hit rate + effective input rate: console route cache page)",
|
|
101
|
+
];
|
|
102
|
+
if (status.tasks.length === 0) {
|
|
103
|
+
lines.push(" task settings : (none pulled yet — settings load on the first run of each task)");
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
lines.push(" task settings (control plane):");
|
|
107
|
+
for (const t of status.tasks) {
|
|
108
|
+
lines.push(` ${t.task}: capture=${t.capture} artifacts=${t.artifacts_enabled ?? "inherit"} ` +
|
|
109
|
+
`content=${t.artifact_content} scrub=${t.scrub}`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return lines.join("\n");
|
|
113
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local artifact-metadata buffer — Expansion E2 chunk B (spec §2).
|
|
3
|
+
*
|
|
4
|
+
* A SEPARATE better-sqlite3 handle over the same telemetry file (the captures.ts
|
|
5
|
+
* pattern), kept out of `store.ts` for cohesion and the LOC cap (P1). UNLIKE
|
|
6
|
+
* captures, these rows ARE exported: artifact METADATA is a cloud surface by
|
|
7
|
+
* ruling (the 00030 counter-annotation). What is NEVER stored — here or anywhere
|
|
8
|
+
* — is artifact CONTENT: `contentRef` is the customer's opaque URI, `contentHash`
|
|
9
|
+
* a sha256 of bytes hashed-then-discarded. Rows are immutable facts: INSERT OR
|
|
10
|
+
* IGNORE, read past a rowid watermark exactly like inferences.
|
|
11
|
+
*/
|
|
12
|
+
import type { ArtifactEdgeRow, ArtifactEvaluationRow, ArtifactRow, EvalCaseRow } from "../types/telemetry";
|
|
13
|
+
export interface ArtifactStore {
|
|
14
|
+
recordArtifact(row: ArtifactRow): void;
|
|
15
|
+
recordArtifactEdge(row: ArtifactEdgeRow): void;
|
|
16
|
+
recordArtifactEvaluation(row: ArtifactEvaluationRow): void;
|
|
17
|
+
/** Record a promoted eval case (E4 chunk A) — rides the `:eval_cases` export
|
|
18
|
+
* stream to the control plane, exactly like the other artifact-graph rows. */
|
|
19
|
+
recordEvalCase(row: EvalCaseRow): void;
|
|
20
|
+
readArtifactsAfter(afterRowid: number, limit: number): Array<{
|
|
21
|
+
rowid: number;
|
|
22
|
+
row: ArtifactRow;
|
|
23
|
+
}>;
|
|
24
|
+
readEdgesAfter(afterRowid: number, limit: number): Array<{
|
|
25
|
+
rowid: number;
|
|
26
|
+
row: ArtifactEdgeRow;
|
|
27
|
+
}>;
|
|
28
|
+
readEvaluationsAfter(afterRowid: number, limit: number): Array<{
|
|
29
|
+
rowid: number;
|
|
30
|
+
row: ArtifactEvaluationRow;
|
|
31
|
+
}>;
|
|
32
|
+
readEvalCasesAfter(afterRowid: number, limit: number): Array<{
|
|
33
|
+
rowid: number;
|
|
34
|
+
row: EvalCaseRow;
|
|
35
|
+
}>;
|
|
36
|
+
/** All eval cases for a task, newest first (E4 chunk A, bake-off selection). */
|
|
37
|
+
readEvalCasesByTask(task: string): EvalCaseRow[];
|
|
38
|
+
/** One artifact row by id, or null (E4 chunk A rehydration — resolves a case's
|
|
39
|
+
* source artifact to its run/step for the input lookup). */
|
|
40
|
+
getArtifactById(id: string): ArtifactRow | null;
|
|
41
|
+
/** Run `fn` inside a single SQLite transaction and return its value (H8).
|
|
42
|
+
* A step's ~9 record* writes then commit with ONE fsync instead of nine.
|
|
43
|
+
* Nestable (better-sqlite3 uses SAVEPOINTs); throws propagate and roll back. */
|
|
44
|
+
batch<T>(fn: () => T): T;
|
|
45
|
+
close(): void;
|
|
46
|
+
}
|
|
47
|
+
export declare function openArtifactStore(dbPath: string): ArtifactStore;
|
|
48
|
+
//# sourceMappingURL=artifact-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artifact-store.d.ts","sourceRoot":"","sources":["../../src/telemetry/artifact-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,WAAW,EACX,WAAW,EACZ,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,aAAa;IAC5B,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC,kBAAkB,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC;IAC/C,wBAAwB,CAAC,GAAG,EAAE,qBAAqB,GAAG,IAAI,CAAC;IAC3D;kFAC8E;IAC9E,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,WAAW,CAAA;KAAE,CAAC,CAAC;IAClG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,eAAe,CAAA;KAAE,CAAC,CAAC;IAClG,oBAAoB,CAClB,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,GACZ,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,qBAAqB,CAAA;KAAE,CAAC,CAAC;IACxD,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,WAAW,CAAA;KAAE,CAAC,CAAC;IAClG,gFAAgF;IAChF,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IACjD;gEAC4D;IAC5D,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC;IAChD;;oFAEgF;IAChF,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACzB,KAAK,IAAI,IAAI,CAAC;CACf;AAwED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAwH/D"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Local artifact-metadata buffer — Expansion E2 chunk B (spec §2).
|
|
4
|
+
*
|
|
5
|
+
* A SEPARATE better-sqlite3 handle over the same telemetry file (the captures.ts
|
|
6
|
+
* pattern), kept out of `store.ts` for cohesion and the LOC cap (P1). UNLIKE
|
|
7
|
+
* captures, these rows ARE exported: artifact METADATA is a cloud surface by
|
|
8
|
+
* ruling (the 00030 counter-annotation). What is NEVER stored — here or anywhere
|
|
9
|
+
* — is artifact CONTENT: `contentRef` is the customer's opaque URI, `contentHash`
|
|
10
|
+
* a sha256 of bytes hashed-then-discarded. Rows are immutable facts: INSERT OR
|
|
11
|
+
* IGNORE, read past a rowid watermark exactly like inferences.
|
|
12
|
+
*/
|
|
13
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
14
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.openArtifactStore = openArtifactStore;
|
|
18
|
+
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
19
|
+
const node_fs_1 = require("node:fs");
|
|
20
|
+
const node_path_1 = require("node:path");
|
|
21
|
+
const ARTIFACTS_SQL = `
|
|
22
|
+
CREATE TABLE IF NOT EXISTS artifacts (
|
|
23
|
+
id TEXT PRIMARY KEY, run_id TEXT, step_key TEXT, attempt_no INTEGER, task TEXT,
|
|
24
|
+
name TEXT NOT NULL, type TEXT NOT NULL, version INTEGER NOT NULL,
|
|
25
|
+
supersedes_artifact_id TEXT, declared_schema TEXT, fingerprint_cell TEXT,
|
|
26
|
+
content_ref TEXT, content_hash TEXT, content_bytes INTEGER,
|
|
27
|
+
data_classification TEXT, expires_at TEXT, scrub TEXT
|
|
28
|
+
);
|
|
29
|
+
CREATE TABLE IF NOT EXISTS artifact_edges (
|
|
30
|
+
from_artifact_id TEXT NOT NULL, to_artifact_id TEXT NOT NULL, kind TEXT NOT NULL,
|
|
31
|
+
PRIMARY KEY (from_artifact_id, to_artifact_id, kind)
|
|
32
|
+
);
|
|
33
|
+
CREATE TABLE IF NOT EXISTS artifact_evaluations (
|
|
34
|
+
id TEXT PRIMARY KEY, artifact_id TEXT, run_id TEXT, evaluator TEXT NOT NULL,
|
|
35
|
+
evaluator_kind TEXT NOT NULL, score REAL, passed INTEGER, label TEXT,
|
|
36
|
+
explanation TEXT, created_by TEXT
|
|
37
|
+
);
|
|
38
|
+
CREATE TABLE IF NOT EXISTS eval_cases (
|
|
39
|
+
id TEXT PRIMARY KEY, task TEXT NOT NULL, source_artifact_id TEXT NOT NULL,
|
|
40
|
+
corrected_artifact_id TEXT, origin TEXT NOT NULL, note TEXT, created_by TEXT
|
|
41
|
+
);
|
|
42
|
+
`;
|
|
43
|
+
function toArtifact(r) {
|
|
44
|
+
return {
|
|
45
|
+
id: r.id, runId: r.run_id, stepKey: r.step_key, attemptNo: r.attempt_no, task: r.task,
|
|
46
|
+
name: r.name, type: r.type, version: r.version, supersedesArtifactId: r.supersedes_artifact_id,
|
|
47
|
+
declaredSchema: r.declared_schema === null ? null : JSON.parse(r.declared_schema),
|
|
48
|
+
fingerprintCell: r.fingerprint_cell, contentRef: r.content_ref, contentHash: r.content_hash,
|
|
49
|
+
contentBytes: r.content_bytes, dataClassification: r.data_classification, expiresAt: r.expires_at,
|
|
50
|
+
scrub: r.scrub === null ? null : JSON.parse(r.scrub),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
function toEdge(r) {
|
|
54
|
+
return { fromArtifactId: r.from_artifact_id, toArtifactId: r.to_artifact_id, kind: r.kind };
|
|
55
|
+
}
|
|
56
|
+
function toEval(r) {
|
|
57
|
+
return {
|
|
58
|
+
id: r.id, artifactId: r.artifact_id, runId: r.run_id, evaluator: r.evaluator,
|
|
59
|
+
evaluatorKind: r.evaluator_kind,
|
|
60
|
+
score: r.score, passed: r.passed === null ? null : r.passed !== 0,
|
|
61
|
+
label: r.label, explanation: r.explanation, createdBy: r.created_by,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function toEvalCase(r) {
|
|
65
|
+
return {
|
|
66
|
+
id: r.id, task: r.task, sourceArtifactId: r.source_artifact_id,
|
|
67
|
+
correctedArtifactId: r.corrected_artifact_id,
|
|
68
|
+
origin: r.origin, note: r.note, createdBy: r.created_by,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
function openArtifactStore(dbPath) {
|
|
72
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(dbPath), { recursive: true });
|
|
73
|
+
const db = new better_sqlite3_1.default(dbPath);
|
|
74
|
+
db.pragma("journal_mode = WAL");
|
|
75
|
+
// Fail-open telemetry does not need FULL fsync durability — NORMAL is the
|
|
76
|
+
// WAL-recommended setting and drops the per-write fsync that made a step's
|
|
77
|
+
// ~9 inserts nine fsyncs (H8). Durability across a power loss is not a
|
|
78
|
+
// telemetry guarantee; the cloud exporter is the durable copy.
|
|
79
|
+
db.pragma("synchronous = NORMAL");
|
|
80
|
+
db.exec(ARTIFACTS_SQL);
|
|
81
|
+
// Additive column for a dev DB that predates E4 (CREATE TABLE IF NOT EXISTS
|
|
82
|
+
// won't add it to an existing table). SQLite has no ADD COLUMN IF NOT EXISTS;
|
|
83
|
+
// a duplicate-column error on re-run is expected and swallowed.
|
|
84
|
+
try {
|
|
85
|
+
db.exec("ALTER TABLE artifacts ADD COLUMN scrub TEXT");
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
/* column already present — no-op */
|
|
89
|
+
}
|
|
90
|
+
// Additive run_id on evaluations (E5 chunk V, subject-one-of). A dev DB that
|
|
91
|
+
// predates E5 keeps artifact_id NOT NULL — only run-subject inserts (new this
|
|
92
|
+
// phase) would need the relaxed column, and a fresh .modelrig DB gets it from
|
|
93
|
+
// CREATE TABLE above. Same swallow-on-duplicate pattern as scrub.
|
|
94
|
+
try {
|
|
95
|
+
db.exec("ALTER TABLE artifact_evaluations ADD COLUMN run_id TEXT");
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
/* column already present — no-op */
|
|
99
|
+
}
|
|
100
|
+
// One reusable transaction wrapper for batch() (better-sqlite3 wraps a fn).
|
|
101
|
+
const runInTransaction = db.transaction((fn) => fn());
|
|
102
|
+
const insArtifact = db.prepare(`
|
|
103
|
+
INSERT OR IGNORE INTO artifacts (
|
|
104
|
+
id, run_id, step_key, attempt_no, task, name, type, version, supersedes_artifact_id,
|
|
105
|
+
declared_schema, fingerprint_cell, content_ref, content_hash, content_bytes,
|
|
106
|
+
data_classification, expires_at, scrub
|
|
107
|
+
) VALUES (
|
|
108
|
+
@id, @run_id, @step_key, @attempt_no, @task, @name, @type, @version, @supersedes_artifact_id,
|
|
109
|
+
@declared_schema, @fingerprint_cell, @content_ref, @content_hash, @content_bytes,
|
|
110
|
+
@data_classification, @expires_at, @scrub
|
|
111
|
+
)`);
|
|
112
|
+
const insEdge = db.prepare("INSERT OR IGNORE INTO artifact_edges (from_artifact_id, to_artifact_id, kind) VALUES (@from_artifact_id, @to_artifact_id, @kind)");
|
|
113
|
+
const insEval = db.prepare(`
|
|
114
|
+
INSERT OR IGNORE INTO artifact_evaluations (
|
|
115
|
+
id, artifact_id, run_id, evaluator, evaluator_kind, score, passed, label, explanation, created_by
|
|
116
|
+
) VALUES (@id, @artifact_id, @run_id, @evaluator, @evaluator_kind, @score, @passed, @label, @explanation, @created_by)`);
|
|
117
|
+
const insEvalCase = db.prepare(`
|
|
118
|
+
INSERT OR IGNORE INTO eval_cases (
|
|
119
|
+
id, task, source_artifact_id, corrected_artifact_id, origin, note, created_by
|
|
120
|
+
) VALUES (@id, @task, @source_artifact_id, @corrected_artifact_id, @origin, @note, @created_by)`);
|
|
121
|
+
return {
|
|
122
|
+
recordArtifact(row) {
|
|
123
|
+
insArtifact.run({
|
|
124
|
+
id: row.id, run_id: row.runId, step_key: row.stepKey, attempt_no: row.attemptNo,
|
|
125
|
+
task: row.task, name: row.name, type: row.type, version: row.version,
|
|
126
|
+
supersedes_artifact_id: row.supersedesArtifactId,
|
|
127
|
+
declared_schema: row.declaredSchema === null || row.declaredSchema === undefined
|
|
128
|
+
? null : JSON.stringify(row.declaredSchema),
|
|
129
|
+
fingerprint_cell: row.fingerprintCell, content_ref: row.contentRef,
|
|
130
|
+
content_hash: row.contentHash, content_bytes: row.contentBytes,
|
|
131
|
+
data_classification: row.dataClassification, expires_at: row.expiresAt,
|
|
132
|
+
scrub: row.scrub === null || row.scrub === undefined ? null : JSON.stringify(row.scrub),
|
|
133
|
+
});
|
|
134
|
+
},
|
|
135
|
+
recordArtifactEdge(row) {
|
|
136
|
+
insEdge.run({ from_artifact_id: row.fromArtifactId, to_artifact_id: row.toArtifactId, kind: row.kind });
|
|
137
|
+
},
|
|
138
|
+
recordArtifactEvaluation(row) {
|
|
139
|
+
insEval.run({
|
|
140
|
+
id: row.id, artifact_id: row.artifactId, run_id: row.runId, evaluator: row.evaluator,
|
|
141
|
+
evaluator_kind: row.evaluatorKind, score: row.score,
|
|
142
|
+
passed: row.passed === null ? null : row.passed ? 1 : 0,
|
|
143
|
+
label: row.label, explanation: row.explanation, created_by: row.createdBy,
|
|
144
|
+
});
|
|
145
|
+
},
|
|
146
|
+
recordEvalCase(row) {
|
|
147
|
+
insEvalCase.run({
|
|
148
|
+
id: row.id, task: row.task, source_artifact_id: row.sourceArtifactId,
|
|
149
|
+
corrected_artifact_id: row.correctedArtifactId, origin: row.origin,
|
|
150
|
+
note: row.note, created_by: row.createdBy,
|
|
151
|
+
});
|
|
152
|
+
},
|
|
153
|
+
readArtifactsAfter(afterRowid, limit) {
|
|
154
|
+
const rows = db.prepare("SELECT rowid AS __r, * FROM artifacts WHERE rowid > ? ORDER BY rowid LIMIT ?")
|
|
155
|
+
.all(afterRowid, limit);
|
|
156
|
+
return rows.map((r) => ({ rowid: r.__r, row: toArtifact(r) }));
|
|
157
|
+
},
|
|
158
|
+
readEdgesAfter(afterRowid, limit) {
|
|
159
|
+
const rows = db.prepare("SELECT rowid AS __r, * FROM artifact_edges WHERE rowid > ? ORDER BY rowid LIMIT ?")
|
|
160
|
+
.all(afterRowid, limit);
|
|
161
|
+
return rows.map((r) => ({ rowid: r.__r, row: toEdge(r) }));
|
|
162
|
+
},
|
|
163
|
+
readEvaluationsAfter(afterRowid, limit) {
|
|
164
|
+
const rows = db.prepare("SELECT rowid AS __r, * FROM artifact_evaluations WHERE rowid > ? ORDER BY rowid LIMIT ?")
|
|
165
|
+
.all(afterRowid, limit);
|
|
166
|
+
return rows.map((r) => ({ rowid: r.__r, row: toEval(r) }));
|
|
167
|
+
},
|
|
168
|
+
readEvalCasesAfter(afterRowid, limit) {
|
|
169
|
+
const rows = db.prepare("SELECT rowid AS __r, * FROM eval_cases WHERE rowid > ? ORDER BY rowid LIMIT ?")
|
|
170
|
+
.all(afterRowid, limit);
|
|
171
|
+
return rows.map((r) => ({ rowid: r.__r, row: toEvalCase(r) }));
|
|
172
|
+
},
|
|
173
|
+
readEvalCasesByTask(task) {
|
|
174
|
+
const rows = db.prepare("SELECT * FROM eval_cases WHERE task = ? ORDER BY rowid DESC")
|
|
175
|
+
.all(task);
|
|
176
|
+
return rows.map(toEvalCase);
|
|
177
|
+
},
|
|
178
|
+
getArtifactById(id) {
|
|
179
|
+
const r = db.prepare("SELECT * FROM artifacts WHERE id = ?").get(id);
|
|
180
|
+
return r === undefined ? null : toArtifact(r);
|
|
181
|
+
},
|
|
182
|
+
batch(fn) {
|
|
183
|
+
return runInTransaction(fn);
|
|
184
|
+
},
|
|
185
|
+
close() {
|
|
186
|
+
db.close();
|
|
187
|
+
},
|
|
188
|
+
};
|
|
189
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Full-price cache-miss streak detection — cache-aware-routing WS-C step 5
|
|
3
|
+
* (spec §5.6, research §2.3 differentiator: "paying full price and don't know it").
|
|
4
|
+
*
|
|
5
|
+
* A route that opted into `cache: auto` but keeps reading ZERO cached tokens on
|
|
6
|
+
* cacheable-size prompts is silently paying the write premium forever. This
|
|
7
|
+
* module is the PURE DETECTION over telemetry rows — a `EndpointHealth`-style
|
|
8
|
+
* companion to health.ts. It produces a structured `CacheMissStreak` per
|
|
9
|
+
* (route, provider); turning that into a `cache_miss_streak` health event + the
|
|
10
|
+
* console badge is the WIRING step, deliberately DEFERRED: per the WS-C fence,
|
|
11
|
+
* `src/watch/monitor.ts` is not touched while WS-B is in flight — this module is
|
|
12
|
+
* wired on the rebase after WS-B merges (never-simultaneous-in-src/watch rule).
|
|
13
|
+
*
|
|
14
|
+
* Opt-in signal WITHOUT a route lookup: a telemetry row carries a non-null
|
|
15
|
+
* `prefixFingerprint` iff its route opted into `cache: auto` (the run loop stamps
|
|
16
|
+
* it for every provider on such a route — cache-stamp.ts). So this detector keys
|
|
17
|
+
* off `prefixFingerprint !== null`; it needs no route registry.
|
|
18
|
+
*
|
|
19
|
+
* Fail-closed, per L5:
|
|
20
|
+
* • only providers whose policy REPORTS read tokens are eligible — where reads
|
|
21
|
+
* ride HTTP headers we don't lift (fireworks), `tokensCached` is structurally
|
|
22
|
+
* 0 and a "miss" is unobservable, so those rows are skipped (never a false alarm);
|
|
23
|
+
* • only providers with a documented `min_cacheable_tokens` are eligible — with
|
|
24
|
+
* no first-party minimum we cannot assert a prompt was large enough to cache,
|
|
25
|
+
* so no streak is claimed;
|
|
26
|
+
* • a row only counts toward the streak if it is BIG ENOUGH to cache
|
|
27
|
+
* (`tokensIn > min_cacheable`); small prompts legitimately never cache.
|
|
28
|
+
*
|
|
29
|
+
* Cross-reference: this is the PRODUCTION-ROUTE-TELEMETRY plane of the same smell
|
|
30
|
+
* the registry surfaces on the PROBE/DECLARED plane as `declared_caching_unrealized`
|
|
31
|
+
* (registry/build.ts): a model that declares caching but shows no observed hits.
|
|
32
|
+
* Same signal, different plane — WS-D's badge should note both.
|
|
33
|
+
*/
|
|
34
|
+
import type { ProviderId } from "../types/provider";
|
|
35
|
+
import type { CachePolicy } from "../registry/cache-policy";
|
|
36
|
+
/** The minimal telemetry projection the detector needs. Both the local SQLite
|
|
37
|
+
* store and the hosted rollup can project rows onto this. */
|
|
38
|
+
export interface CacheMissRow {
|
|
39
|
+
readonly route: string;
|
|
40
|
+
readonly provider: string;
|
|
41
|
+
readonly ts: string;
|
|
42
|
+
readonly tokensIn: number;
|
|
43
|
+
readonly tokensCached: number;
|
|
44
|
+
/** Non-null ⇒ the route opted into `cache: auto` (run loop stamps it). */
|
|
45
|
+
readonly prefixFingerprint: string | null;
|
|
46
|
+
/** Replay/shadow rows are excluded (odometer rule: tags.replay !== undefined).
|
|
47
|
+
* The caller computes this from `tagsJson`; the detector treats it as ground. */
|
|
48
|
+
readonly isReplay: boolean;
|
|
49
|
+
}
|
|
50
|
+
/** A detected full-price streak on one (route, provider). */
|
|
51
|
+
export interface CacheMissStreak {
|
|
52
|
+
readonly route: string;
|
|
53
|
+
readonly provider: ProviderId;
|
|
54
|
+
/** How many consecutive recent cacheable-size rows read zero cached tokens. */
|
|
55
|
+
readonly streakLength: number;
|
|
56
|
+
/** The policy minimum the streak rows all exceeded. */
|
|
57
|
+
readonly minCacheableTokens: number;
|
|
58
|
+
/** Mean input tokens across the streak (how much is being paid full price). */
|
|
59
|
+
readonly meanTokensIn: number;
|
|
60
|
+
/** ISO ts of the most recent row in the streak. */
|
|
61
|
+
readonly lastSeenAt: string;
|
|
62
|
+
}
|
|
63
|
+
export interface CacheMissStreakOptions {
|
|
64
|
+
/** Consecutive full-price rows required to raise the alert (spec §5.6: 50). */
|
|
65
|
+
readonly streakLength?: number;
|
|
66
|
+
/** Policy accessor — injectable for tests; defaults to the registry. */
|
|
67
|
+
readonly getPolicy?: (provider: ProviderId) => CachePolicy;
|
|
68
|
+
}
|
|
69
|
+
export declare const DEFAULT_STREAK_LENGTH = 50;
|
|
70
|
+
/**
|
|
71
|
+
* Detect full-price cache-miss streaks across `rows`. Pure and total.
|
|
72
|
+
*
|
|
73
|
+
* Groups eligible rows (cache:auto, production, observable provider, cacheable
|
|
74
|
+
* size) by (route, provider); a group raises a streak when its most recent
|
|
75
|
+
* `streakLength` such rows ALL read zero cached tokens. Output is sorted by
|
|
76
|
+
* route then provider for deterministic rollups.
|
|
77
|
+
*/
|
|
78
|
+
export declare function detectCacheMissStreaks(rows: readonly CacheMissRow[], opts?: CacheMissStreakOptions): CacheMissStreak[];
|
|
79
|
+
//# sourceMappingURL=cache-miss-streak.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache-miss-streak.d.ts","sourceRoot":"","sources":["../../src/telemetry/cache-miss-streak.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAE5D;6DAC6D;AAC7D,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,0EAA0E;IAC1E,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C;qFACiF;IACjF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED,6DAA6D;AAC7D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,+EAA+E;IAC/E,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,uDAAuD;IACvD,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,+EAA+E;IAC/E,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,mDAAmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,sBAAsB;IACrC,+EAA+E;IAC/E,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,wEAAwE;IACxE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,KAAK,WAAW,CAAC;CAC5D;AAED,eAAO,MAAM,qBAAqB,KAAK,CAAC;AAIxC;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,SAAS,YAAY,EAAE,EAC7B,IAAI,GAAE,sBAA2B,GAChC,eAAe,EAAE,CAmDnB"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Full-price cache-miss streak detection — cache-aware-routing WS-C step 5
|
|
4
|
+
* (spec §5.6, research §2.3 differentiator: "paying full price and don't know it").
|
|
5
|
+
*
|
|
6
|
+
* A route that opted into `cache: auto` but keeps reading ZERO cached tokens on
|
|
7
|
+
* cacheable-size prompts is silently paying the write premium forever. This
|
|
8
|
+
* module is the PURE DETECTION over telemetry rows — a `EndpointHealth`-style
|
|
9
|
+
* companion to health.ts. It produces a structured `CacheMissStreak` per
|
|
10
|
+
* (route, provider); turning that into a `cache_miss_streak` health event + the
|
|
11
|
+
* console badge is the WIRING step, deliberately DEFERRED: per the WS-C fence,
|
|
12
|
+
* `src/watch/monitor.ts` is not touched while WS-B is in flight — this module is
|
|
13
|
+
* wired on the rebase after WS-B merges (never-simultaneous-in-src/watch rule).
|
|
14
|
+
*
|
|
15
|
+
* Opt-in signal WITHOUT a route lookup: a telemetry row carries a non-null
|
|
16
|
+
* `prefixFingerprint` iff its route opted into `cache: auto` (the run loop stamps
|
|
17
|
+
* it for every provider on such a route — cache-stamp.ts). So this detector keys
|
|
18
|
+
* off `prefixFingerprint !== null`; it needs no route registry.
|
|
19
|
+
*
|
|
20
|
+
* Fail-closed, per L5:
|
|
21
|
+
* • only providers whose policy REPORTS read tokens are eligible — where reads
|
|
22
|
+
* ride HTTP headers we don't lift (fireworks), `tokensCached` is structurally
|
|
23
|
+
* 0 and a "miss" is unobservable, so those rows are skipped (never a false alarm);
|
|
24
|
+
* • only providers with a documented `min_cacheable_tokens` are eligible — with
|
|
25
|
+
* no first-party minimum we cannot assert a prompt was large enough to cache,
|
|
26
|
+
* so no streak is claimed;
|
|
27
|
+
* • a row only counts toward the streak if it is BIG ENOUGH to cache
|
|
28
|
+
* (`tokensIn > min_cacheable`); small prompts legitimately never cache.
|
|
29
|
+
*
|
|
30
|
+
* Cross-reference: this is the PRODUCTION-ROUTE-TELEMETRY plane of the same smell
|
|
31
|
+
* the registry surfaces on the PROBE/DECLARED plane as `declared_caching_unrealized`
|
|
32
|
+
* (registry/build.ts): a model that declares caching but shows no observed hits.
|
|
33
|
+
* Same signal, different plane — WS-D's badge should note both.
|
|
34
|
+
*/
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.DEFAULT_STREAK_LENGTH = void 0;
|
|
37
|
+
exports.detectCacheMissStreaks = detectCacheMissStreaks;
|
|
38
|
+
const cache_policy_1 = require("../registry/cache-policy");
|
|
39
|
+
exports.DEFAULT_STREAK_LENGTH = 50;
|
|
40
|
+
const KEY_SEP = "|";
|
|
41
|
+
/**
|
|
42
|
+
* Detect full-price cache-miss streaks across `rows`. Pure and total.
|
|
43
|
+
*
|
|
44
|
+
* Groups eligible rows (cache:auto, production, observable provider, cacheable
|
|
45
|
+
* size) by (route, provider); a group raises a streak when its most recent
|
|
46
|
+
* `streakLength` such rows ALL read zero cached tokens. Output is sorted by
|
|
47
|
+
* route then provider for deterministic rollups.
|
|
48
|
+
*/
|
|
49
|
+
function detectCacheMissStreaks(rows, opts = {}) {
|
|
50
|
+
const streakLength = opts.streakLength ?? exports.DEFAULT_STREAK_LENGTH;
|
|
51
|
+
const getPolicy = opts.getPolicy ?? cache_policy_1.getCachePolicy;
|
|
52
|
+
// Bucket eligible rows by (route, provider), remembering each provider's
|
|
53
|
+
// documented minimum (looked up once per provider).
|
|
54
|
+
const byKey = new Map();
|
|
55
|
+
const minByProvider = new Map();
|
|
56
|
+
for (const row of rows) {
|
|
57
|
+
if (row.prefixFingerprint === null)
|
|
58
|
+
continue; // route didn't opt into cache:auto
|
|
59
|
+
if (row.isReplay)
|
|
60
|
+
continue; // odometer replay-exclusion rule
|
|
61
|
+
let min = minByProvider.get(row.provider);
|
|
62
|
+
if (min === undefined) {
|
|
63
|
+
const policy = getPolicy(row.provider);
|
|
64
|
+
// Only providers we can OBSERVE hits for, with a sourced minimum, are eligible.
|
|
65
|
+
min =
|
|
66
|
+
policy.cache_supported && policy.reports_read_tokens ? policy.min_cacheable_tokens : null;
|
|
67
|
+
minByProvider.set(row.provider, min);
|
|
68
|
+
}
|
|
69
|
+
if (min === null)
|
|
70
|
+
continue; // unobservable or no first-party minimum → fail-closed
|
|
71
|
+
if (row.tokensIn <= min)
|
|
72
|
+
continue; // too small to cache — not a "miss"
|
|
73
|
+
const key = `${row.route}${KEY_SEP}${row.provider}`;
|
|
74
|
+
const list = byKey.get(key) ?? [];
|
|
75
|
+
list.push(row);
|
|
76
|
+
byKey.set(key, list);
|
|
77
|
+
}
|
|
78
|
+
const out = [];
|
|
79
|
+
for (const [key, group] of byKey) {
|
|
80
|
+
if (group.length < streakLength)
|
|
81
|
+
continue; // not enough evidence yet
|
|
82
|
+
// Most-recent `streakLength` rows (ascending ts, take the tail).
|
|
83
|
+
const sorted = [...group].sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts));
|
|
84
|
+
const recent = sorted.slice(sorted.length - streakLength);
|
|
85
|
+
if (!recent.every((r) => r.tokensCached === 0))
|
|
86
|
+
continue; // a hit broke the streak
|
|
87
|
+
const [route, provider] = key.split(KEY_SEP);
|
|
88
|
+
const meanTokensIn = Math.round(recent.reduce((sum, r) => sum + r.tokensIn, 0) / recent.length);
|
|
89
|
+
out.push({
|
|
90
|
+
route,
|
|
91
|
+
provider: provider,
|
|
92
|
+
streakLength,
|
|
93
|
+
minCacheableTokens: minByProvider.get(provider),
|
|
94
|
+
meanTokensIn,
|
|
95
|
+
lastSeenAt: recent[recent.length - 1].ts,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
return out.sort((a, b) => a.route === b.route ? a.provider.localeCompare(b.provider) : a.route.localeCompare(b.route));
|
|
99
|
+
}
|
|
@@ -17,6 +17,9 @@ export interface CaptureStore {
|
|
|
17
17
|
record(row: CaptureRow): void;
|
|
18
18
|
/** Most recent N captures for a route (replay's read path). */
|
|
19
19
|
readLast(route: string, n: number): CaptureRow[];
|
|
20
|
+
/** One capture by its inference id, or null (E4 chunk A: rehydrate an eval
|
|
21
|
+
* case's input from the inference that produced its source artifact). */
|
|
22
|
+
getByInferenceId(inferenceId: string): CaptureRow | null;
|
|
20
23
|
close(): void;
|
|
21
24
|
}
|
|
22
25
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"capture.d.ts","sourceRoot":"","sources":["../../src/telemetry/capture.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC9B,+DAA+D;IAC/D,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CAAC;IACjD,KAAK,IAAI,IAAI,CAAC;CACf;AAyBD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,
|
|
1
|
+
{"version":3,"file":"capture.d.ts","sourceRoot":"","sources":["../../src/telemetry/capture.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC9B,+DAA+D;IAC/D,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CAAC;IACjD;6EACyE;IACzE,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC;IACzD,KAAK,IAAI,IAAI,CAAC;CACf;AAyBD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAoD7D"}
|
|
@@ -45,6 +45,14 @@ function openCaptureStore(dbPath) {
|
|
|
45
45
|
const selectLast = db.prepare(`
|
|
46
46
|
SELECT * FROM captures WHERE route = ? ORDER BY ts DESC, rowid DESC LIMIT ?
|
|
47
47
|
`);
|
|
48
|
+
const toRow = (r) => ({
|
|
49
|
+
inferenceId: r.inference_id,
|
|
50
|
+
ts: r.ts,
|
|
51
|
+
route: r.route,
|
|
52
|
+
routeVersion: r.route_version,
|
|
53
|
+
variablesJson: r.variables_json,
|
|
54
|
+
outputText: r.output_text,
|
|
55
|
+
});
|
|
48
56
|
return {
|
|
49
57
|
record(row) {
|
|
50
58
|
insert.run({
|
|
@@ -58,14 +66,13 @@ function openCaptureStore(dbPath) {
|
|
|
58
66
|
},
|
|
59
67
|
readLast(route, n) {
|
|
60
68
|
const rows = selectLast.all(route, n);
|
|
61
|
-
return rows.map(
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
}));
|
|
69
|
+
return rows.map(toRow);
|
|
70
|
+
},
|
|
71
|
+
getByInferenceId(inferenceId) {
|
|
72
|
+
const r = db
|
|
73
|
+
.prepare("SELECT * FROM captures WHERE inference_id = ?")
|
|
74
|
+
.get(inferenceId);
|
|
75
|
+
return r === undefined ? null : toRow(r);
|
|
69
76
|
},
|
|
70
77
|
close() {
|
|
71
78
|
db.close();
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build-and-start the telemetry exporter from resolved config. Extracted from
|
|
3
|
+
* index.ts (the SDK entrypoint) so that file stays under the LOC cap while
|
|
4
|
+
* Phase 2 adds the settings resolver + boot banner beside it — the exporter
|
|
5
|
+
* wiring is mechanical and has one caller, so it lives here rather than inline.
|
|
6
|
+
*/
|
|
7
|
+
import type { RigManifest } from "../routes/manifest";
|
|
8
|
+
import type { ServeabilityWarning } from "../routes/serveability";
|
|
9
|
+
import type { ApiExportConfig, DirectExportConfig } from "../config";
|
|
10
|
+
import type { Exporter, ExportedRouteMirror } from "./exporter";
|
|
11
|
+
import type { ArtifactStore } from "./artifact-store";
|
|
12
|
+
import type { ExportableTelemetryStore } from "./store";
|
|
13
|
+
export interface ExporterSetupExtras {
|
|
14
|
+
readonly routes: readonly ExportedRouteMirror[];
|
|
15
|
+
readonly manifest: RigManifest | null;
|
|
16
|
+
readonly serveability: readonly ServeabilityWarning[];
|
|
17
|
+
/** v2 wire + runs/steps/artifact streams only when the namespace is enabled. */
|
|
18
|
+
readonly artifactsEnabled: boolean;
|
|
19
|
+
readonly artifactStore: ArtifactStore | null;
|
|
20
|
+
/** E4 chunk 0: retry 404'd content after a confirmed artifacts flush. */
|
|
21
|
+
readonly onArtifactsFlushed?: () => Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Construct the sink for the configured mode, wire the exporter, and start its
|
|
25
|
+
* flush loop. Returns the running exporter (the caller stops it on close()).
|
|
26
|
+
*/
|
|
27
|
+
export declare function startTelemetryExporter(telemetry: ExportableTelemetryStore, exportConfig: ApiExportConfig | DirectExportConfig, extras: ExporterSetupExtras): Exporter;
|
|
28
|
+
//# sourceMappingURL=exporter-setup.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exporter-setup.d.ts","sourceRoot":"","sources":["../../src/telemetry/exporter-setup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAErE,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAC;AAExD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,MAAM,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAChD,QAAQ,CAAC,QAAQ,EAAE,WAAW,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,YAAY,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACtD,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7C,yEAAyE;IACzE,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACnD;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,wBAAwB,EACnC,YAAY,EAAE,eAAe,GAAG,kBAAkB,EAClD,MAAM,EAAE,mBAAmB,GAC1B,QAAQ,CAiBV"}
|