modelrig 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +41 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +46 -6
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +105 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +37 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +138 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +122 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
|
@@ -13,9 +13,24 @@ export interface TelemetryRow {
|
|
|
13
13
|
readonly model: string;
|
|
14
14
|
readonly requestedTier: string;
|
|
15
15
|
readonly servedTier: string;
|
|
16
|
+
/** Total input tokens. NORMALIZATION INVARIANT (WS-A / spec §2): INCLUDES
|
|
17
|
+
* `tokensCached` and `tokensCacheWrite` as subsets — mirrors
|
|
18
|
+
* {@link AdapterUsage.tokensIn}. */
|
|
16
19
|
readonly tokensIn: number;
|
|
17
20
|
readonly tokensOut: number;
|
|
21
|
+
/** Cache-READ input tokens (subset of `tokensIn`). */
|
|
18
22
|
readonly tokensCached: number;
|
|
23
|
+
/** Cache-CREATION (write) input tokens (subset of `tokensIn`) — WS-A. Mirrors
|
|
24
|
+
* {@link AdapterUsage.tokensCacheWrite}; 0 when the provider does not write
|
|
25
|
+
* cache or does not report it. Persisted to `inferences.tokens_cache_write`
|
|
26
|
+
* (migration 00049) and read back as 0 for rows recorded before it. OPTIONAL
|
|
27
|
+
* & additive: the store binds/reads `?? 0`, so a row that omits it is a
|
|
28
|
+
* zero-write row (the fail-closed default) — no construction site is forced
|
|
29
|
+
* to change. Every SDK write path (run loop, raw lane) sets it explicitly. */
|
|
30
|
+
readonly tokensCacheWrite?: number;
|
|
31
|
+
/** 1h-TTL portion of `tokensCacheWrite` (Anthropic; bills at the 2×
|
|
32
|
+
* multiplier). Subset of `tokensCacheWrite`; 0 = all 5m-tier. Optional/additive. */
|
|
33
|
+
readonly tokensCacheWrite1h?: number;
|
|
19
34
|
readonly costEstimateUsd: number;
|
|
20
35
|
readonly latencyMs: number;
|
|
21
36
|
/** Time to first body byte, ms (Phase 4 WS3) — null when nothing arrived
|
|
@@ -27,6 +42,14 @@ export interface TelemetryRow {
|
|
|
27
42
|
/** Schema-fingerprint cell key (registry/fingerprint.ts) — null for
|
|
28
43
|
* unstructured routes and rows recorded before the v2 migration. */
|
|
29
44
|
readonly fingerprintCell: string | null;
|
|
45
|
+
/** Provider cache key / `prompt_cache_key` sent with the request, if any
|
|
46
|
+
* (WS-A substrate; the WS-C request-path opt-in fills it). null everywhere
|
|
47
|
+
* in WS-A — accounting ships dark, no request stamps a key yet. Optional &
|
|
48
|
+
* additive: the store binds/reads `?? null`. */
|
|
49
|
+
readonly cacheKey?: string | null;
|
|
50
|
+
/** Hash of the stable prompt prefix (WS-A substrate; WS-C fills it with
|
|
51
|
+
* `hash(route, template_version, toolschema_hash)`). null in WS-A. Optional. */
|
|
52
|
+
readonly prefixFingerprint?: string | null;
|
|
30
53
|
/** True when this attempt's model had no pricing entry (addendum A1):
|
|
31
54
|
* cost_estimate_usd stays 0 (no invented dollars) and any active envelope
|
|
32
55
|
* was charged the conservative unpriced rate instead. */
|
|
@@ -77,6 +100,100 @@ export interface FeedbackRow {
|
|
|
77
100
|
* missing data, and is never backfilled. */
|
|
78
101
|
readonly createdBy: string | null;
|
|
79
102
|
}
|
|
103
|
+
/** One execution of a customer pipeline (Expansion E2 chunk A). Local SQLite is
|
|
104
|
+
* the write path (run.start/run.end); the exporter mirrors it to `public.runs`
|
|
105
|
+
* on its own watermark. A mutable projection: start → end. `source_id`/`org_id`
|
|
106
|
+
* are stamped at export from the principal, never held locally (one rig = one
|
|
107
|
+
* source). */
|
|
108
|
+
export interface RunRow {
|
|
109
|
+
readonly id: string;
|
|
110
|
+
readonly pipeline: string;
|
|
111
|
+
readonly pipelineVersion: string | null;
|
|
112
|
+
readonly environment: string | null;
|
|
113
|
+
readonly episodeKey: string | null;
|
|
114
|
+
readonly startedAt: string;
|
|
115
|
+
readonly endedAt: string | null;
|
|
116
|
+
readonly status: "running" | "succeeded" | "failed" | "abandoned";
|
|
117
|
+
}
|
|
118
|
+
/** One ATTEMPT of one operation within a run (Expansion E2 chunk A, ruling c).
|
|
119
|
+
* A fact — one row per attempt, PK (run_id, step_key, attempt_no). Provenance is
|
|
120
|
+
* ground-truth (an inference reference) XOR attested (a foreign OTLP span);
|
|
121
|
+
* nothing writes attested rows this mission. */
|
|
122
|
+
export interface StepRow {
|
|
123
|
+
readonly runId: string;
|
|
124
|
+
readonly stepKey: string;
|
|
125
|
+
readonly attemptNo: number;
|
|
126
|
+
readonly task: string | null;
|
|
127
|
+
readonly provenance: "ground-truth" | "attested";
|
|
128
|
+
readonly inferenceId: string | null;
|
|
129
|
+
readonly spanTraceId: string | null;
|
|
130
|
+
readonly spanId: string | null;
|
|
131
|
+
readonly startedAt: string;
|
|
132
|
+
readonly endedAt: string | null;
|
|
133
|
+
readonly failureClass: string | null;
|
|
134
|
+
}
|
|
135
|
+
/** Artifact METADATA (Expansion E2 chunk B). Content is never held — contentRef
|
|
136
|
+
* is the customer's opaque URI, contentHash a sha256 of bytes hashed-then-
|
|
137
|
+
* discarded. A fact; local SQLite write path (artifact.save), exporter mirrors. */
|
|
138
|
+
export interface ArtifactRow {
|
|
139
|
+
readonly id: string;
|
|
140
|
+
readonly runId: string | null;
|
|
141
|
+
readonly stepKey: string | null;
|
|
142
|
+
readonly attemptNo: number | null;
|
|
143
|
+
readonly task: string | null;
|
|
144
|
+
readonly name: string;
|
|
145
|
+
readonly type: string;
|
|
146
|
+
readonly version: number;
|
|
147
|
+
readonly supersedesArtifactId: string | null;
|
|
148
|
+
readonly declaredSchema: unknown;
|
|
149
|
+
readonly fingerprintCell: string | null;
|
|
150
|
+
readonly contentRef: string | null;
|
|
151
|
+
readonly contentHash: string | null;
|
|
152
|
+
readonly contentBytes: number | null;
|
|
153
|
+
readonly dataClassification: string | null;
|
|
154
|
+
readonly expiresAt: string | null;
|
|
155
|
+
/** Per-row scrub findings (E4 chunk 0, migration 00039) — the `ScrubSummary`
|
|
156
|
+
* shape `{mode, blocked, findings:[{type,count}]}`, COUNTS never values.
|
|
157
|
+
* Absent/null ⇒ scrub did not run for this artifact (custody off, caller-
|
|
158
|
+
* supplied contentRef, or a pre-E4 row). Opaque here (T7). */
|
|
159
|
+
readonly scrub?: unknown;
|
|
160
|
+
}
|
|
161
|
+
/** A lineage edge between two artifacts (chunk B). */
|
|
162
|
+
export interface ArtifactEdgeRow {
|
|
163
|
+
readonly fromArtifactId: string;
|
|
164
|
+
readonly toArtifactId: string;
|
|
165
|
+
readonly kind: "consumed" | "derived_from";
|
|
166
|
+
}
|
|
167
|
+
/** An evaluation recorded against an artifact (chunk B). */
|
|
168
|
+
export interface ArtifactEvaluationRow {
|
|
169
|
+
readonly id: string;
|
|
170
|
+
/** The artifact subject — null when this evaluates a whole run (E5 chunk V). */
|
|
171
|
+
readonly artifactId: string | null;
|
|
172
|
+
/** The run subject (E5 chunk V) — null when this evaluates an artifact.
|
|
173
|
+
* Exactly one of artifactId / runId is set. */
|
|
174
|
+
readonly runId: string | null;
|
|
175
|
+
readonly evaluator: string;
|
|
176
|
+
readonly evaluatorKind: "deterministic" | "model-judge" | "human";
|
|
177
|
+
readonly score: number | null;
|
|
178
|
+
readonly passed: boolean | null;
|
|
179
|
+
readonly label: string | null;
|
|
180
|
+
readonly explanation: string | null;
|
|
181
|
+
readonly createdBy: string | null;
|
|
182
|
+
}
|
|
183
|
+
/** How a case entered the eval suite (E4 chunk A, migration 00040 CHECK). */
|
|
184
|
+
export type EvalCaseOrigin = "promoted-failure" | "human-corrected" | "manual";
|
|
185
|
+
/** A per-org regression/golden eval case (E4 chunk A). `correctedArtifactId`
|
|
186
|
+
* null ⇒ a failure-case; set ⇒ a golden-corrected pair. A fact on ingest
|
|
187
|
+
* (created via promote); edits/deletes are server-side, not on this stream. */
|
|
188
|
+
export interface EvalCaseRow {
|
|
189
|
+
readonly id: string;
|
|
190
|
+
readonly task: string;
|
|
191
|
+
readonly sourceArtifactId: string;
|
|
192
|
+
readonly correctedArtifactId: string | null;
|
|
193
|
+
readonly origin: EvalCaseOrigin;
|
|
194
|
+
readonly note: string | null;
|
|
195
|
+
readonly createdBy: string | null;
|
|
196
|
+
}
|
|
80
197
|
export interface TelemetryStore {
|
|
81
198
|
record(row: TelemetryRow): void;
|
|
82
199
|
recordFeedback(row: FeedbackRow): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAEvC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;uDACmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;wEACoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;6DAEyD;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;oDACgD;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;uDACmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC;CACvC;AAED;;6EAE6E;AAC7E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;gDAE4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,GAAG,EAAE,MAAM,OAAO,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC;IAC7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9C,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC;IAC1D,KAAK,IAAI,IAAI,CAAC;CACf"}
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAEvC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;wCAEoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;kFAM8E;IAC9E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;wFACoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;uDACmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;wEACoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;oDAGgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;oFACgF;IAChF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C;;6DAEyD;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;oDACgD;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;uDACmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC;CACvC;AAED;;6EAE6E;AAC7E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;gDAE4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;cAIc;AACd,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;CACnE;AAED;;;gDAGgD;AAChD,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,cAAc,GAAG,UAAU,CAAC;IACjD,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED;;mFAEmF;AACnF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;kEAG8D;IAC9D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC;CAC5C;AAED,4DAA4D;AAC5D,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;mDAC+C;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,eAAe,GAAG,aAAa,GAAG,OAAO,CAAC;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,6EAA6E;AAC7E,MAAM,MAAM,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,QAAQ,CAAC;AAE/E;;+EAE+E;AAC/E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,GAAG,EAAE,MAAM,OAAO,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC;IAC7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9C,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC;IAC1D,KAAK,IAAI,IAAI,CAAC;CACf"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `modelrig validate` (CE-6, spec §1, walkthrough §8.7a) — extracted from
|
|
3
|
+
* cli.ts to keep it under the 700-LOC cap. Structurally loads rig.yaml + every
|
|
4
|
+
* route bundle and reports per-route OK / config error, exiting nonzero on any
|
|
5
|
+
* failure. Requires NO provider key: this bypasses the serveable-candidate gate
|
|
6
|
+
* (which needs keys and stays fail-closed for SERVING) so a newbie can check
|
|
7
|
+
* their authoring before wiring credentials. A bare
|
|
8
|
+
* `createRig(loadConfigFromEnv())` fail-closes keyless with "no serveable
|
|
9
|
+
* candidate" — this is the dry structural check that doesn't.
|
|
10
|
+
*/
|
|
11
|
+
export declare function runValidateCli(): number;
|
|
12
|
+
//# sourceMappingURL=validate-cli.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAOH,wBAAgB,cAAc,IAAI,MAAM,CAmCvC"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `modelrig validate` (CE-6, spec §1, walkthrough §8.7a) — extracted from
|
|
4
|
+
* cli.ts to keep it under the 700-LOC cap. Structurally loads rig.yaml + every
|
|
5
|
+
* route bundle and reports per-route OK / config error, exiting nonzero on any
|
|
6
|
+
* failure. Requires NO provider key: this bypasses the serveable-candidate gate
|
|
7
|
+
* (which needs keys and stays fail-closed for SERVING) so a newbie can check
|
|
8
|
+
* their authoring before wiring credentials. A bare
|
|
9
|
+
* `createRig(loadConfigFromEnv())` fail-closes keyless with "no serveable
|
|
10
|
+
* candidate" — this is the dry structural check that doesn't.
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.runValidateCli = runValidateCli;
|
|
14
|
+
const node_path_1 = require("node:path");
|
|
15
|
+
const config_1 = require("./config");
|
|
16
|
+
const loader_1 = require("./routes/loader");
|
|
17
|
+
const manifest_1 = require("./routes/manifest");
|
|
18
|
+
function runValidateCli() {
|
|
19
|
+
const config = (0, config_1.loadConfigFromEnv)();
|
|
20
|
+
const rigDir = (0, node_path_1.dirname)(config.routesDir);
|
|
21
|
+
let failures = 0;
|
|
22
|
+
// rig.yaml (manifest) — absence is fine (never gates); a malformed one is a
|
|
23
|
+
// config error surfaced like any route.
|
|
24
|
+
try {
|
|
25
|
+
const manifest = (0, manifest_1.loadManifest)(rigDir);
|
|
26
|
+
console.log(manifest === null ? `— rig.yaml (none at ${rigDir})` : "OK rig.yaml");
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
failures += 1;
|
|
30
|
+
console.error(`FAIL rig.yaml: ${configErrorText(err)}`);
|
|
31
|
+
}
|
|
32
|
+
// Every route bundle — one bad file never hides the rest.
|
|
33
|
+
const results = (0, loader_1.validateRoutesDir)(config.routesDir);
|
|
34
|
+
if (results.length === 0) {
|
|
35
|
+
console.log(`(no route bundles in ${config.routesDir})`);
|
|
36
|
+
}
|
|
37
|
+
for (const r of results) {
|
|
38
|
+
if (r.ok) {
|
|
39
|
+
console.log(`OK ${r.route} (${r.file})`);
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
failures += 1;
|
|
43
|
+
console.error(`FAIL ${r.route ?? r.file}: ${r.problems.join("; ")}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (failures === 0) {
|
|
47
|
+
console.log(`\n${results.length} route(s) valid — no provider key required for this check.`);
|
|
48
|
+
return 0;
|
|
49
|
+
}
|
|
50
|
+
console.error(`\n${failures} problem(s) found.`);
|
|
51
|
+
return 1;
|
|
52
|
+
}
|
|
53
|
+
function configErrorText(err) {
|
|
54
|
+
if (err instanceof loader_1.RouteConfigError && err.problems.length > 0)
|
|
55
|
+
return err.problems.join("; ");
|
|
56
|
+
return err instanceof Error ? err.message : String(err);
|
|
57
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The capture step — records N invocations of the ORIGINAL call site's
|
|
3
|
+
* inputs/outputs LOCALLY (spec §9 Tier-2 verify). Two surfaces:
|
|
4
|
+
*
|
|
5
|
+
* - `captureOriginal(site, originalFn, inputs)` — invoke the original call
|
|
6
|
+
* (a black-box function the skill/harness wraps around the customer's real
|
|
7
|
+
* call) once per input and record the I/O into a fixture. Honest refusal is
|
|
8
|
+
* built in: a streaming or dynamically-assembled-prompt call site is REFUSED
|
|
9
|
+
* up front (those are T0/T1 territory), and an input the original throws on
|
|
10
|
+
* is recorded as a per-input refusal, never a lost sample.
|
|
11
|
+
* - `openShadowCaptureStore(dbPath)` — a durable local SQLite buffer for the
|
|
12
|
+
* same captures, the same posture as telemetry/capture.ts: a plain
|
|
13
|
+
* better-sqlite3 handle with record/readAll/close and NO network or upload
|
|
14
|
+
* accessor. Captures stay in the SDK process; nothing here uploads.
|
|
15
|
+
*
|
|
16
|
+
* The portable `ShadowCaptureFixture` (JSON) is the hand-off the CLI reads and
|
|
17
|
+
* the skill embeds context from — the only thing that leaves this module, and
|
|
18
|
+
* only to local disk.
|
|
19
|
+
*/
|
|
20
|
+
import type { CandidateSpec } from "../types/route";
|
|
21
|
+
import type { ShadowCapture, ShadowCaptureFixture } from "./types";
|
|
22
|
+
/** A call site being captured. `streaming`/`dynamicPrompt` are the skill's
|
|
23
|
+
* honest declarations that this site cannot be captured/replayed as a
|
|
24
|
+
* task-shaped bundle — refusing is the correct output. */
|
|
25
|
+
export interface OriginalCallSite {
|
|
26
|
+
readonly callSiteId: string;
|
|
27
|
+
/** provider+model of the original call — the route must pin exactly this. */
|
|
28
|
+
readonly original: CandidateSpec;
|
|
29
|
+
readonly streaming?: boolean;
|
|
30
|
+
readonly dynamicPrompt?: boolean;
|
|
31
|
+
readonly nondeterministicPaths?: readonly string[];
|
|
32
|
+
}
|
|
33
|
+
/** The original call, wrapped as a pure input→output-text function. */
|
|
34
|
+
export type OriginalCallFn = (input: Record<string, unknown>) => Promise<string>;
|
|
35
|
+
export interface CaptureOptions {
|
|
36
|
+
readonly now?: () => string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Capture N original invocations into a fixture. Streaming/dynamic sites are
|
|
40
|
+
* refused before any call; per-input failures become `uncapturable` refusals.
|
|
41
|
+
*/
|
|
42
|
+
export declare function captureOriginal(site: OriginalCallSite, originalFn: OriginalCallFn, inputs: readonly Record<string, unknown>[], opts?: CaptureOptions): Promise<ShadowCaptureFixture>;
|
|
43
|
+
export interface ShadowCaptureRecord {
|
|
44
|
+
readonly callSiteId: string;
|
|
45
|
+
readonly provider: string;
|
|
46
|
+
readonly model: string;
|
|
47
|
+
readonly input: Record<string, unknown>;
|
|
48
|
+
readonly output: string;
|
|
49
|
+
}
|
|
50
|
+
export interface ShadowCaptureStore {
|
|
51
|
+
record(row: ShadowCaptureRecord): void;
|
|
52
|
+
/** Captures for a call site, in insertion order (the replay input set). */
|
|
53
|
+
readAll(callSiteId: string): ShadowCapture[];
|
|
54
|
+
close(): void;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* A separate better-sqlite3 handle — deliberately NOT wired to any exporter or
|
|
58
|
+
* upload path. Shadow captures are original-call outputs used only to prove
|
|
59
|
+
* equivalence locally; nothing here ever leaves the machine (the SDK's
|
|
60
|
+
* export-isolation posture, extended to the verifier).
|
|
61
|
+
*/
|
|
62
|
+
export declare function openShadowCaptureStore(dbPath: string): ShadowCaptureStore;
|
|
63
|
+
export declare function serializeFixture(fixture: ShadowCaptureFixture): string;
|
|
64
|
+
/** Parse + shape-validate a fixture (errors teach — the CLI surfaces them). */
|
|
65
|
+
export declare function parseFixture(json: string): ShadowCaptureFixture;
|
|
66
|
+
//# sourceMappingURL=capture.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"capture.d.ts","sourceRoot":"","sources":["../../src/verify/capture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,KAAK,EAAkB,aAAa,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAEnF;;0DAE0D;AAC1D,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpD;AAED,uEAAuE;AACvE,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,gBAAgB,EACtB,UAAU,EAAE,cAAc,EAC1B,MAAM,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC1C,IAAI,GAAE,cAAmB,GACxB,OAAO,CAAC,oBAAoB,CAAC,CAwC/B;AAwBD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,GAAG,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACvC,2EAA2E;IAC3E,OAAO,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE,CAAC;IAC7C,KAAK,IAAI,IAAI,CAAC;CACf;AAmBD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB,CAuCzE;AAID,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CAEtE;AAOD,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,CA0B/D"}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The capture step — records N invocations of the ORIGINAL call site's
|
|
4
|
+
* inputs/outputs LOCALLY (spec §9 Tier-2 verify). Two surfaces:
|
|
5
|
+
*
|
|
6
|
+
* - `captureOriginal(site, originalFn, inputs)` — invoke the original call
|
|
7
|
+
* (a black-box function the skill/harness wraps around the customer's real
|
|
8
|
+
* call) once per input and record the I/O into a fixture. Honest refusal is
|
|
9
|
+
* built in: a streaming or dynamically-assembled-prompt call site is REFUSED
|
|
10
|
+
* up front (those are T0/T1 territory), and an input the original throws on
|
|
11
|
+
* is recorded as a per-input refusal, never a lost sample.
|
|
12
|
+
* - `openShadowCaptureStore(dbPath)` — a durable local SQLite buffer for the
|
|
13
|
+
* same captures, the same posture as telemetry/capture.ts: a plain
|
|
14
|
+
* better-sqlite3 handle with record/readAll/close and NO network or upload
|
|
15
|
+
* accessor. Captures stay in the SDK process; nothing here uploads.
|
|
16
|
+
*
|
|
17
|
+
* The portable `ShadowCaptureFixture` (JSON) is the hand-off the CLI reads and
|
|
18
|
+
* the skill embeds context from — the only thing that leaves this module, and
|
|
19
|
+
* only to local disk.
|
|
20
|
+
*/
|
|
21
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
22
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
23
|
+
};
|
|
24
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
exports.captureOriginal = captureOriginal;
|
|
26
|
+
exports.openShadowCaptureStore = openShadowCaptureStore;
|
|
27
|
+
exports.serializeFixture = serializeFixture;
|
|
28
|
+
exports.parseFixture = parseFixture;
|
|
29
|
+
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
30
|
+
const node_fs_1 = require("node:fs");
|
|
31
|
+
const node_path_1 = require("node:path");
|
|
32
|
+
/**
|
|
33
|
+
* Capture N original invocations into a fixture. Streaming/dynamic sites are
|
|
34
|
+
* refused before any call; per-input failures become `uncapturable` refusals.
|
|
35
|
+
*/
|
|
36
|
+
async function captureOriginal(site, originalFn, inputs, opts = {}) {
|
|
37
|
+
const capturedAt = (opts.now ?? (() => new Date().toISOString()))();
|
|
38
|
+
const nd = site.nondeterministicPaths;
|
|
39
|
+
const upfront = refuseUpFront(site);
|
|
40
|
+
if (upfront !== null) {
|
|
41
|
+
return {
|
|
42
|
+
callSiteId: site.callSiteId,
|
|
43
|
+
original: site.original,
|
|
44
|
+
capturedAt,
|
|
45
|
+
samples: [],
|
|
46
|
+
refusals: [upfront],
|
|
47
|
+
...(nd !== undefined ? { nondeterministicPaths: nd } : {}),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
const samples = [];
|
|
51
|
+
const refusals = [];
|
|
52
|
+
for (const input of inputs) {
|
|
53
|
+
try {
|
|
54
|
+
const output = await originalFn(input);
|
|
55
|
+
samples.push({ input, output });
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
refusals.push({
|
|
59
|
+
reason: "uncapturable",
|
|
60
|
+
detail: `input ${JSON.stringify(input)} could not be captured: ` +
|
|
61
|
+
(err instanceof Error ? err.message : String(err)),
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
callSiteId: site.callSiteId,
|
|
67
|
+
original: site.original,
|
|
68
|
+
capturedAt,
|
|
69
|
+
samples,
|
|
70
|
+
...(refusals.length > 0 ? { refusals } : {}),
|
|
71
|
+
...(nd !== undefined ? { nondeterministicPaths: nd } : {}),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function refuseUpFront(site) {
|
|
75
|
+
if (site.streaming === true) {
|
|
76
|
+
return {
|
|
77
|
+
reason: "streaming",
|
|
78
|
+
detail: `call site ${site.callSiteId} streams tokens — a streaming call has no single ` +
|
|
79
|
+
"terminal output to replay against; per the batch-first deferral this is T0/T1 territory",
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
if (site.dynamicPrompt === true) {
|
|
83
|
+
return {
|
|
84
|
+
reason: "dynamic-prompt",
|
|
85
|
+
detail: `call site ${site.callSiteId} assembles its prompt dynamically — it does not map to a ` +
|
|
86
|
+
"stable template + variables, so it cannot be replayed through a task route (T0/T1 territory)",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
const SHADOW_SQL = `
|
|
92
|
+
CREATE TABLE IF NOT EXISTS shadow_captures (
|
|
93
|
+
call_site_id TEXT NOT NULL,
|
|
94
|
+
seq INTEGER NOT NULL,
|
|
95
|
+
provider TEXT NOT NULL,
|
|
96
|
+
model TEXT NOT NULL,
|
|
97
|
+
input_json TEXT NOT NULL,
|
|
98
|
+
output_text TEXT NOT NULL,
|
|
99
|
+
PRIMARY KEY (call_site_id, seq)
|
|
100
|
+
);
|
|
101
|
+
`;
|
|
102
|
+
/**
|
|
103
|
+
* A separate better-sqlite3 handle — deliberately NOT wired to any exporter or
|
|
104
|
+
* upload path. Shadow captures are original-call outputs used only to prove
|
|
105
|
+
* equivalence locally; nothing here ever leaves the machine (the SDK's
|
|
106
|
+
* export-isolation posture, extended to the verifier).
|
|
107
|
+
*/
|
|
108
|
+
function openShadowCaptureStore(dbPath) {
|
|
109
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(dbPath), { recursive: true });
|
|
110
|
+
const db = new better_sqlite3_1.default(dbPath);
|
|
111
|
+
db.pragma("journal_mode = WAL");
|
|
112
|
+
db.exec(SHADOW_SQL);
|
|
113
|
+
const nextSeq = db.prepare("SELECT COALESCE(MAX(seq), -1) + 1 AS n FROM shadow_captures WHERE call_site_id = ?");
|
|
114
|
+
const insert = db.prepare(`
|
|
115
|
+
INSERT INTO shadow_captures (call_site_id, seq, provider, model, input_json, output_text)
|
|
116
|
+
VALUES (@call_site_id, @seq, @provider, @model, @input_json, @output_text)
|
|
117
|
+
`);
|
|
118
|
+
const selectAll = db.prepare("SELECT input_json, output_text FROM shadow_captures WHERE call_site_id = ? ORDER BY seq ASC");
|
|
119
|
+
return {
|
|
120
|
+
record(row) {
|
|
121
|
+
const seq = nextSeq.get(row.callSiteId).n;
|
|
122
|
+
insert.run({
|
|
123
|
+
call_site_id: row.callSiteId,
|
|
124
|
+
seq,
|
|
125
|
+
provider: row.provider,
|
|
126
|
+
model: row.model,
|
|
127
|
+
input_json: JSON.stringify(row.input),
|
|
128
|
+
output_text: row.output,
|
|
129
|
+
});
|
|
130
|
+
},
|
|
131
|
+
readAll(callSiteId) {
|
|
132
|
+
return selectAll.all(callSiteId).map((r) => ({
|
|
133
|
+
input: JSON.parse(r.input_json),
|
|
134
|
+
output: r.output_text,
|
|
135
|
+
}));
|
|
136
|
+
},
|
|
137
|
+
close() {
|
|
138
|
+
db.close();
|
|
139
|
+
},
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
// ── Portable fixture JSON (the CLI hand-off) ─────────────────────────────────
|
|
143
|
+
function serializeFixture(fixture) {
|
|
144
|
+
return `${JSON.stringify(fixture, null, 2)}\n`;
|
|
145
|
+
}
|
|
146
|
+
/** A non-null, non-array object (the shape a JSON `{...}` decodes to). */
|
|
147
|
+
function isPlainObject(v) {
|
|
148
|
+
return v !== null && typeof v === "object" && !Array.isArray(v);
|
|
149
|
+
}
|
|
150
|
+
/** Parse + shape-validate a fixture (errors teach — the CLI surfaces them). */
|
|
151
|
+
function parseFixture(json) {
|
|
152
|
+
let raw;
|
|
153
|
+
try {
|
|
154
|
+
raw = JSON.parse(json);
|
|
155
|
+
}
|
|
156
|
+
catch (err) {
|
|
157
|
+
throw new Error(`capture fixture is not valid JSON: ${err.message}`);
|
|
158
|
+
}
|
|
159
|
+
if (raw === null || typeof raw !== "object") {
|
|
160
|
+
throw new Error("capture fixture must be a JSON object");
|
|
161
|
+
}
|
|
162
|
+
const o = raw;
|
|
163
|
+
if (typeof o.callSiteId !== "string")
|
|
164
|
+
throw new Error('capture fixture is missing "callSiteId"');
|
|
165
|
+
const original = o.original;
|
|
166
|
+
if (!isPlainObject(original) || typeof original.provider !== "string" || typeof original.model !== "string") {
|
|
167
|
+
throw new Error('capture fixture is missing a valid "original" { provider, model }');
|
|
168
|
+
}
|
|
169
|
+
if (!Array.isArray(o.samples))
|
|
170
|
+
throw new Error('capture fixture is missing a "samples" array');
|
|
171
|
+
for (const [i, s] of o.samples.entries()) {
|
|
172
|
+
const sample = s;
|
|
173
|
+
// `input` must be a plain object of template variables — null and arrays
|
|
174
|
+
// pass a bare `typeof === "object"` check but break render/replay downstream.
|
|
175
|
+
if (!isPlainObject(sample) || typeof sample.output !== "string" || !isPlainObject(sample.input)) {
|
|
176
|
+
throw new Error(`capture fixture sample ${i} must be { input: object, output: string }`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return raw;
|
|
180
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `modelrig verify-swap` runner — reads a local capture fixture, replays it
|
|
3
|
+
* through the drafted same-model route, and writes BOTH report forms: the
|
|
4
|
+
* machine-readable JSON (the skill parses it) and the human-readable Markdown
|
|
5
|
+
* (the Tier-2 PR embeds it). Lives here (not cli.ts) for the LOC cap (P1),
|
|
6
|
+
* beside the report renderer it drives — same split as bakeoff/report.ts.
|
|
7
|
+
*
|
|
8
|
+
* Exit code is the gate the skill reads: 0 = equivalent / equivalent-modulo,
|
|
9
|
+
* 1 = DIVERGENT (do not merge), 2 = REFUSED (not a Tier-2 replacement — a
|
|
10
|
+
* correct outcome that needs a human's T0/T1 rethink, not a merge).
|
|
11
|
+
*/
|
|
12
|
+
import type { RigConfig } from "../config";
|
|
13
|
+
export interface VerifySwapCliOptions {
|
|
14
|
+
readonly route: string;
|
|
15
|
+
readonly fixturePath: string;
|
|
16
|
+
readonly envelopeUsd?: number;
|
|
17
|
+
/** Print the machine-readable JSON to stdout instead of the Markdown. */
|
|
18
|
+
readonly asJson?: boolean;
|
|
19
|
+
/** Also write the Markdown report to this path (the PR body source). */
|
|
20
|
+
readonly outPath?: string;
|
|
21
|
+
}
|
|
22
|
+
export declare function runVerifySwapCli(config: RigConfig, opts: VerifySwapCliOptions): Promise<number>;
|
|
23
|
+
//# sourceMappingURL=cli-run.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-run.d.ts","sourceRoot":"","sources":["../../src/verify/cli-run.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAM3C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,yEAAyE;IACzE,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,wEAAwE;IACxE,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAQD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,MAAM,CAAC,CA+BjB"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* `modelrig verify-swap` runner — reads a local capture fixture, replays it
|
|
4
|
+
* through the drafted same-model route, and writes BOTH report forms: the
|
|
5
|
+
* machine-readable JSON (the skill parses it) and the human-readable Markdown
|
|
6
|
+
* (the Tier-2 PR embeds it). Lives here (not cli.ts) for the LOC cap (P1),
|
|
7
|
+
* beside the report renderer it drives — same split as bakeoff/report.ts.
|
|
8
|
+
*
|
|
9
|
+
* Exit code is the gate the skill reads: 0 = equivalent / equivalent-modulo,
|
|
10
|
+
* 1 = DIVERGENT (do not merge), 2 = REFUSED (not a Tier-2 replacement — a
|
|
11
|
+
* correct outcome that needs a human's T0/T1 rethink, not a merge).
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.runVerifySwapCli = runVerifySwapCli;
|
|
15
|
+
const node_fs_1 = require("node:fs");
|
|
16
|
+
const node_path_1 = require("node:path");
|
|
17
|
+
const index_1 = require("../index");
|
|
18
|
+
const capture_1 = require("./capture");
|
|
19
|
+
const report_1 = require("./report");
|
|
20
|
+
function exitFor(report) {
|
|
21
|
+
if (report.overall === "divergent")
|
|
22
|
+
return 1;
|
|
23
|
+
if (report.overall === "refused")
|
|
24
|
+
return 2;
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
27
|
+
async function runVerifySwapCli(config, opts) {
|
|
28
|
+
const fixture = (0, capture_1.parseFixture)((0, node_fs_1.readFileSync)(opts.fixturePath, "utf8"));
|
|
29
|
+
const rig = (0, index_1.createRig)(config);
|
|
30
|
+
try {
|
|
31
|
+
const report = await rig.verifySwap(opts.route, fixture, {
|
|
32
|
+
...(opts.envelopeUsd !== undefined ? { envelopeUsd: opts.envelopeUsd } : {}),
|
|
33
|
+
});
|
|
34
|
+
// Persist both forms next to the telemetry db (captures never travel; the
|
|
35
|
+
// report is aggregate + local).
|
|
36
|
+
const outDir = (0, node_path_1.join)((0, node_path_1.dirname)(config.telemetryDbPath), "verify");
|
|
37
|
+
(0, node_fs_1.mkdirSync)(outDir, { recursive: true });
|
|
38
|
+
const stem = (0, node_path_1.join)(outDir, `${report.route}@${report.routeVersion}`);
|
|
39
|
+
(0, node_fs_1.writeFileSync)(`${stem}.json`, `${JSON.stringify(report, null, 2)}\n`);
|
|
40
|
+
const markdown = (0, report_1.renderReportMarkdown)(report);
|
|
41
|
+
(0, node_fs_1.writeFileSync)(`${stem}.md`, markdown);
|
|
42
|
+
if (opts.outPath !== undefined) {
|
|
43
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(opts.outPath), { recursive: true });
|
|
44
|
+
(0, node_fs_1.writeFileSync)(opts.outPath, markdown);
|
|
45
|
+
}
|
|
46
|
+
if (opts.asJson)
|
|
47
|
+
console.log(JSON.stringify(report, null, 2));
|
|
48
|
+
else
|
|
49
|
+
console.log(markdown);
|
|
50
|
+
console.log(`report → ${stem}.json · ${stem}.md` + (opts.outPath !== undefined ? ` · ${opts.outPath}` : ""));
|
|
51
|
+
return exitFor(report);
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
// One-shot CLI: await the settled drain so the exporter's final batch lands.
|
|
55
|
+
await rig.close();
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The diff engine — the honest verdict vocabulary that makes the verifier's
|
|
3
|
+
* report trustworthy. Pure (no adapters, no run loop): it compares an
|
|
4
|
+
* original output against a routed one and returns one of three verdicts.
|
|
5
|
+
*
|
|
6
|
+
* Design decision (flagged for architect review): the routed output arrives
|
|
7
|
+
* as PARSED JSON — `run()` only returns after the output validated against the
|
|
8
|
+
* route schema (or it throws), so a routed sample is guaranteed schema-
|
|
9
|
+
* conformant and JSON. We therefore compare STRUCTURE, not raw bytes. For JSON,
|
|
10
|
+
* key order and whitespace are not behavior, so "byte-identical where
|
|
11
|
+
* deterministic" is operationalized as CANONICAL identity (sorted-key
|
|
12
|
+
* serialization). Structured-field equality — with declared nondeterministic
|
|
13
|
+
* fields excluded — is the fallback, exactly as the spec's §6 evaluation
|
|
14
|
+
* contract prescribes.
|
|
15
|
+
*/
|
|
16
|
+
import type { SampleDiff } from "./types";
|
|
17
|
+
/** Deterministic JSON serialization: object keys sorted recursively, arrays
|
|
18
|
+
* left in order (order is semantic). The operative notion of "byte-identical"
|
|
19
|
+
* for JSON outputs. */
|
|
20
|
+
export declare function canonicalJson(value: unknown): string;
|
|
21
|
+
export interface Divergence {
|
|
22
|
+
/** JSON-pointer path to the first differing node. */
|
|
23
|
+
readonly path: string;
|
|
24
|
+
readonly originalValue: unknown;
|
|
25
|
+
readonly routedValue: unknown;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* First point at which `original` and `routed` structurally differ, in a
|
|
29
|
+
* stable depth-first order, or null when they are deep-equal after masking the
|
|
30
|
+
* declared nondeterministic paths. Object keys are visited in sorted order so
|
|
31
|
+
* the "first" differing field is deterministic across runs.
|
|
32
|
+
*/
|
|
33
|
+
export declare function firstDivergence(original: unknown, routed: unknown, nondeterministicPaths: readonly string[], path?: string): Divergence | null;
|
|
34
|
+
/**
|
|
35
|
+
* Diff an original output (raw text) against a routed output (already parsed by
|
|
36
|
+
* `run()`), returning the verdict. `routedParsed` is guaranteed schema-
|
|
37
|
+
* conformant JSON by the run loop; `originalText` is whatever the original call
|
|
38
|
+
* site emitted.
|
|
39
|
+
*
|
|
40
|
+
* - byte / canonical identity ⇒ `equivalent`
|
|
41
|
+
* - structurally equal after masking declared nd-paths ⇒
|
|
42
|
+
* `equivalent-modulo-nondeterminism`
|
|
43
|
+
* - a real structured field differs ⇒ `divergent` (first field named)
|
|
44
|
+
* - the original is not JSON ⇒ `divergent` (a free-text call site — this
|
|
45
|
+
* isn't route-shaped work; T0/T1 territory, honestly flagged not forced)
|
|
46
|
+
*/
|
|
47
|
+
export declare function diffOutputs(originalText: string, routedParsed: unknown, routedText: string, nondeterministicPaths: readonly string[]): SampleDiff;
|
|
48
|
+
//# sourceMappingURL=diff.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../src/verify/diff.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C;;uBAEuB;AACvB,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEpD;AAcD,MAAM,WAAW,UAAU;IACzB,qDAAqD;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;CAC/B;AAQD;;;;;GAKG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,OAAO,EACjB,MAAM,EAAE,OAAO,EACf,qBAAqB,EAAE,SAAS,MAAM,EAAE,EACxC,IAAI,SAAK,GACR,UAAU,GAAG,IAAI,CAuCnB;AAwBD;;;;;;;;;;;;GAYG;AACH,wBAAgB,WAAW,CACzB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,OAAO,EACrB,UAAU,EAAE,MAAM,EAClB,qBAAqB,EAAE,SAAS,MAAM,EAAE,GACvC,UAAU,CAyCZ"}
|