modelrig 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +41 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +46 -6
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +105 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +37 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +138 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +122 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The run loop's per-attempt telemetry recorder, extracted from run.ts (WS-A).
|
|
4
|
+
*
|
|
5
|
+
* run.ts sits at the 700-LOC cap; threading cache-write accounting through the
|
|
6
|
+
* row builder pushed it over, so the recorder — a self-contained mapping from
|
|
7
|
+
* an `AttemptTelemetry` to a persisted `TelemetryRow` plus the ground-truth
|
|
8
|
+
* step mirror — moves here as a factory over the loop's per-run state. Behavior
|
|
9
|
+
* is byte-identical to the former inline closure, save the additive cache
|
|
10
|
+
* fields. Keeping it separate also shrinks the run-loop conflict surface for the
|
|
11
|
+
* in-flight E4 / capture-write work (plan coordination note).
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.createRecordAttempt = createRecordAttempt;
|
|
15
|
+
const node_crypto_1 = require("node:crypto");
|
|
16
|
+
const failures_1 = require("../types/failures");
|
|
17
|
+
const cache_stamp_1 = require("./cache-stamp");
|
|
18
|
+
const cache_handle_1 = require("./cache-handle");
|
|
19
|
+
/** Build the run loop's `recordAttempt(attempt) => inferenceId` over this run's
|
|
20
|
+
* state. Records one inference row and, with an active run context, mirrors it
|
|
21
|
+
* as a ground-truth step (fail-open). */
|
|
22
|
+
function createRecordAttempt(deps) {
|
|
23
|
+
const { telemetry, route, ledger, effectiveTags, servedVariant, runCtx, autoStepKey } = deps;
|
|
24
|
+
const { cache, droppedCacheHandle } = deps;
|
|
25
|
+
return function recordAttempt(attempt) {
|
|
26
|
+
// Request-path cache substrate (WS-C): derived from the SAME pure gate the
|
|
27
|
+
// dispatcher stamps the request with, so `cache_key`/`prefix_fingerprint` on
|
|
28
|
+
// the row match what was sent for this candidate. No-op (both null) unless
|
|
29
|
+
// the route opted into `cache: auto` — pre-WS-C rows are byte-identical.
|
|
30
|
+
// PRIMARY-attempt only: a repair call (repairedBy set) re-asks with a
|
|
31
|
+
// different prompt (error feedback / repair-model framing), so its prefix
|
|
32
|
+
// does not share the route's cacheable identity — stamping it would file a
|
|
33
|
+
// cold, different-prefix call under the primary fingerprint and pollute
|
|
34
|
+
// WS-B's warm-set derivation and the §5.6 streak alert. Repair rows carry
|
|
35
|
+
// neither key nor fingerprint (matching buildRepairAdapterRequest, which
|
|
36
|
+
// sends no cache directive).
|
|
37
|
+
// CE-1 (spec §2.3): derive the customer key from the SAME gate as the request
|
|
38
|
+
// builder — a matching-provider, non-suppressed handle WINS over the WS-C
|
|
39
|
+
// generated stamp. `prefix_fingerprint` is unchanged, so origin stays
|
|
40
|
+
// derivable with NO new column (customer key != fingerprint ⇒ customer;
|
|
41
|
+
// key == fingerprint ⇒ generated — cacheKeyProvenance). Both derivations
|
|
42
|
+
// skip repair rows symmetrically (their prefix differs from the route's).
|
|
43
|
+
const isPrimary = attempt.repairedBy === undefined;
|
|
44
|
+
const cacheStamp = isPrimary ? (0, cache_stamp_1.cacheStampForCandidate)(route, attempt.candidate) : null;
|
|
45
|
+
// The customer key marks rows whose REQUEST carried the handle. A terminal
|
|
46
|
+
// failure (budget_exhausted / invariant_violation) is recorded by failRun
|
|
47
|
+
// BEFORE any dispatch — the wire never carried the handle — so those rows
|
|
48
|
+
// must not claim a customer key. Terminal classes come only from failRun;
|
|
49
|
+
// adapters never return them. (A grounding-search failure is the one
|
|
50
|
+
// non-terminal pre-dispatch path, but a customer cachedContents handle and
|
|
51
|
+
// the inject rung are mutually exclusive, so it is unreachable in practice.)
|
|
52
|
+
const preDispatch = attempt.failure !== null && failures_1.TERMINAL_CLASSES.has(attempt.failure.class);
|
|
53
|
+
const dropped = droppedCacheHandle.has(`${attempt.candidate.provider} ${attempt.candidate.model}`);
|
|
54
|
+
const customer = isPrimary && !preDispatch ? (0, cache_handle_1.customerCacheHandle)(cache, attempt.candidate, dropped) : null;
|
|
55
|
+
const cacheKey = customer?.key ?? cacheStamp?.cacheKey ?? null;
|
|
56
|
+
const row = {
|
|
57
|
+
id: (0, node_crypto_1.randomUUID)(),
|
|
58
|
+
ts: new Date().toISOString(),
|
|
59
|
+
route: route.route,
|
|
60
|
+
routeVersion: route.version,
|
|
61
|
+
provider: attempt.candidate.provider,
|
|
62
|
+
model: attempt.candidate.model,
|
|
63
|
+
requestedTier: route.policy.tier,
|
|
64
|
+
servedTier: attempt.servedTier,
|
|
65
|
+
tokensIn: attempt.tokensIn,
|
|
66
|
+
tokensOut: attempt.tokensOut,
|
|
67
|
+
tokensCached: attempt.tokensCached,
|
|
68
|
+
// Cache-write accounting (WS-A) + request-path substrate (WS-C).
|
|
69
|
+
tokensCacheWrite: attempt.tokensCacheWrite ?? 0,
|
|
70
|
+
tokensCacheWrite1h: attempt.tokensCacheWrite1h ?? 0,
|
|
71
|
+
cacheKey,
|
|
72
|
+
prefixFingerprint: cacheStamp?.prefixFingerprint ?? null,
|
|
73
|
+
costEstimateUsd: attempt.costUsd,
|
|
74
|
+
latencyMs: attempt.latencyMs,
|
|
75
|
+
ttfbMs: attempt.ttfbMs ?? null,
|
|
76
|
+
failureClass: attempt.failure?.class ?? null,
|
|
77
|
+
attemptsJson: JSON.stringify(ledger.snapshot()),
|
|
78
|
+
tagsJson: JSON.stringify(effectiveTags),
|
|
79
|
+
fingerprintCell: route.fingerprintCell,
|
|
80
|
+
servedVariant,
|
|
81
|
+
repairedBy: attempt.repairedBy ?? null,
|
|
82
|
+
pricingMissing: attempt.pricingMissing ?? false,
|
|
83
|
+
};
|
|
84
|
+
telemetry.record(row);
|
|
85
|
+
// Mirror this inference into a ground-truth step attempt (fail-open — the
|
|
86
|
+
// context swallows buffer errors; the run loop is never affected).
|
|
87
|
+
if (runCtx !== null && autoStepKey !== null) {
|
|
88
|
+
runCtx.emitGroundTruthStep({
|
|
89
|
+
stepKey: autoStepKey,
|
|
90
|
+
task: route.route,
|
|
91
|
+
inferenceId: row.id,
|
|
92
|
+
startedAt: row.ts,
|
|
93
|
+
endedAt: new Date().toISOString(),
|
|
94
|
+
failureClass: row.failureClass,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
return row.id;
|
|
98
|
+
};
|
|
99
|
+
}
|
package/dist/run/resolve.d.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* route.require, ordered by prefer:[cost] via the pricing snapshot; on a
|
|
6
6
|
* retryable failure the run loop asks for the next candidate.
|
|
7
7
|
*/
|
|
8
|
-
import type { RouteBundle } from "../types/route";
|
|
8
|
+
import type { RouteBundle, RunOptions } from "../types/route";
|
|
9
9
|
import type { CandidateRef, CapabilityFlag, ProviderId } from "../types/provider";
|
|
10
10
|
/** Static capability stub — replaced by the probed registry in Phase 2. */
|
|
11
11
|
export type CapabilityLookup = (provider: ProviderId, model: string) => ReadonlySet<CapabilityFlag>;
|
|
@@ -13,7 +13,80 @@ export interface CandidateIterator {
|
|
|
13
13
|
/** Next eligible candidate, or null when the set is exhausted. */
|
|
14
14
|
next(): CandidateRef | null;
|
|
15
15
|
}
|
|
16
|
-
|
|
16
|
+
/**
|
|
17
|
+
* Request-time candidate filters (G-2) — optional, applied by resolveCandidates
|
|
18
|
+
* BEFORE the prefer:[cost] sort. Distinct from route.require (capability + ZDR):
|
|
19
|
+
* these are per-run narrowings a caller passes on RunOptions. `maxPriceUsdPerMTok`
|
|
20
|
+
* reuses the same rate prefer:[cost] sorts on, so filter and sort agree.
|
|
21
|
+
*/
|
|
22
|
+
export interface CandidateFilter {
|
|
23
|
+
readonly maxPriceUsdPerMTok?: number;
|
|
24
|
+
readonly only?: readonly ProviderId[];
|
|
25
|
+
readonly ignore?: readonly ProviderId[];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* SHADOW-MODE warmth-aware ordering (cache-aware-routing spec §5.5, WS-B step 8;
|
|
29
|
+
* [Briefing] "auto only after validation"). Opt-in and NON-ACTUATING: when
|
|
30
|
+
* supplied, `resolveCandidates` computes the ordering that ranking by
|
|
31
|
+
* cache-ADJUSTED effective cost WOULD produce, hands the chosen-vs-recommended
|
|
32
|
+
* comparison to `log`, and then returns the UNCHANGED list-cost order. Absent ⇒
|
|
33
|
+
* the resolver is byte-for-byte its pre-WS-B self. No behavior flip ships this
|
|
34
|
+
* mission — the shadow log is reviewed before any warmth ordering activates
|
|
35
|
+
* (post-deploy gate ledger).
|
|
36
|
+
*/
|
|
37
|
+
export interface WarmthShadowOptions {
|
|
38
|
+
/** Cache-adjusted effective cost for a candidate (USD/MTok), or null when
|
|
39
|
+
* unknown — computed by the caller from the derived ledger + effective-rate
|
|
40
|
+
* (kept out of the resolver so this file stays free of pricing/cache logic). */
|
|
41
|
+
readonly effectiveCostUsdPerMTok: (provider: ProviderId, model: string) => number | null;
|
|
42
|
+
/** Sink for the shadow comparison — a logger/telemetry hook. */
|
|
43
|
+
readonly log: (shadow: WarmthShadowLog) => void;
|
|
44
|
+
}
|
|
45
|
+
/** One shadow observation: the order the resolver ACTUALLY returned (list-cost)
|
|
46
|
+
* vs the order warmth-aware ranking would recommend, and whether they diverge. */
|
|
47
|
+
export interface WarmthShadowLog {
|
|
48
|
+
readonly route: string;
|
|
49
|
+
readonly chosen: readonly {
|
|
50
|
+
readonly provider: ProviderId;
|
|
51
|
+
readonly model: string;
|
|
52
|
+
}[];
|
|
53
|
+
readonly recommended: readonly {
|
|
54
|
+
readonly provider: ProviderId;
|
|
55
|
+
readonly model: string;
|
|
56
|
+
}[];
|
|
57
|
+
readonly diverged: boolean;
|
|
58
|
+
}
|
|
59
|
+
export declare function resolveCandidates(route: RouteBundle, capabilities: CapabilityLookup, costUsdPerMTok: (provider: ProviderId, model: string) => number | null, searchAvailable?: boolean, filter?: CandidateFilter, shadowWarmth?: WarmthShadowOptions): CandidateIterator;
|
|
60
|
+
/**
|
|
61
|
+
* Per-run requirement opt-ins (G-3). A caller can demand zero-retention routing
|
|
62
|
+
* for a single run without editing the route bundle: `zero_retention` is unioned
|
|
63
|
+
* into the effective `require`, so the existing satisfiesRequirements filter, the
|
|
64
|
+
* no-eligible-candidate message, and the fail-closed empty-set path all apply
|
|
65
|
+
* unchanged. Additive only — a run opts INTO a stricter requirement, never OUT of
|
|
66
|
+
* a route's declared one (opting out would be a governance downgrade).
|
|
67
|
+
*/
|
|
68
|
+
export declare function applyRunRequirements(route: RouteBundle, zeroRetention: boolean): RouteBundle;
|
|
69
|
+
/** Map a run's RunOptions to a CandidateFilter (G-2), or undefined when no knob
|
|
70
|
+
* is set — so the resolver takes the byte-identical pre-G-2 path (back-compat). */
|
|
71
|
+
export declare function candidateFilterFromOptions(opts: RunOptions): CandidateFilter | undefined;
|
|
72
|
+
/** A human-readable suffix naming the binding request-time filters, for the
|
|
73
|
+
* existing no-eligible-candidate error (spec §4.3 — explainability, not a new
|
|
74
|
+
* error class). Empty when no filter is set. */
|
|
75
|
+
export declare function filterSummary(opts: RunOptions): string;
|
|
76
|
+
/**
|
|
77
|
+
* Mint a CandidateRef for the RAW LANE (http-gateway spec §4.2) — a
|
|
78
|
+
* `provider/model` the caller named explicitly, dispatched at zero margin on
|
|
79
|
+
* the caller's own BYOK key with NO route and NO verified pipeline.
|
|
80
|
+
*
|
|
81
|
+
* The candidate-set invariant guards against substituting a model outside a
|
|
82
|
+
* ROUTE's declared list; a route-free, explicitly-named passthrough is outside
|
|
83
|
+
* that invariant's scope by construction (there is no list to escape). This is
|
|
84
|
+
* the ONE sanctioned brand site for it, so the "no `as CandidateRef`" grep stays
|
|
85
|
+
* clean and the brand stays centralized here. The authorization that replaces
|
|
86
|
+
* route membership lives server-side: the raw lane dispatches only after
|
|
87
|
+
* `resolveProviderKeyForCall` returns a BYOK key fail-closed (spec §4.2).
|
|
88
|
+
*/
|
|
89
|
+
export declare function mintRawCandidate(provider: ProviderId, model: string): CandidateRef;
|
|
17
90
|
/**
|
|
18
91
|
* Mint a CandidateRef for a SPECIFIC declared candidate (the repair rung's
|
|
19
92
|
* designated repair model — loader-validated to be in the route's candidate
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolve.d.ts","sourceRoot":"","sources":["../../src/run/resolve.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"resolve.d.ts","sourceRoot":"","sources":["../../src/run/resolve.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAElF,2EAA2E;AAC3E,MAAM,MAAM,gBAAgB,GAAG,CAC7B,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,MAAM,KACV,WAAW,CAAC,cAAc,CAAC,CAAC;AAEjC,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,IAAI,IAAI,YAAY,GAAG,IAAI,CAAC;CAC7B;AA6CD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;CACzC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC;;oFAEgF;IAChF,QAAQ,CAAC,uBAAuB,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACzF,gEAAgE;IAChE,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;CACjD;AAED;kFACkF;AAClF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS;QAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtF,QAAQ,CAAC,WAAW,EAAE,SAAS;QAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC3F,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,WAAW,EAClB,YAAY,EAAE,gBAAgB,EAC9B,cAAc,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,EACtE,eAAe,UAAQ,EACvB,MAAM,CAAC,EAAE,eAAe,EACxB,YAAY,CAAC,EAAE,mBAAmB,GACjC,iBAAiB,CAkFnB;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,GAAG,WAAW,CAM5F;AAED;mFACmF;AACnF,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,GAAG,SAAS,CAUxF;AAED;;gDAEgD;AAChD,wBAAgB,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAMtD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,CAElF;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE;IAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC9D,YAAY,GAAG,IAAI,CAMrB"}
|
package/dist/run/resolve.js
CHANGED
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
10
|
exports.resolveCandidates = resolveCandidates;
|
|
11
|
+
exports.applyRunRequirements = applyRunRequirements;
|
|
12
|
+
exports.candidateFilterFromOptions = candidateFilterFromOptions;
|
|
13
|
+
exports.filterSummary = filterSummary;
|
|
14
|
+
exports.mintRawCandidate = mintRawCandidate;
|
|
11
15
|
exports.resolveDeclaredCandidate = resolveDeclaredCandidate;
|
|
12
16
|
function satisfiesRequirements(route, flags, searchAvailable) {
|
|
13
17
|
for (const requirement of route.require) {
|
|
@@ -38,15 +42,43 @@ function satisfiesRequirements(route, flags, searchAvailable) {
|
|
|
38
42
|
return false;
|
|
39
43
|
break;
|
|
40
44
|
case "zero_retention":
|
|
41
|
-
//
|
|
42
|
-
//
|
|
45
|
+
// G-3: opt-in data-governance filter. Route only to endpoints the
|
|
46
|
+
// credential-scoped registry facts designate zero-retention (claims.ts
|
|
47
|
+
// → facts.ts materializes the flag under the managed/any scope). A hard
|
|
48
|
+
// capability filter exactly like trace_visible; the empty-set case falls
|
|
49
|
+
// through to the existing no-eligible-candidate path (fail-closed — never
|
|
50
|
+
// a silent non-ZDR dispatch). The claim ceiling: "route only to
|
|
51
|
+
// zero-retention–designated endpoints," never "verified non-retention."
|
|
52
|
+
if (!flags.has("zero_retention"))
|
|
53
|
+
return false;
|
|
43
54
|
break;
|
|
44
55
|
}
|
|
45
56
|
}
|
|
46
57
|
return true;
|
|
47
58
|
}
|
|
48
|
-
function resolveCandidates(route, capabilities, costUsdPerMTok, searchAvailable = false) {
|
|
49
|
-
|
|
59
|
+
function resolveCandidates(route, capabilities, costUsdPerMTok, searchAvailable = false, filter, shadowWarmth) {
|
|
60
|
+
// Filter order (spec §4.1): provider only → provider ignore → capability+ZDR
|
|
61
|
+
// → maxPrice → prefer:[cost]. When `filter` is absent every step is a no-op,
|
|
62
|
+
// so the path is byte-for-byte the pre-G-2 behavior (back-compat).
|
|
63
|
+
let pool = route.candidates;
|
|
64
|
+
if (filter?.only && filter.only.length > 0) {
|
|
65
|
+
const only = new Set(filter.only);
|
|
66
|
+
pool = pool.filter((c) => only.has(c.provider));
|
|
67
|
+
}
|
|
68
|
+
if (filter?.ignore && filter.ignore.length > 0) {
|
|
69
|
+
const ignore = new Set(filter.ignore);
|
|
70
|
+
pool = pool.filter((c) => !ignore.has(c.provider));
|
|
71
|
+
}
|
|
72
|
+
let eligible = pool.filter((c) => satisfiesRequirements(route, capabilities(c.provider, c.model), searchAvailable));
|
|
73
|
+
if (filter?.maxPriceUsdPerMTok != null) {
|
|
74
|
+
const cap = filter.maxPriceUsdPerMTok;
|
|
75
|
+
// Unknown cost + a ceiling set → DROP (fail-closed: we can't prove it is
|
|
76
|
+
// under the cap; same stance as the envelope on unpriced models).
|
|
77
|
+
eligible = eligible.filter((c) => {
|
|
78
|
+
const usd = costUsdPerMTok(c.provider, c.model);
|
|
79
|
+
return usd !== null && usd <= cap;
|
|
80
|
+
});
|
|
81
|
+
}
|
|
50
82
|
// prefer: [cost] — ascending, unknown cost sorts last, stable otherwise.
|
|
51
83
|
// "latency" is advisory in Phase 1 (no probe data): declared order stands.
|
|
52
84
|
const ordered = [...eligible];
|
|
@@ -56,6 +88,37 @@ function resolveCandidates(route, capabilities, costUsdPerMTok, searchAvailable
|
|
|
56
88
|
return usd === null ? Number.POSITIVE_INFINITY : usd;
|
|
57
89
|
};
|
|
58
90
|
ordered.sort((a, b) => cost(a) - cost(b));
|
|
91
|
+
// SHADOW MODE (spec §5.5, step 8): compute what ordering by cache-ADJUSTED
|
|
92
|
+
// effective cost would recommend, log chosen-vs-recommended, and DO NOTHING
|
|
93
|
+
// ELSE. `ordered` (the list-cost order) is what we return — warmth never
|
|
94
|
+
// flips a route this mission. A no-op when `shadowWarmth` is absent, so the
|
|
95
|
+
// returned order is byte-identical to the pre-WS-B resolver.
|
|
96
|
+
//
|
|
97
|
+
// WRAPPED so shadow-mode is STRICTLY non-interfering: this is the actuation
|
|
98
|
+
// path (the only CandidateRef minter). A throwing warmth/log callback is a
|
|
99
|
+
// shadow-observation bug and must NEVER take down live candidate resolution
|
|
100
|
+
// — swallow it. `ordered` is already fixed above; nothing here can change it.
|
|
101
|
+
if (shadowWarmth !== undefined) {
|
|
102
|
+
try {
|
|
103
|
+
const effCost = (c) => {
|
|
104
|
+
const usd = shadowWarmth.effectiveCostUsdPerMTok(c.provider, c.model);
|
|
105
|
+
return usd === null ? Number.POSITIVE_INFINITY : usd;
|
|
106
|
+
};
|
|
107
|
+
const recommended = [...eligible].sort((a, b) => effCost(a) - effCost(b));
|
|
108
|
+
const same = recommended.length === ordered.length &&
|
|
109
|
+
recommended.every((c, i) => c.provider === ordered[i].provider && c.model === ordered[i].model);
|
|
110
|
+
shadowWarmth.log({
|
|
111
|
+
route: route.route,
|
|
112
|
+
chosen: ordered.map((c) => ({ provider: c.provider, model: c.model })),
|
|
113
|
+
recommended: recommended.map((c) => ({ provider: c.provider, model: c.model })),
|
|
114
|
+
diverged: !same,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
// Shadow observation failed — intentionally ignored. Routing proceeds on
|
|
119
|
+
// the list-cost order exactly as if no warmth had been supplied.
|
|
120
|
+
}
|
|
121
|
+
}
|
|
59
122
|
}
|
|
60
123
|
// THE brand site — the only place in the codebase a CandidateRef may be
|
|
61
124
|
// minted. The cast below is the single sanctioned exception to the
|
|
@@ -68,6 +131,65 @@ function resolveCandidates(route, capabilities, costUsdPerMTok, searchAvailable
|
|
|
68
131
|
},
|
|
69
132
|
};
|
|
70
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Per-run requirement opt-ins (G-3). A caller can demand zero-retention routing
|
|
136
|
+
* for a single run without editing the route bundle: `zero_retention` is unioned
|
|
137
|
+
* into the effective `require`, so the existing satisfiesRequirements filter, the
|
|
138
|
+
* no-eligible-candidate message, and the fail-closed empty-set path all apply
|
|
139
|
+
* unchanged. Additive only — a run opts INTO a stricter requirement, never OUT of
|
|
140
|
+
* a route's declared one (opting out would be a governance downgrade).
|
|
141
|
+
*/
|
|
142
|
+
function applyRunRequirements(route, zeroRetention) {
|
|
143
|
+
if (!zeroRetention || route.require.includes("zero_retention"))
|
|
144
|
+
return route;
|
|
145
|
+
return Object.freeze({
|
|
146
|
+
...route,
|
|
147
|
+
require: Object.freeze([...route.require, "zero_retention"]),
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
/** Map a run's RunOptions to a CandidateFilter (G-2), or undefined when no knob
|
|
151
|
+
* is set — so the resolver takes the byte-identical pre-G-2 path (back-compat). */
|
|
152
|
+
function candidateFilterFromOptions(opts) {
|
|
153
|
+
const only = opts.providers?.only;
|
|
154
|
+
const ignore = opts.providers?.ignore;
|
|
155
|
+
const max = opts.maxPriceUsdPerMTok;
|
|
156
|
+
if (max == null && (only?.length ?? 0) === 0 && (ignore?.length ?? 0) === 0)
|
|
157
|
+
return undefined;
|
|
158
|
+
return {
|
|
159
|
+
...(max != null ? { maxPriceUsdPerMTok: max } : {}),
|
|
160
|
+
...(only ? { only } : {}),
|
|
161
|
+
...(ignore ? { ignore } : {}),
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
/** A human-readable suffix naming the binding request-time filters, for the
|
|
165
|
+
* existing no-eligible-candidate error (spec §4.3 — explainability, not a new
|
|
166
|
+
* error class). Empty when no filter is set. */
|
|
167
|
+
function filterSummary(opts) {
|
|
168
|
+
const parts = [];
|
|
169
|
+
if (opts.maxPriceUsdPerMTok != null)
|
|
170
|
+
parts.push(`maxPrice ≤ $${opts.maxPriceUsdPerMTok}/MTok`);
|
|
171
|
+
if (opts.providers?.only?.length)
|
|
172
|
+
parts.push(`only=[${opts.providers.only.join(", ")}]`);
|
|
173
|
+
if (opts.providers?.ignore?.length)
|
|
174
|
+
parts.push(`ignore=[${opts.providers.ignore.join(", ")}]`);
|
|
175
|
+
return parts.length > 0 ? ` and request filters {${parts.join("; ")}}` : "";
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Mint a CandidateRef for the RAW LANE (http-gateway spec §4.2) — a
|
|
179
|
+
* `provider/model` the caller named explicitly, dispatched at zero margin on
|
|
180
|
+
* the caller's own BYOK key with NO route and NO verified pipeline.
|
|
181
|
+
*
|
|
182
|
+
* The candidate-set invariant guards against substituting a model outside a
|
|
183
|
+
* ROUTE's declared list; a route-free, explicitly-named passthrough is outside
|
|
184
|
+
* that invariant's scope by construction (there is no list to escape). This is
|
|
185
|
+
* the ONE sanctioned brand site for it, so the "no `as CandidateRef`" grep stays
|
|
186
|
+
* clean and the brand stays centralized here. The authorization that replaces
|
|
187
|
+
* route membership lives server-side: the raw lane dispatches only after
|
|
188
|
+
* `resolveProviderKeyForCall` returns a BYOK key fail-closed (spec §4.2).
|
|
189
|
+
*/
|
|
190
|
+
function mintRawCandidate(provider, model) {
|
|
191
|
+
return Object.freeze({ provider, model });
|
|
192
|
+
}
|
|
71
193
|
/**
|
|
72
194
|
* Mint a CandidateRef for a SPECIFIC declared candidate (the repair rung's
|
|
73
195
|
* designated repair model — loader-validated to be in the route's candidate
|
package/dist/run/run.d.ts
CHANGED
|
@@ -14,6 +14,8 @@ import type { TelemetryStore } from "../types/telemetry";
|
|
|
14
14
|
import type { CaptureStore } from "../telemetry/capture";
|
|
15
15
|
import type { CapabilityLookup } from "./resolve";
|
|
16
16
|
import type { SearchProvider } from "./grounding";
|
|
17
|
+
import type { StreamDeltaSink } from "./dispatch";
|
|
18
|
+
import type { TaskSettingsResolver } from "../settings/task-settings";
|
|
17
19
|
export interface RunDeps {
|
|
18
20
|
readonly routes: LoadedRoutes;
|
|
19
21
|
readonly adapters: ReadonlyMap<string, ProviderAdapter>;
|
|
@@ -36,6 +38,18 @@ export interface RunDeps {
|
|
|
36
38
|
readonly search?: SearchProvider | null;
|
|
37
39
|
/** Injectable backoff sleep — tests pass a no-op. */
|
|
38
40
|
readonly sleep?: (ms: number) => Promise<void>;
|
|
41
|
+
/** Streaming side-channel (G-1/G-5 Phase A). When set AND the serving
|
|
42
|
+
* adapter can stream, the primary attempt forwards raw text deltas here as
|
|
43
|
+
* they arrive; the buffered terminal result still drives validation/telemetry
|
|
44
|
+
* unchanged. `rig.runStream()` wires this; `rig.run()` leaves it undefined. */
|
|
45
|
+
readonly onStreamDelta?: StreamDeltaSink;
|
|
46
|
+
/** Control-plane task-settings resolver (settings-plane §1.5) — present only
|
|
47
|
+
* in api mode. The capture-WRITE gate consults it: `effectiveCapture` lets a
|
|
48
|
+
* control-plane flip WIN over the YAML `capture:` flag (a console flip turns
|
|
49
|
+
* capture on/off with no redeploy), and the task's scrub setting governs
|
|
50
|
+
* content before a capture row is written. null/omitted = local-only (the
|
|
51
|
+
* YAML flag governs, no per-task scrub policy). */
|
|
52
|
+
readonly settings?: TaskSettingsResolver | null;
|
|
39
53
|
}
|
|
40
54
|
export declare function run(deps: RunDeps, routeName: string, opts: RunOptions): Promise<RunResult>;
|
|
41
55
|
//# sourceMappingURL=run.d.ts.map
|
package/dist/run/run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/run/run.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/run/run.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,mBAAmB,CAAC;AAGvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAsC,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChG,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAQlD,OAAO,KAAK,EAAE,cAAc,EAAoC,MAAM,aAAa,CAAC;AASpF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAIlD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC;IACxC,+EAA+E;IAC/E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;;gDAE4C;IAC5C,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC;2EACuE;IACvE,QAAQ,CAAC,QAAQ,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACxC;;0BAEsB;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IACxC,qDAAqD;IACrD,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;mFAG+E;IAC/E,QAAQ,CAAC,aAAa,CAAC,EAAE,eAAe,CAAC;IACzC;;;;;uDAKmD;IACnD,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACjD;AAsED,wBAAsB,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAoiBhG"}
|
package/dist/run/run.js
CHANGED
|
Binary file
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming surface on the run path — G-1/G-5 Phase A (http-gateway spec §4.3).
|
|
3
|
+
*
|
|
4
|
+
* `runStream` wraps the existing `run()` loop with a delta side-channel: as the
|
|
5
|
+
* serving adapter streams, raw text tokens are emitted as `delta` events; when
|
|
6
|
+
* the run resolves, its terminal `RunResult` is emitted as the `final` event.
|
|
7
|
+
* Crucially the SAME pipeline (resolve → dispatch → validate → repair →
|
|
8
|
+
* telemetry) drives both streamed and buffered runs — streaming is purely a
|
|
9
|
+
* side-channel, so validation, repair, ZDR (G-3) and routing controls (G-2)
|
|
10
|
+
* all apply unchanged. `rig.run()` is left exactly as it was.
|
|
11
|
+
*
|
|
12
|
+
* The structured-route caveat (spec §4.3): streamed `delta` bytes are the
|
|
13
|
+
* model's RAW pre-validation output. The `final` event is authoritative — it
|
|
14
|
+
* carries the schema-validated (and, if repair fired, repaired) object plus
|
|
15
|
+
* `meta.validated` / `meta.repairedBy`. Callers wanting validated-only output
|
|
16
|
+
* pass `buffered: true` (no deltas, just the final) — the SDK mirror of the
|
|
17
|
+
* endpoint's `x-modelrig-stream: buffered`.
|
|
18
|
+
*
|
|
19
|
+
* ModelRig never executes a tool (spec §4.4); a tool-call turn surfaces on the
|
|
20
|
+
* final event's `meta.toolCalls` exactly as it does through `run()`.
|
|
21
|
+
*/
|
|
22
|
+
import type { RunDeps } from "./run";
|
|
23
|
+
import type { RunOptions, RunResult } from "../types/route";
|
|
24
|
+
export type RunStreamEvent = {
|
|
25
|
+
readonly type: "delta";
|
|
26
|
+
readonly text: string;
|
|
27
|
+
} | {
|
|
28
|
+
readonly type: "final";
|
|
29
|
+
readonly result: RunResult;
|
|
30
|
+
};
|
|
31
|
+
export interface RunStreamOptions extends RunOptions {
|
|
32
|
+
/** Buffered mode (spec §4.3): suppress token deltas and emit only the
|
|
33
|
+
* terminal (validated) result. Default false = stream-then-final. */
|
|
34
|
+
readonly buffered?: boolean;
|
|
35
|
+
}
|
|
36
|
+
export interface RunStreamHandle {
|
|
37
|
+
/** Async-iterable of `delta` events then a single `final`. Re-throws the
|
|
38
|
+
* run's error if it failed (after any deltas already emitted). */
|
|
39
|
+
readonly events: AsyncIterable<RunStreamEvent>;
|
|
40
|
+
/** The terminal RunResult, or a rejection with the run's typed failure.
|
|
41
|
+
* Resolves with the same object carried by the `final` event. */
|
|
42
|
+
readonly result: Promise<RunResult>;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Drive a route with token streaming. Returns an async-iterable of events plus
|
|
46
|
+
* a `result` promise for callers that only want the terminal value.
|
|
47
|
+
*/
|
|
48
|
+
export declare function runStream(deps: RunDeps, routeName: string, opts: RunStreamOptions): RunStreamHandle;
|
|
49
|
+
//# sourceMappingURL=stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/run/stream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AACrC,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE5D,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAA;CAAE,CAAC;AAE3D,MAAM,WAAW,gBAAiB,SAAQ,UAAU;IAClD;yEACqE;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B;sEACkE;IAClE,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IAC/C;qEACiE;IACjE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;CACrC;AAED;;;GAGG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,gBAAgB,GACrB,eAAe,CA6DjB"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Streaming surface on the run path — G-1/G-5 Phase A (http-gateway spec §4.3).
|
|
4
|
+
*
|
|
5
|
+
* `runStream` wraps the existing `run()` loop with a delta side-channel: as the
|
|
6
|
+
* serving adapter streams, raw text tokens are emitted as `delta` events; when
|
|
7
|
+
* the run resolves, its terminal `RunResult` is emitted as the `final` event.
|
|
8
|
+
* Crucially the SAME pipeline (resolve → dispatch → validate → repair →
|
|
9
|
+
* telemetry) drives both streamed and buffered runs — streaming is purely a
|
|
10
|
+
* side-channel, so validation, repair, ZDR (G-3) and routing controls (G-2)
|
|
11
|
+
* all apply unchanged. `rig.run()` is left exactly as it was.
|
|
12
|
+
*
|
|
13
|
+
* The structured-route caveat (spec §4.3): streamed `delta` bytes are the
|
|
14
|
+
* model's RAW pre-validation output. The `final` event is authoritative — it
|
|
15
|
+
* carries the schema-validated (and, if repair fired, repaired) object plus
|
|
16
|
+
* `meta.validated` / `meta.repairedBy`. Callers wanting validated-only output
|
|
17
|
+
* pass `buffered: true` (no deltas, just the final) — the SDK mirror of the
|
|
18
|
+
* endpoint's `x-modelrig-stream: buffered`.
|
|
19
|
+
*
|
|
20
|
+
* ModelRig never executes a tool (spec §4.4); a tool-call turn surfaces on the
|
|
21
|
+
* final event's `meta.toolCalls` exactly as it does through `run()`.
|
|
22
|
+
*/
|
|
23
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
24
|
+
exports.runStream = runStream;
|
|
25
|
+
const run_1 = require("./run");
|
|
26
|
+
/**
|
|
27
|
+
* Drive a route with token streaming. Returns an async-iterable of events plus
|
|
28
|
+
* a `result` promise for callers that only want the terminal value.
|
|
29
|
+
*/
|
|
30
|
+
function runStream(deps, routeName, opts) {
|
|
31
|
+
const queue = [];
|
|
32
|
+
let wake = null;
|
|
33
|
+
let done = false;
|
|
34
|
+
let failure = null;
|
|
35
|
+
const emit = (ev) => {
|
|
36
|
+
queue.push(ev);
|
|
37
|
+
if (wake !== null) {
|
|
38
|
+
wake();
|
|
39
|
+
wake = null;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
const finish = () => {
|
|
43
|
+
done = true;
|
|
44
|
+
if (wake !== null) {
|
|
45
|
+
wake();
|
|
46
|
+
wake = null;
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
// Buffered mode omits the delta sink entirely — the loop then dispatches via
|
|
50
|
+
// the adapter's buffered call() (no true streaming), exactly as before.
|
|
51
|
+
const streamDeps = opts.buffered === true
|
|
52
|
+
? deps
|
|
53
|
+
: { ...deps, onStreamDelta: ({ text }) => emit({ type: "delta", text }) };
|
|
54
|
+
const result = (0, run_1.run)(streamDeps, routeName, opts)
|
|
55
|
+
.then((r) => {
|
|
56
|
+
emit({ type: "final", result: r });
|
|
57
|
+
return r;
|
|
58
|
+
})
|
|
59
|
+
.catch((e) => {
|
|
60
|
+
failure = e;
|
|
61
|
+
throw e;
|
|
62
|
+
})
|
|
63
|
+
.finally(finish);
|
|
64
|
+
// A caller may consume only `events`; keep the promise from becoming an
|
|
65
|
+
// unhandled rejection (the error still surfaces through the iterator).
|
|
66
|
+
result.catch(() => undefined);
|
|
67
|
+
const events = {
|
|
68
|
+
async *[Symbol.asyncIterator]() {
|
|
69
|
+
for (;;) {
|
|
70
|
+
while (queue.length > 0) {
|
|
71
|
+
const ev = queue.shift();
|
|
72
|
+
if (ev !== undefined)
|
|
73
|
+
yield ev;
|
|
74
|
+
}
|
|
75
|
+
if (done) {
|
|
76
|
+
if (failure !== null)
|
|
77
|
+
throw failure;
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
await new Promise((res) => {
|
|
81
|
+
wake = res;
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
return { events, result };
|
|
87
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/run/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/run/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAGlD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;CAC1B;AAwBD,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CA6BlF"}
|
package/dist/run/validate.js
CHANGED
|
@@ -4,15 +4,14 @@
|
|
|
4
4
|
* Phase 1 checks schema conformance; value-accuracy gates arrive with the
|
|
5
5
|
* bake-off in Phase 3.
|
|
6
6
|
*/
|
|
7
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
8
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
9
|
-
};
|
|
10
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
8
|
exports.validateOutput = validateOutput;
|
|
12
|
-
const ajv_1 =
|
|
13
|
-
// Mirrors the production probe harness
|
|
14
|
-
//
|
|
15
|
-
|
|
9
|
+
const ajv_1 = require("./ajv");
|
|
10
|
+
// Mirrors the production probe harness (packages/modelrig-probes/src/harness.ts):
|
|
11
|
+
// an Ajv-2020 instance with the draft-07 meta added for back-compat — the two
|
|
12
|
+
// must enforce identically, or a fixture would score differently than it serves.
|
|
13
|
+
// Schemas trip ajv strict-mode warnings by design (strict:false).
|
|
14
|
+
const ajv = (0, ajv_1.createModelrigAjv)();
|
|
16
15
|
const MAX_ERRORS = 12;
|
|
17
16
|
/** Compiled validators cached per (route, version) — recompiling a 300-line
|
|
18
17
|
* schema per attempt is measurable. */
|