modelrig 0.10.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +96 -0
  2. package/dist/cli.d.ts.map +1 -1
  3. package/dist/cli.js +1 -0
  4. package/dist/config.d.ts +18 -0
  5. package/dist/config.d.ts.map +1 -1
  6. package/dist/config.js +23 -3
  7. package/dist/doctor-cli.d.ts.map +1 -1
  8. package/dist/doctor-cli.js +2 -1
  9. package/dist/doctor-preflight.d.ts +14 -2
  10. package/dist/doctor-preflight.d.ts.map +1 -1
  11. package/dist/doctor-preflight.js +65 -5
  12. package/dist/feedback/coverage.d.ts.map +1 -1
  13. package/dist/feedback/coverage.js +9 -2
  14. package/dist/index.d.ts +7 -1
  15. package/dist/index.d.ts.map +1 -1
  16. package/dist/index.js +18 -8
  17. package/dist/init/init-cli.d.ts.map +1 -1
  18. package/dist/init/init-cli.js +21 -4
  19. package/dist/init/observe-cli.js +1 -1
  20. package/dist/init/observe.d.ts +7 -1
  21. package/dist/init/observe.d.ts.map +1 -1
  22. package/dist/init/observe.js +74 -22
  23. package/dist/init/starter.d.ts +36 -0
  24. package/dist/init/starter.d.ts.map +1 -0
  25. package/dist/init/starter.js +85 -0
  26. package/dist/run/raw-finalize.d.ts +49 -3
  27. package/dist/run/raw-finalize.d.ts.map +1 -1
  28. package/dist/run/raw-finalize.js +31 -14
  29. package/dist/run/raw-search-cost.d.ts +3 -2
  30. package/dist/run/raw-search-cost.d.ts.map +1 -1
  31. package/dist/run/raw-search-cost.js +1 -0
  32. package/dist/run/rawonly.d.ts +29 -5
  33. package/dist/run/rawonly.d.ts.map +1 -1
  34. package/dist/run/rawonly.js +54 -19
  35. package/dist/run/record-call.d.ts +104 -0
  36. package/dist/run/record-call.d.ts.map +1 -0
  37. package/dist/run/record-call.js +277 -0
  38. package/dist/telemetry/attempt-meta.d.ts +11 -0
  39. package/dist/telemetry/attempt-meta.d.ts.map +1 -1
  40. package/dist/telemetry/attempt-meta.js +3 -0
  41. package/dist/telemetry/lanes.d.ts +31 -0
  42. package/dist/telemetry/lanes.d.ts.map +1 -0
  43. package/dist/telemetry/lanes.js +41 -0
  44. package/dist/telemetry/org-identity.d.ts +5 -2
  45. package/dist/telemetry/org-identity.d.ts.map +1 -1
  46. package/dist/telemetry/org-identity.js +10 -8
  47. package/docs/agent-skill/SKILL.md +109 -10
  48. package/docs/how-it-fits.md +1 -1
  49. package/docs/instrumentation.md +152 -40
  50. package/docs/llms.txt +92 -9
  51. package/docs/quickstart.md +127 -15
  52. package/package.json +1 -1
@@ -11,7 +11,9 @@
11
11
  */
12
12
  Object.defineProperty(exports, "__esModule", { value: true });
13
13
  exports.rawTimeoutFixHint = rawTimeoutFixHint;
14
+ exports.recordRawRow = recordRawRow;
14
15
  exports.prepareRawStep = prepareRawStep;
16
+ exports.emitRawStep = emitRawStep;
15
17
  exports.finalizeRaw = finalizeRaw;
16
18
  const node_crypto_1 = require("node:crypto");
17
19
  const cost_1 = require("./cost");
@@ -55,7 +57,7 @@ function recordRawRow(telemetry, p) {
55
57
  provider: p.provider,
56
58
  model: p.model,
57
59
  requestedTier: p.requestedTier,
58
- servedTier: p.usage?.servedTier ?? "standard",
60
+ servedTier: p.usage?.servedTier ?? p.servedTier ?? "standard",
59
61
  tokensIn: p.usage?.tokensIn ?? 0,
60
62
  tokensOut: p.usage?.tokensOut ?? 0,
61
63
  tokensCached: p.usage?.tokensCached ?? 0,
@@ -106,6 +108,22 @@ function prepareRawStep(deps, input) {
106
108
  const stepKey = (0, run_context_1.resolveStepKey)(input.stepKey, input.tags?.step, () => runCtx.nextStepKey(rawModelId));
107
109
  return { runCtx, stepKey, startedAtIso: new Date().toISOString() };
108
110
  }
111
+ /** G1: mirror one lane-B inference (raw, or an OBS-1 recorded call) into a
112
+ * ground-truth step — parity with the routed lane's record-attempt. No-op when
113
+ * no run context was captured at entry. Fail-open is handled inside
114
+ * `emitGroundTruthStep` (buffer errors are swallowed). */
115
+ function emitRawStep(step, task, inferenceId, failureClass) {
116
+ if (step === null)
117
+ return;
118
+ step.runCtx.emitGroundTruthStep({
119
+ stepKey: step.stepKey,
120
+ task,
121
+ inferenceId,
122
+ startedAt: step.startedAtIso,
123
+ endedAt: new Date().toISOString(),
124
+ failureClass,
125
+ });
126
+ }
109
127
  /**
110
128
  * Turn an `AdapterResult` into a RunResult, recording exactly one telemetry row.
111
129
  * Shared by the buffered and streaming paths so cost/telemetry are byte-identical.
@@ -132,7 +150,17 @@ function finalizeRaw(deps, input, result, latencyMs, step) {
132
150
  // caller set none — spread FIRST so a caller value wins, and omitted entirely
133
151
  // outside a run context so the row stays byte-identical (parity with RC-1).
134
152
  const stepTag = step !== null ? { step: step.stepKey } : {};
135
- const tags = { ...stepTag, ...(0, run_context_1.withRunIdTag)(input.tags ?? {}, step?.runCtx ?? null), lane: "raw" };
153
+ // L6 (OBS-1 follow-up, architect ruling Q2): the raw lane dispatches ONLY on
154
+ // the caller's own provider key (`input.apiKey` — there is no managed-key path
155
+ // here; the hosted gateway resolves a BYOK vault key fail-closed), so every raw
156
+ // row is BYOK by construction. `key_ownership: "byok"` is SDK-owned and stamped
157
+ // LAST beside `lane`, so an in-process runRaw row can never settle as managed.
158
+ const tags = {
159
+ ...stepTag,
160
+ ...(0, run_context_1.withRunIdTag)(input.tags ?? {}, step?.runCtx ?? null),
161
+ key_ownership: "byok",
162
+ lane: "raw",
163
+ };
136
164
  // WI-1 (spec §2, §6.3 lane parity): the attempt envelope, built by the SAME
137
165
  // producer + rendered-request canonicalization the routed lane uses. R2: a
138
166
  // declared `responseSchema` joins the input_hash exactly as a route schema
@@ -154,18 +182,7 @@ function finalizeRaw(deps, input, result, latencyMs, step) {
154
182
  // G1: mirror this raw inference into a ground-truth step (parity with the
155
183
  // routed lane's record-attempt). task = tags.step ?? provider/model (spec F1).
156
184
  // Fail-open is handled inside emitGroundTruthStep (buffer errors are swallowed).
157
- const emitStep = (inferenceId, failureClass) => {
158
- if (step === null)
159
- return;
160
- step.runCtx.emitGroundTruthStep({
161
- stepKey: step.stepKey,
162
- task: input.tags?.step ?? rawModelId,
163
- inferenceId,
164
- startedAt: step.startedAtIso,
165
- endedAt: new Date().toISOString(),
166
- failureClass,
167
- });
168
- };
185
+ const emitStep = (inferenceId, failureClass) => emitRawStep(step, input.tags?.step ?? rawModelId, inferenceId, failureClass);
169
186
  if (!result.ok) {
170
187
  let costUsd = 0;
171
188
  let pricingMissing = false;
@@ -25,8 +25,9 @@ import type { TelemetryStore } from "../types/telemetry";
25
25
  * provider is the real dispatch provider so the surcharge is attributed to it. */
26
26
  export declare const NATIVE_WEB_SEARCH_MODEL = "web_search";
27
27
  export interface RawSearchCostArgs {
28
- /** The real dispatch provider (openai / grok) — the row's `provider`. */
29
- readonly provider: ProviderId;
28
+ /** The real dispatch provider (openai / grok) — the row's `provider`. An
29
+ * OBS-1 recorded call may name a provider ModelRig has no rate for (⇒ $0). */
30
+ readonly provider: ProviderId | string;
30
31
  /** The `provider/model` handle the caller used — the row's `route`. */
31
32
  readonly rawModelId: string;
32
33
  /** The effective tags already stamped on the primary raw row (lane:"raw",
@@ -1 +1 @@
1
- {"version":3,"file":"raw-search-cost.d.ts","sourceRoot":"","sources":["../../src/run/raw-search-cost.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAgB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAIvE;;kFAEkF;AAClF,eAAO,MAAM,uBAAuB,eAAe,CAAC;AAEpD,MAAM,WAAW,iBAAiB;IAChC,yEAAyE;IACzE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,uEAAuE;IACvE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;kEAC8D;IAC9D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,iBAAiB,GAAG,IAAI,CAyC/F"}
1
+ {"version":3,"file":"raw-search-cost.d.ts","sourceRoot":"","sources":["../../src/run/raw-search-cost.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAgB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAIvE;;kFAEkF;AAClF,eAAO,MAAM,uBAAuB,eAAe,CAAC;AAEpD,MAAM,WAAW,iBAAiB;IAChC;kFAC8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAAC;IACvC,uEAAuE;IACvE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;kEAC8D;IAC9D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,iBAAiB,GAAG,IAAI,CA0C/F"}
@@ -36,6 +36,7 @@ exports.NATIVE_WEB_SEARCH_MODEL = "web_search";
36
36
  */
37
37
  function recordRawSearchCostRow(telemetry, args) {
38
38
  const searchCalls = Math.max(1, args.searchCalls);
39
+ // An unknown provider has no WEB_SEARCH_PRICING entry ⇒ webSearchPerCallUsd's `?.` yields 0.
39
40
  const costEstimateUsd = (0, web_search_pricing_1.webSearchPerCallUsd)(args.provider) * searchCalls;
40
41
  // U4: assembled by the ONE shared builder (./telemetry-row.ts). `rung:"search"`
41
42
  // and `attemptSeq: 2` (so it never collides with the primary raw row's
@@ -17,7 +17,7 @@
17
17
  * 700-LOC cap — `createRig` remains the ONE construction path (a separate
18
18
  * `createRawRig()` factory was considered and REJECTED, spec §2 R1).
19
19
  */
20
- import type { RigConfig } from "../config";
20
+ import { type RigConfig } from "../config";
21
21
  import { type LoadedRoutes } from "../routes/loader";
22
22
  import { RigFailureError } from "../types/failures";
23
23
  import type { Rig } from "../index";
@@ -27,7 +27,12 @@ export declare const RAW_ONLY_FIX_HINT = "this rig was constructed runRaw-only (
27
27
  /** Build the TERMINAL failure a route-dependent surface throws when the rig was
28
28
  * constructed runRaw-only. Reuses `invariant_violation` (already TERMINAL + in
29
29
  * the teaching table) — no new FailureClass (spec §3 rationale). */
30
- export declare function rawOnlyRouteFailure(surface: string): RigFailureError;
30
+ /** Why a rig is raw-only: constructed with `routesDir: null`, or fell back
31
+ * because the default routes directory is absent (SURF-1). */
32
+ export type RawOnlyReason = "routesDir-null" | "absent-default";
33
+ /** The fix hint when the absent-default fallback made the rig raw-only. */
34
+ export declare const ABSENT_DEFAULT_FIX_HINT = "create ./modelrig/routes with your route files (`npx modelrig init` drafts them), or set MODELRIG_ROUTES_DIR";
35
+ export declare function rawOnlyRouteFailure(surface: string, reason?: RawOnlyReason): RigFailureError;
31
36
  /** Narrow a config's `routesDir` to a string for a route-REQUIRED context (the
32
37
  * route-oriented CLI subcommands: bakeoff/watch/swap/init/observe/validate),
33
38
  * throwing the teaching failure if the rig was built runRaw-only.
@@ -35,6 +40,21 @@ export declare function rawOnlyRouteFailure(surface: string): RigFailureError;
35
40
  * env-built configs — it guards a hand-built `routesDir: null` reaching a
36
41
  * route-only command, and satisfies the `string | null` type at those seams. */
37
42
  export declare function requireRoutesDir(config: RigConfig, surface: string): string;
43
+ /** The warning createRig prints when it falls back to a raw-only rig because the
44
+ * DEFAULT routes directory does not exist (SURF-1, audit #1). One constant so
45
+ * the docs and the test quote the same words. */
46
+ export declare const ABSENT_DEFAULT_ROUTES_WARNING: string;
47
+ /**
48
+ * The routes directory createRig actually loads (SURF-1 ruling on audit #1):
49
+ * - `null` stays `null` (CE-8 runRaw-only, silent);
50
+ * - the UNCONFIGURED default (loadConfigFromEnv filled in `./modelrig/routes`;
51
+ * no MODELRIG_ROUTES_DIR, no routesDir override, no `extraRoutesDir`) that
52
+ * does not exist on disk → warn once, return `null`, so
53
+ * a customer who only records calls never crashes on import;
54
+ * - anything explicitly configured is returned as-is, and a missing one still
55
+ * throws the teaching RouteConfigError from the loader.
56
+ */
57
+ export declare function effectiveRoutesDir(config: RigConfig): string | null;
38
58
  /** The empty route store for a runRaw-only rig: `list()` is `[]` (so
39
59
  * serveability, capture-detection, and the exported route mirror are all no-ops)
40
60
  * and `get()` teaches, though no public surface reaches it — every route method
@@ -48,13 +68,17 @@ export declare function loadRoutesOrReport(config: RigConfig, routesDir: string)
48
68
  * `routesDir === null` so every one of them throws the teaching failure rather
49
69
  * than dereferencing an empty store. Spread LAST over the real methods in
50
70
  * `createRig`, so later keys win. */
51
- type RouteMethods = Pick<Rig, "run" | "runStream" | "runBundle" | "runBundleStream" | "replay" | "bakeoff" | "bakeoffFromEvalCases" | "verifySwap">;
52
- /** The throwing overrides for a runRaw-only rig (spec §2 R1). Each names the
71
+ type RouteMethods = Pick<Rig, "run" | "runStream" | "runBundle" | "runBundleStream" | "replay" | "bakeoff" | "verifySwap">;
72
+ /** The throwing overrides for a runRaw-only rig (spec §2 R1).
73
+ * `bakeoffFromEvalCases` is deliberately NOT overridden (SURF-1, audit #12): it
74
+ * reads eval cases from the local store first, so with none promoted (or none
75
+ * rehydratable) it reports honestly without any routes dir; only a case that
76
+ * actually replays reaches `routes.get`, which teaches the same failure. Each names the
53
77
  * surface + the route/task the caller asked for, so the terminal error points at
54
78
  * exactly the call that needs a `routesDir`. Promise-returning surfaces REJECT
55
79
  * (matching the routed lane, where errors surface as rejections); the two
56
80
  * streaming surfaces return a handle synchronously, so they throw synchronously
57
81
  * — the caller never reaches an event to await. */
58
- export declare function rawOnlyRouteMethods(): RouteMethods;
82
+ export declare function rawOnlyRouteMethods(reason?: RawOnlyReason): RouteMethods;
59
83
  export {};
60
84
  //# sourceMappingURL=rawonly.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"rawonly.d.ts","sourceRoot":"","sources":["../../src/run/rawonly.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAyC,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAE5F,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAEpC;sEACsE;AACtE,eAAO,MAAM,iBAAiB,uGACwE,CAAC;AAEvG;;oEAEoE;AACpE,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe,CAMpE;AAsBD;;;;;gFAKgF;AAChF,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG3E;AAED;;;mDAGmD;AACnD,wBAAgB,eAAe,IAAI,YAAY,CAS9C;AAED;;0EAE0E;AAC1E,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,CASrF;AAED;;;qCAGqC;AACrC,KAAK,YAAY,GAAG,IAAI,CACtB,GAAG,EACD,KAAK,GACL,WAAW,GACX,WAAW,GACX,iBAAiB,GACjB,QAAQ,GACR,SAAS,GACT,sBAAsB,GACtB,YAAY,CACf,CAAC;AAEF;;;;;mDAKmD;AACnD,wBAAgB,mBAAmB,IAAI,YAAY,CAYlD"}
1
+ {"version":3,"file":"rawonly.d.ts","sourceRoot":"","sources":["../../src/run/rawonly.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EAAsB,KAAK,SAAS,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EAAyC,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAE5F,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAEpC;sEACsE;AACtE,eAAO,MAAM,iBAAiB,uGACwE,CAAC;AAEvG;;oEAEoE;AACpE;8DAC8D;AAC9D,MAAM,MAAM,aAAa,GAAG,gBAAgB,GAAG,gBAAgB,CAAC;AAEhE,2EAA2E;AAC3E,eAAO,MAAM,uBAAuB,iHAC4E,CAAC;AAEjH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,GAAE,aAAgC,GAAG,eAAe,CAS9G;AAsBD;;;;;gFAKgF;AAChF,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG3E;AAED;;iDAEiD;AACjD,eAAO,MAAM,6BAA6B,QAIW,CAAC;AAEtD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CAQnE;AAED;;;mDAGmD;AACnD,wBAAgB,eAAe,IAAI,YAAY,CAS9C;AAED;;0EAE0E;AAC1E,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,CASrF;AAED;;;qCAGqC;AACrC,KAAK,YAAY,GAAG,IAAI,CACtB,GAAG,EACD,KAAK,GACL,WAAW,GACX,WAAW,GACX,iBAAiB,GACjB,QAAQ,GACR,SAAS,GACT,YAAY,CACf,CAAC;AAEF;;;;;;;;;mDASmD;AACnD,wBAAgB,mBAAmB,CAAC,MAAM,GAAE,aAAgC,GAAG,YAAY,CAW1F"}
@@ -19,26 +19,30 @@
19
19
  * `createRawRig()` factory was considered and REJECTED, spec §2 R1).
20
20
  */
21
21
  Object.defineProperty(exports, "__esModule", { value: true });
22
- exports.RAW_ONLY_FIX_HINT = void 0;
22
+ exports.ABSENT_DEFAULT_ROUTES_WARNING = exports.ABSENT_DEFAULT_FIX_HINT = exports.RAW_ONLY_FIX_HINT = void 0;
23
23
  exports.rawOnlyRouteFailure = rawOnlyRouteFailure;
24
24
  exports.requireRoutesDir = requireRoutesDir;
25
+ exports.effectiveRoutesDir = effectiveRoutesDir;
25
26
  exports.emptyRouteStore = emptyRouteStore;
26
27
  exports.loadRoutesOrReport = loadRoutesOrReport;
27
28
  exports.rawOnlyRouteMethods = rawOnlyRouteMethods;
29
+ const node_fs_1 = require("node:fs");
30
+ const config_1 = require("../config");
28
31
  const loader_1 = require("../routes/loader");
29
32
  const load_error_report_1 = require("../telemetry/load-error-report");
30
33
  const failures_1 = require("../types/failures");
31
34
  /** The teaching remediation for a route-lane call on a runRaw-only rig (spec §2
32
35
  * R1). One constant so the failure and the docs stay in lock-step. */
33
36
  exports.RAW_ONLY_FIX_HINT = "this rig was constructed runRaw-only (routesDir: null); give createRig a routesDir to serve routes";
34
- /** Build the TERMINAL failure a route-dependent surface throws when the rig was
35
- * constructed runRaw-only. Reuses `invariant_violation` (already TERMINAL + in
36
- * the teaching table) — no new FailureClass (spec §3 rationale). */
37
- function rawOnlyRouteFailure(surface) {
37
+ /** The fix hint when the absent-default fallback made the rig raw-only. */
38
+ exports.ABSENT_DEFAULT_FIX_HINT = "create ./modelrig/routes with your route files (`npx modelrig init` drafts them), or set MODELRIG_ROUTES_DIR";
39
+ function rawOnlyRouteFailure(surface, reason = "routesDir-null") {
38
40
  return new failures_1.RigFailureError({
39
41
  class: "invariant_violation",
40
- message: `${surface} needs routes, but this rig was constructed runRaw-only (routesDir: null)`,
41
- fixHint: exports.RAW_ONLY_FIX_HINT,
42
+ message: reason === "absent-default"
43
+ ? `${surface} needs routes, but there is no routes directory at ./modelrig/routes, so this rig is raw-only`
44
+ : `${surface} needs routes, but this rig was constructed runRaw-only (routesDir: null)`,
45
+ fixHint: reason === "absent-default" ? exports.ABSENT_DEFAULT_FIX_HINT : exports.RAW_ONLY_FIX_HINT,
42
46
  });
43
47
  }
44
48
  /** A `RunStreamHandle` that fails the runRaw-only way: `result` rejects and the
@@ -46,8 +50,8 @@ function rawOnlyRouteFailure(surface) {
46
50
  * throws synchronously (a stream surface always hands back a handle, errors ride
47
51
  * `result`/`events`). A synchronous throw here would break the caller's
48
52
  * `const h = rig.runStream(...); await h.result` shape. */
49
- function rawOnlyStreamHandle(surface) {
50
- const err = rawOnlyRouteFailure(surface);
53
+ function rawOnlyStreamHandle(surface, reason) {
54
+ const err = rawOnlyRouteFailure(surface, reason);
51
55
  const result = Promise.reject(err);
52
56
  // Keep the rejection handled so an unconsumed handle never becomes an
53
57
  // unhandledRejection; the events iterator surfaces the same error.
@@ -71,6 +75,33 @@ function requireRoutesDir(config, surface) {
71
75
  throw rawOnlyRouteFailure(surface);
72
76
  return config.routesDir;
73
77
  }
78
+ /** The warning createRig prints when it falls back to a raw-only rig because the
79
+ * DEFAULT routes directory does not exist (SURF-1, audit #1). One constant so
80
+ * the docs and the test quote the same words. */
81
+ exports.ABSENT_DEFAULT_ROUTES_WARNING = "[modelrig] no routes directory at ./modelrig/routes, so this rig is raw-only: rig.runRaw, " +
82
+ "rig.runRawStream and rig.recordCall work, and rig.run / rig.runBundle need routes. " +
83
+ "Create routes with `npx modelrig init`, set MODELRIG_ROUTES_DIR, or pass `routesDir: null` " +
84
+ "to say raw-only on purpose and silence this line.";
85
+ /**
86
+ * The routes directory createRig actually loads (SURF-1 ruling on audit #1):
87
+ * - `null` stays `null` (CE-8 runRaw-only, silent);
88
+ * - the UNCONFIGURED default (loadConfigFromEnv filled in `./modelrig/routes`;
89
+ * no MODELRIG_ROUTES_DIR, no routesDir override, no `extraRoutesDir`) that
90
+ * does not exist on disk → warn once, return `null`, so
91
+ * a customer who only records calls never crashes on import;
92
+ * - anything explicitly configured is returned as-is, and a missing one still
93
+ * throws the teaching RouteConfigError from the loader.
94
+ */
95
+ function effectiveRoutesDir(config) {
96
+ const dir = config.routesDir;
97
+ if (dir === null)
98
+ return null;
99
+ if (config.extraRoutesDir === undefined && (0, config_1.isDefaultRoutesDir)(config) && !(0, node_fs_1.existsSync)(dir)) {
100
+ console.warn(exports.ABSENT_DEFAULT_ROUTES_WARNING);
101
+ return null;
102
+ }
103
+ return dir;
104
+ }
74
105
  /** The empty route store for a runRaw-only rig: `list()` is `[]` (so
75
106
  * serveability, capture-detection, and the exported route mirror are all no-ops)
76
107
  * and `get()` teaches, though no public surface reaches it — every route method
@@ -100,21 +131,25 @@ function loadRoutesOrReport(config, routesDir) {
100
131
  throw err;
101
132
  }
102
133
  }
103
- /** The throwing overrides for a runRaw-only rig (spec §2 R1). Each names the
134
+ /** The throwing overrides for a runRaw-only rig (spec §2 R1).
135
+ * `bakeoffFromEvalCases` is deliberately NOT overridden (SURF-1, audit #12): it
136
+ * reads eval cases from the local store first, so with none promoted (or none
137
+ * rehydratable) it reports honestly without any routes dir; only a case that
138
+ * actually replays reaches `routes.get`, which teaches the same failure. Each names the
104
139
  * surface + the route/task the caller asked for, so the terminal error points at
105
140
  * exactly the call that needs a `routesDir`. Promise-returning surfaces REJECT
106
141
  * (matching the routed lane, where errors surface as rejections); the two
107
142
  * streaming surfaces return a handle synchronously, so they throw synchronously
108
143
  * — the caller never reaches an event to await. */
109
- function rawOnlyRouteMethods() {
144
+ function rawOnlyRouteMethods(reason = "routesDir-null") {
145
+ const fail = (surface) => Promise.reject(rawOnlyRouteFailure(surface, reason));
110
146
  return {
111
- run: (route) => Promise.reject(rawOnlyRouteFailure(`run("${route}")`)),
112
- runStream: (route) => rawOnlyStreamHandle(`runStream("${route}")`),
113
- runBundle: (bundle) => Promise.reject(rawOnlyRouteFailure(`runBundle("${bundle.route}")`)),
114
- runBundleStream: (bundle) => rawOnlyStreamHandle(`runBundleStream("${bundle.route}")`),
115
- replay: (route) => Promise.reject(rawOnlyRouteFailure(`replay("${route}")`)),
116
- bakeoff: (route) => Promise.reject(rawOnlyRouteFailure(`bakeoff("${route}")`)),
117
- bakeoffFromEvalCases: (task) => Promise.reject(rawOnlyRouteFailure(`bakeoffFromEvalCases("${task}")`)),
118
- verifySwap: (route) => Promise.reject(rawOnlyRouteFailure(`verifySwap("${route}")`)),
147
+ run: (route) => fail(`run("${route}")`),
148
+ runStream: (route) => rawOnlyStreamHandle(`runStream("${route}")`, reason),
149
+ runBundle: (bundle) => fail(`runBundle("${bundle.route}")`),
150
+ runBundleStream: (bundle) => rawOnlyStreamHandle(`runBundleStream("${bundle.route}")`, reason),
151
+ replay: (route) => fail(`replay("${route}")`),
152
+ bakeoff: (route) => fail(`bakeoff("${route}")`),
153
+ verifySwap: (route) => fail(`verifySwap("${route}")`),
119
154
  };
120
155
  }
@@ -0,0 +1,104 @@
1
+ /**
2
+ * OBS-1 — `rig.recordCall()`: record a provider call ModelRig did NOT dispatch
3
+ * (acceptance ruling M12, Option A). The customer's own code made the call with
4
+ * its own key; it reports what the provider returned (tokens, tier, latency,
5
+ * failure) and ModelRig writes ONE telemetry row for it, exactly the shape the
6
+ * raw lane writes — same builder, same cost function, same step emission — so
7
+ * the call shows on /inferences, joins its run and step, and can be graded.
8
+ *
9
+ * WHAT IS DIFFERENT FROM THE RAW LANE (and nothing else is):
10
+ * - `tags.lane = "external"` (SDK-owned, stamped LAST — a caller tag cannot
11
+ * override it). Named `external`, not `direct`: ruling D-2 already uses
12
+ * "direct" for `destination`.
13
+ * - `tags.key_ownership = "byok"` (SDK-owned). Gate B-0 (usage-aggregation.ts):
14
+ * a BYOK seam must stamp the tag at that seam. A recorded call is always the
15
+ * customer's key — ModelRig bore no provider cost — so it can never settle as
16
+ * a managed charge.
17
+ * - `meta.provenance = "caller-reported"`: the numbers are the caller's, not a
18
+ * measurement. Nothing was validated (`validated` is never claimed).
19
+ *
20
+ * CONTRACT: synchronous (the row is written to the local store; the exporter
21
+ * ships it on its normal cadence) and NEVER throws into the caller. Bad input is
22
+ * warned with a teaching fix (P4) and returns `{ inferenceId: null, … }` with no
23
+ * row written.
24
+ *
25
+ * No route/registry/capability-gate dependency beyond the raw lane's own (cost
26
+ * + telemetry + run context) — the raw lane's no-hard-hot-path invariant holds.
27
+ */
28
+ import type { FailureClass } from "../types/failures";
29
+ import type { ProviderId, ServiceTier } from "../types/provider";
30
+ import type { RunTags } from "../types/route";
31
+ import type { RawRunDeps } from "./raw";
32
+ /** The lane tag a recorded call carries (SDK-owned) — the shared classifier's constant. */
33
+ export { EXTERNAL_LANE } from "../telemetry/lanes";
34
+ /** Where the warning's fix points (the "record them" section). */
35
+ export declare const RECORD_CALL_DOCS_URL = "https://modelrig.dev/not-captured.html";
36
+ /** Usage the caller's provider response reported. Same convention as every
37
+ * adapter (`AdapterUsage`): `tokensIn` is the TOTAL input and INCLUDES
38
+ * `tokensCached` and `tokensCacheWrite` as subsets. Token counts (and
39
+ * `latencyMs`/`ttfbMs`) are ROUNDED to the nearest integer and clamped
40
+ * (tokens ≤ Number.MAX_SAFE_INTEGER, milliseconds ≤ 2³¹−1) before recording. */
41
+ export interface RecordCallUsage {
42
+ readonly tokensIn: number;
43
+ readonly tokensOut: number;
44
+ readonly tokensCached?: number;
45
+ readonly tokensCacheWrite?: number;
46
+ readonly reasoningTokens?: number;
47
+ }
48
+ export interface RecordCallInput {
49
+ /** A ModelRig `ProviderId` is priced; any other non-empty provider id is
50
+ * recorded at $0 with `pricingMissing: true`. */
51
+ readonly provider: ProviderId | (string & {});
52
+ readonly model: string;
53
+ readonly usage: RecordCallUsage;
54
+ /** The tier that SERVED the call — the row is priced at it (flex is cheaper),
55
+ * exactly as the raw lane prices. Default `"standard"`. */
56
+ readonly servedTier?: ServiceTier;
57
+ /** The tier your code asked for. Default: `servedTier`. */
58
+ readonly requestedTier?: ServiceTier;
59
+ readonly latencyMs: number;
60
+ readonly ttfbMs?: number;
61
+ /** A failed attempt that the provider still billed is still a row. */
62
+ readonly failureClass?: FailureClass | null;
63
+ /** Native web-search tool calls billed on this call — writes the separate
64
+ * `rung:"search"` cost row, as the raw lane does for native grounding. */
65
+ readonly webSearchCalls?: number;
66
+ /** The ground-truth step inside an active run (same rule as `runRaw`). */
67
+ readonly stepKey?: string;
68
+ readonly tags?: RunTags;
69
+ readonly providerRequestId?: string;
70
+ /** The provider's native finish reason (`STOP`, `end_turn`, `stop`, …) or a
71
+ * normalized one; mapped to the envelope enum. */
72
+ readonly finishReason?: string;
73
+ }
74
+ export interface RecordCallResult {
75
+ /** The recorded inference id (pass it to `rig.grade`), or null when the input
76
+ * was rejected and nothing was recorded. */
77
+ readonly inferenceId: string | null;
78
+ readonly callId: string | null;
79
+ /** The token row's list-price cost (a `webSearchCalls` surcharge rides its own
80
+ * row, as on the raw lane). 0 when unpriced or rejected. */
81
+ readonly costEstimateUsd: number;
82
+ readonly pricingMissing: boolean;
83
+ }
84
+ /** Server ingest's route cap (`modelrig-server` ingest.ts `isNonEmptyString(raw.route, 200)`). */
85
+ export declare const MAX_ROUTE_CHARS = 200;
86
+ /** Test-only: clear the once-per-rule warning latch. */
87
+ export declare function resetRecordCallWarningsForTest(): void;
88
+ /** The ceiling for latency/ttfb (Postgres `integer`, 2³¹−1 ms ≈ 24.8 days). */
89
+ export declare const MAX_MS = 2147483647;
90
+ /**
91
+ * L6 MUST-3 integer hygiene. Telemetry columns are integers, but a caller's
92
+ * `performance.now()` delta is fractional — so ROUND to the nearest integer
93
+ * (friendlier than rejecting) and CLAMP: latency/ttfb to ≤ 2³¹−1 ms, token
94
+ * counts to ≤ Number.MAX_SAFE_INTEGER. Applied before any check that compares
95
+ * token counts, so validation and the recorded row see the same numbers.
96
+ */
97
+ export declare function toCount(v: number, max: number): number;
98
+ /**
99
+ * Record one call the caller made itself. See the module header for the
100
+ * contract. `deps` is the rig's raw-lane deps (telemetry + unpriced rate +
101
+ * manifest) — the same object `runRaw` receives.
102
+ */
103
+ export declare function recordCall(deps: RawRunDeps, input: RecordCallInput): RecordCallResult;
104
+ //# sourceMappingURL=record-call.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"record-call.d.ts","sourceRoot":"","sources":["../../src/run/record-call.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAYH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAgB,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AAExC,2FAA2F;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,kEAAkE;AAClE,eAAO,MAAM,oBAAoB,2CAA2C,CAAC;AAE7E;;;;gFAIgF;AAChF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B;qDACiD;IACjD,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;+DAC2D;IAC3D,QAAQ,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC;IAClC,2DAA2D;IAC3D,QAAQ,CAAC,aAAa,CAAC,EAAE,WAAW,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,sEAAsE;IACtE,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5C;8EAC0E;IAC1E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;sDACkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B;gEAC4D;IAC5D,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAQD,kGAAkG;AAClG,eAAO,MAAM,eAAe,MAAM,CAAC;AAcnC,wDAAwD;AACxD,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AAMD,+EAA+E;AAC/E,eAAO,MAAM,MAAM,aAAgB,CAAC;AAEpC;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAEtD;AA6GD;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,GAAG,gBAAgB,CAarF"}