modelrig 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1 -0
- package/dist/config.d.ts +18 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +23 -3
- package/dist/doctor-cli.d.ts.map +1 -1
- package/dist/doctor-cli.js +2 -1
- package/dist/doctor-preflight.d.ts +14 -2
- package/dist/doctor-preflight.d.ts.map +1 -1
- package/dist/doctor-preflight.js +65 -5
- package/dist/feedback/coverage.d.ts.map +1 -1
- package/dist/feedback/coverage.js +9 -2
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -8
- package/dist/init/init-cli.d.ts.map +1 -1
- package/dist/init/init-cli.js +21 -4
- package/dist/init/observe-cli.js +1 -1
- package/dist/init/observe.d.ts +7 -1
- package/dist/init/observe.d.ts.map +1 -1
- package/dist/init/observe.js +74 -22
- package/dist/init/starter.d.ts +36 -0
- package/dist/init/starter.d.ts.map +1 -0
- package/dist/init/starter.js +85 -0
- package/dist/run/raw-finalize.d.ts +49 -3
- package/dist/run/raw-finalize.d.ts.map +1 -1
- package/dist/run/raw-finalize.js +31 -14
- package/dist/run/raw-search-cost.d.ts +3 -2
- package/dist/run/raw-search-cost.d.ts.map +1 -1
- package/dist/run/raw-search-cost.js +1 -0
- package/dist/run/rawonly.d.ts +29 -5
- package/dist/run/rawonly.d.ts.map +1 -1
- package/dist/run/rawonly.js +54 -19
- package/dist/run/record-call.d.ts +104 -0
- package/dist/run/record-call.d.ts.map +1 -0
- package/dist/run/record-call.js +277 -0
- package/dist/telemetry/attempt-meta.d.ts +11 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +3 -0
- package/dist/telemetry/lanes.d.ts +31 -0
- package/dist/telemetry/lanes.d.ts.map +1 -0
- package/dist/telemetry/lanes.js +41 -0
- package/dist/telemetry/org-identity.d.ts +5 -2
- package/dist/telemetry/org-identity.d.ts.map +1 -1
- package/dist/telemetry/org-identity.js +10 -8
- package/docs/agent-skill/SKILL.md +109 -10
- package/docs/how-it-fits.md +1 -1
- package/docs/instrumentation.md +152 -40
- package/docs/llms.txt +92 -9
- package/docs/quickstart.md +127 -15
- package/package.json +1 -1
package/dist/run/raw-finalize.js
CHANGED
|
@@ -11,7 +11,9 @@
|
|
|
11
11
|
*/
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.rawTimeoutFixHint = rawTimeoutFixHint;
|
|
14
|
+
exports.recordRawRow = recordRawRow;
|
|
14
15
|
exports.prepareRawStep = prepareRawStep;
|
|
16
|
+
exports.emitRawStep = emitRawStep;
|
|
15
17
|
exports.finalizeRaw = finalizeRaw;
|
|
16
18
|
const node_crypto_1 = require("node:crypto");
|
|
17
19
|
const cost_1 = require("./cost");
|
|
@@ -55,7 +57,7 @@ function recordRawRow(telemetry, p) {
|
|
|
55
57
|
provider: p.provider,
|
|
56
58
|
model: p.model,
|
|
57
59
|
requestedTier: p.requestedTier,
|
|
58
|
-
servedTier: p.usage?.servedTier ?? "standard",
|
|
60
|
+
servedTier: p.usage?.servedTier ?? p.servedTier ?? "standard",
|
|
59
61
|
tokensIn: p.usage?.tokensIn ?? 0,
|
|
60
62
|
tokensOut: p.usage?.tokensOut ?? 0,
|
|
61
63
|
tokensCached: p.usage?.tokensCached ?? 0,
|
|
@@ -106,6 +108,22 @@ function prepareRawStep(deps, input) {
|
|
|
106
108
|
const stepKey = (0, run_context_1.resolveStepKey)(input.stepKey, input.tags?.step, () => runCtx.nextStepKey(rawModelId));
|
|
107
109
|
return { runCtx, stepKey, startedAtIso: new Date().toISOString() };
|
|
108
110
|
}
|
|
111
|
+
/** G1: mirror one lane-B inference (raw, or an OBS-1 recorded call) into a
|
|
112
|
+
* ground-truth step — parity with the routed lane's record-attempt. No-op when
|
|
113
|
+
* no run context was captured at entry. Fail-open is handled inside
|
|
114
|
+
* `emitGroundTruthStep` (buffer errors are swallowed). */
|
|
115
|
+
function emitRawStep(step, task, inferenceId, failureClass) {
|
|
116
|
+
if (step === null)
|
|
117
|
+
return;
|
|
118
|
+
step.runCtx.emitGroundTruthStep({
|
|
119
|
+
stepKey: step.stepKey,
|
|
120
|
+
task,
|
|
121
|
+
inferenceId,
|
|
122
|
+
startedAt: step.startedAtIso,
|
|
123
|
+
endedAt: new Date().toISOString(),
|
|
124
|
+
failureClass,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
109
127
|
/**
|
|
110
128
|
* Turn an `AdapterResult` into a RunResult, recording exactly one telemetry row.
|
|
111
129
|
* Shared by the buffered and streaming paths so cost/telemetry are byte-identical.
|
|
@@ -132,7 +150,17 @@ function finalizeRaw(deps, input, result, latencyMs, step) {
|
|
|
132
150
|
// caller set none — spread FIRST so a caller value wins, and omitted entirely
|
|
133
151
|
// outside a run context so the row stays byte-identical (parity with RC-1).
|
|
134
152
|
const stepTag = step !== null ? { step: step.stepKey } : {};
|
|
135
|
-
|
|
153
|
+
// L6 (OBS-1 follow-up, architect ruling Q2): the raw lane dispatches ONLY on
|
|
154
|
+
// the caller's own provider key (`input.apiKey` — there is no managed-key path
|
|
155
|
+
// here; the hosted gateway resolves a BYOK vault key fail-closed), so every raw
|
|
156
|
+
// row is BYOK by construction. `key_ownership: "byok"` is SDK-owned and stamped
|
|
157
|
+
// LAST beside `lane`, so an in-process runRaw row can never settle as managed.
|
|
158
|
+
const tags = {
|
|
159
|
+
...stepTag,
|
|
160
|
+
...(0, run_context_1.withRunIdTag)(input.tags ?? {}, step?.runCtx ?? null),
|
|
161
|
+
key_ownership: "byok",
|
|
162
|
+
lane: "raw",
|
|
163
|
+
};
|
|
136
164
|
// WI-1 (spec §2, §6.3 lane parity): the attempt envelope, built by the SAME
|
|
137
165
|
// producer + rendered-request canonicalization the routed lane uses. R2: a
|
|
138
166
|
// declared `responseSchema` joins the input_hash exactly as a route schema
|
|
@@ -154,18 +182,7 @@ function finalizeRaw(deps, input, result, latencyMs, step) {
|
|
|
154
182
|
// G1: mirror this raw inference into a ground-truth step (parity with the
|
|
155
183
|
// routed lane's record-attempt). task = tags.step ?? provider/model (spec F1).
|
|
156
184
|
// Fail-open is handled inside emitGroundTruthStep (buffer errors are swallowed).
|
|
157
|
-
const emitStep = (inferenceId, failureClass) =>
|
|
158
|
-
if (step === null)
|
|
159
|
-
return;
|
|
160
|
-
step.runCtx.emitGroundTruthStep({
|
|
161
|
-
stepKey: step.stepKey,
|
|
162
|
-
task: input.tags?.step ?? rawModelId,
|
|
163
|
-
inferenceId,
|
|
164
|
-
startedAt: step.startedAtIso,
|
|
165
|
-
endedAt: new Date().toISOString(),
|
|
166
|
-
failureClass,
|
|
167
|
-
});
|
|
168
|
-
};
|
|
185
|
+
const emitStep = (inferenceId, failureClass) => emitRawStep(step, input.tags?.step ?? rawModelId, inferenceId, failureClass);
|
|
169
186
|
if (!result.ok) {
|
|
170
187
|
let costUsd = 0;
|
|
171
188
|
let pricingMissing = false;
|
|
@@ -25,8 +25,9 @@ import type { TelemetryStore } from "../types/telemetry";
|
|
|
25
25
|
* provider is the real dispatch provider so the surcharge is attributed to it. */
|
|
26
26
|
export declare const NATIVE_WEB_SEARCH_MODEL = "web_search";
|
|
27
27
|
export interface RawSearchCostArgs {
|
|
28
|
-
/** The real dispatch provider (openai / grok) — the row's `provider`.
|
|
29
|
-
|
|
28
|
+
/** The real dispatch provider (openai / grok) — the row's `provider`. An
|
|
29
|
+
* OBS-1 recorded call may name a provider ModelRig has no rate for (⇒ $0). */
|
|
30
|
+
readonly provider: ProviderId | string;
|
|
30
31
|
/** The `provider/model` handle the caller used — the row's `route`. */
|
|
31
32
|
readonly rawModelId: string;
|
|
32
33
|
/** The effective tags already stamped on the primary raw row (lane:"raw",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"raw-search-cost.d.ts","sourceRoot":"","sources":["../../src/run/raw-search-cost.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAgB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAIvE;;kFAEkF;AAClF,eAAO,MAAM,uBAAuB,eAAe,CAAC;AAEpD,MAAM,WAAW,iBAAiB;IAChC
|
|
1
|
+
{"version":3,"file":"raw-search-cost.d.ts","sourceRoot":"","sources":["../../src/run/raw-search-cost.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAgB,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAIvE;;kFAEkF;AAClF,eAAO,MAAM,uBAAuB,eAAe,CAAC;AAEpD,MAAM,WAAW,iBAAiB;IAChC;kFAC8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,MAAM,CAAC;IACvC,uEAAuE;IACvE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;kEAC8D;IAC9D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,iBAAiB,GAAG,IAAI,CA0C/F"}
|
|
@@ -36,6 +36,7 @@ exports.NATIVE_WEB_SEARCH_MODEL = "web_search";
|
|
|
36
36
|
*/
|
|
37
37
|
function recordRawSearchCostRow(telemetry, args) {
|
|
38
38
|
const searchCalls = Math.max(1, args.searchCalls);
|
|
39
|
+
// An unknown provider has no WEB_SEARCH_PRICING entry ⇒ webSearchPerCallUsd's `?.` yields 0.
|
|
39
40
|
const costEstimateUsd = (0, web_search_pricing_1.webSearchPerCallUsd)(args.provider) * searchCalls;
|
|
40
41
|
// U4: assembled by the ONE shared builder (./telemetry-row.ts). `rung:"search"`
|
|
41
42
|
// and `attemptSeq: 2` (so it never collides with the primary raw row's
|
package/dist/run/rawonly.d.ts
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* 700-LOC cap — `createRig` remains the ONE construction path (a separate
|
|
18
18
|
* `createRawRig()` factory was considered and REJECTED, spec §2 R1).
|
|
19
19
|
*/
|
|
20
|
-
import type
|
|
20
|
+
import { type RigConfig } from "../config";
|
|
21
21
|
import { type LoadedRoutes } from "../routes/loader";
|
|
22
22
|
import { RigFailureError } from "../types/failures";
|
|
23
23
|
import type { Rig } from "../index";
|
|
@@ -27,7 +27,12 @@ export declare const RAW_ONLY_FIX_HINT = "this rig was constructed runRaw-only (
|
|
|
27
27
|
/** Build the TERMINAL failure a route-dependent surface throws when the rig was
|
|
28
28
|
* constructed runRaw-only. Reuses `invariant_violation` (already TERMINAL + in
|
|
29
29
|
* the teaching table) — no new FailureClass (spec §3 rationale). */
|
|
30
|
-
|
|
30
|
+
/** Why a rig is raw-only: constructed with `routesDir: null`, or fell back
|
|
31
|
+
* because the default routes directory is absent (SURF-1). */
|
|
32
|
+
export type RawOnlyReason = "routesDir-null" | "absent-default";
|
|
33
|
+
/** The fix hint when the absent-default fallback made the rig raw-only. */
|
|
34
|
+
export declare const ABSENT_DEFAULT_FIX_HINT = "create ./modelrig/routes with your route files (`npx modelrig init` drafts them), or set MODELRIG_ROUTES_DIR";
|
|
35
|
+
export declare function rawOnlyRouteFailure(surface: string, reason?: RawOnlyReason): RigFailureError;
|
|
31
36
|
/** Narrow a config's `routesDir` to a string for a route-REQUIRED context (the
|
|
32
37
|
* route-oriented CLI subcommands: bakeoff/watch/swap/init/observe/validate),
|
|
33
38
|
* throwing the teaching failure if the rig was built runRaw-only.
|
|
@@ -35,6 +40,21 @@ export declare function rawOnlyRouteFailure(surface: string): RigFailureError;
|
|
|
35
40
|
* env-built configs — it guards a hand-built `routesDir: null` reaching a
|
|
36
41
|
* route-only command, and satisfies the `string | null` type at those seams. */
|
|
37
42
|
export declare function requireRoutesDir(config: RigConfig, surface: string): string;
|
|
43
|
+
/** The warning createRig prints when it falls back to a raw-only rig because the
|
|
44
|
+
* DEFAULT routes directory does not exist (SURF-1, audit #1). One constant so
|
|
45
|
+
* the docs and the test quote the same words. */
|
|
46
|
+
export declare const ABSENT_DEFAULT_ROUTES_WARNING: string;
|
|
47
|
+
/**
|
|
48
|
+
* The routes directory createRig actually loads (SURF-1 ruling on audit #1):
|
|
49
|
+
* - `null` stays `null` (CE-8 runRaw-only, silent);
|
|
50
|
+
* - the UNCONFIGURED default (loadConfigFromEnv filled in `./modelrig/routes`;
|
|
51
|
+
* no MODELRIG_ROUTES_DIR, no routesDir override, no `extraRoutesDir`) that
|
|
52
|
+
* does not exist on disk → warn once, return `null`, so
|
|
53
|
+
* a customer who only records calls never crashes on import;
|
|
54
|
+
* - anything explicitly configured is returned as-is, and a missing one still
|
|
55
|
+
* throws the teaching RouteConfigError from the loader.
|
|
56
|
+
*/
|
|
57
|
+
export declare function effectiveRoutesDir(config: RigConfig): string | null;
|
|
38
58
|
/** The empty route store for a runRaw-only rig: `list()` is `[]` (so
|
|
39
59
|
* serveability, capture-detection, and the exported route mirror are all no-ops)
|
|
40
60
|
* and `get()` teaches, though no public surface reaches it — every route method
|
|
@@ -48,13 +68,17 @@ export declare function loadRoutesOrReport(config: RigConfig, routesDir: string)
|
|
|
48
68
|
* `routesDir === null` so every one of them throws the teaching failure rather
|
|
49
69
|
* than dereferencing an empty store. Spread LAST over the real methods in
|
|
50
70
|
* `createRig`, so later keys win. */
|
|
51
|
-
type RouteMethods = Pick<Rig, "run" | "runStream" | "runBundle" | "runBundleStream" | "replay" | "bakeoff" | "
|
|
52
|
-
/** The throwing overrides for a runRaw-only rig (spec §2 R1).
|
|
71
|
+
type RouteMethods = Pick<Rig, "run" | "runStream" | "runBundle" | "runBundleStream" | "replay" | "bakeoff" | "verifySwap">;
|
|
72
|
+
/** The throwing overrides for a runRaw-only rig (spec §2 R1).
|
|
73
|
+
* `bakeoffFromEvalCases` is deliberately NOT overridden (SURF-1, audit #12): it
|
|
74
|
+
* reads eval cases from the local store first, so with none promoted (or none
|
|
75
|
+
* rehydratable) it reports honestly without any routes dir; only a case that
|
|
76
|
+
* actually replays reaches `routes.get`, which teaches the same failure. Each names the
|
|
53
77
|
* surface + the route/task the caller asked for, so the terminal error points at
|
|
54
78
|
* exactly the call that needs a `routesDir`. Promise-returning surfaces REJECT
|
|
55
79
|
* (matching the routed lane, where errors surface as rejections); the two
|
|
56
80
|
* streaming surfaces return a handle synchronously, so they throw synchronously
|
|
57
81
|
* — the caller never reaches an event to await. */
|
|
58
|
-
export declare function rawOnlyRouteMethods(): RouteMethods;
|
|
82
|
+
export declare function rawOnlyRouteMethods(reason?: RawOnlyReason): RouteMethods;
|
|
59
83
|
export {};
|
|
60
84
|
//# sourceMappingURL=rawonly.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rawonly.d.ts","sourceRoot":"","sources":["../../src/run/rawonly.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;
|
|
1
|
+
{"version":3,"file":"rawonly.d.ts","sourceRoot":"","sources":["../../src/run/rawonly.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EAAsB,KAAK,SAAS,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EAAyC,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAE5F,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAEpC;sEACsE;AACtE,eAAO,MAAM,iBAAiB,uGACwE,CAAC;AAEvG;;oEAEoE;AACpE;8DAC8D;AAC9D,MAAM,MAAM,aAAa,GAAG,gBAAgB,GAAG,gBAAgB,CAAC;AAEhE,2EAA2E;AAC3E,eAAO,MAAM,uBAAuB,iHAC4E,CAAC;AAEjH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,GAAE,aAAgC,GAAG,eAAe,CAS9G;AAsBD;;;;;gFAKgF;AAChF,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAG3E;AAED;;iDAEiD;AACjD,eAAO,MAAM,6BAA6B,QAIW,CAAC;AAEtD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CAQnE;AAED;;;mDAGmD;AACnD,wBAAgB,eAAe,IAAI,YAAY,CAS9C;AAED;;0EAE0E;AAC1E,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,CASrF;AAED;;;qCAGqC;AACrC,KAAK,YAAY,GAAG,IAAI,CACtB,GAAG,EACD,KAAK,GACL,WAAW,GACX,WAAW,GACX,iBAAiB,GACjB,QAAQ,GACR,SAAS,GACT,YAAY,CACf,CAAC;AAEF;;;;;;;;;mDASmD;AACnD,wBAAgB,mBAAmB,CAAC,MAAM,GAAE,aAAgC,GAAG,YAAY,CAW1F"}
|
package/dist/run/rawonly.js
CHANGED
|
@@ -19,26 +19,30 @@
|
|
|
19
19
|
* `createRawRig()` factory was considered and REJECTED, spec §2 R1).
|
|
20
20
|
*/
|
|
21
21
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
-
exports.RAW_ONLY_FIX_HINT = void 0;
|
|
22
|
+
exports.ABSENT_DEFAULT_ROUTES_WARNING = exports.ABSENT_DEFAULT_FIX_HINT = exports.RAW_ONLY_FIX_HINT = void 0;
|
|
23
23
|
exports.rawOnlyRouteFailure = rawOnlyRouteFailure;
|
|
24
24
|
exports.requireRoutesDir = requireRoutesDir;
|
|
25
|
+
exports.effectiveRoutesDir = effectiveRoutesDir;
|
|
25
26
|
exports.emptyRouteStore = emptyRouteStore;
|
|
26
27
|
exports.loadRoutesOrReport = loadRoutesOrReport;
|
|
27
28
|
exports.rawOnlyRouteMethods = rawOnlyRouteMethods;
|
|
29
|
+
const node_fs_1 = require("node:fs");
|
|
30
|
+
const config_1 = require("../config");
|
|
28
31
|
const loader_1 = require("../routes/loader");
|
|
29
32
|
const load_error_report_1 = require("../telemetry/load-error-report");
|
|
30
33
|
const failures_1 = require("../types/failures");
|
|
31
34
|
/** The teaching remediation for a route-lane call on a runRaw-only rig (spec §2
|
|
32
35
|
* R1). One constant so the failure and the docs stay in lock-step. */
|
|
33
36
|
exports.RAW_ONLY_FIX_HINT = "this rig was constructed runRaw-only (routesDir: null); give createRig a routesDir to serve routes";
|
|
34
|
-
/**
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
function rawOnlyRouteFailure(surface) {
|
|
37
|
+
/** The fix hint when the absent-default fallback made the rig raw-only. */
|
|
38
|
+
exports.ABSENT_DEFAULT_FIX_HINT = "create ./modelrig/routes with your route files (`npx modelrig init` drafts them), or set MODELRIG_ROUTES_DIR";
|
|
39
|
+
function rawOnlyRouteFailure(surface, reason = "routesDir-null") {
|
|
38
40
|
return new failures_1.RigFailureError({
|
|
39
41
|
class: "invariant_violation",
|
|
40
|
-
message:
|
|
41
|
-
|
|
42
|
+
message: reason === "absent-default"
|
|
43
|
+
? `${surface} needs routes, but there is no routes directory at ./modelrig/routes, so this rig is raw-only`
|
|
44
|
+
: `${surface} needs routes, but this rig was constructed runRaw-only (routesDir: null)`,
|
|
45
|
+
fixHint: reason === "absent-default" ? exports.ABSENT_DEFAULT_FIX_HINT : exports.RAW_ONLY_FIX_HINT,
|
|
42
46
|
});
|
|
43
47
|
}
|
|
44
48
|
/** A `RunStreamHandle` that fails the runRaw-only way: `result` rejects and the
|
|
@@ -46,8 +50,8 @@ function rawOnlyRouteFailure(surface) {
|
|
|
46
50
|
* throws synchronously (a stream surface always hands back a handle, errors ride
|
|
47
51
|
* `result`/`events`). A synchronous throw here would break the caller's
|
|
48
52
|
* `const h = rig.runStream(...); await h.result` shape. */
|
|
49
|
-
function rawOnlyStreamHandle(surface) {
|
|
50
|
-
const err = rawOnlyRouteFailure(surface);
|
|
53
|
+
function rawOnlyStreamHandle(surface, reason) {
|
|
54
|
+
const err = rawOnlyRouteFailure(surface, reason);
|
|
51
55
|
const result = Promise.reject(err);
|
|
52
56
|
// Keep the rejection handled so an unconsumed handle never becomes an
|
|
53
57
|
// unhandledRejection; the events iterator surfaces the same error.
|
|
@@ -71,6 +75,33 @@ function requireRoutesDir(config, surface) {
|
|
|
71
75
|
throw rawOnlyRouteFailure(surface);
|
|
72
76
|
return config.routesDir;
|
|
73
77
|
}
|
|
78
|
+
/** The warning createRig prints when it falls back to a raw-only rig because the
|
|
79
|
+
* DEFAULT routes directory does not exist (SURF-1, audit #1). One constant so
|
|
80
|
+
* the docs and the test quote the same words. */
|
|
81
|
+
exports.ABSENT_DEFAULT_ROUTES_WARNING = "[modelrig] no routes directory at ./modelrig/routes, so this rig is raw-only: rig.runRaw, " +
|
|
82
|
+
"rig.runRawStream and rig.recordCall work, and rig.run / rig.runBundle need routes. " +
|
|
83
|
+
"Create routes with `npx modelrig init`, set MODELRIG_ROUTES_DIR, or pass `routesDir: null` " +
|
|
84
|
+
"to say raw-only on purpose and silence this line.";
|
|
85
|
+
/**
|
|
86
|
+
* The routes directory createRig actually loads (SURF-1 ruling on audit #1):
|
|
87
|
+
* - `null` stays `null` (CE-8 runRaw-only, silent);
|
|
88
|
+
* - the UNCONFIGURED default (loadConfigFromEnv filled in `./modelrig/routes`;
|
|
89
|
+
* no MODELRIG_ROUTES_DIR, no routesDir override, no `extraRoutesDir`) that
|
|
90
|
+
* does not exist on disk → warn once, return `null`, so
|
|
91
|
+
* a customer who only records calls never crashes on import;
|
|
92
|
+
* - anything explicitly configured is returned as-is, and a missing one still
|
|
93
|
+
* throws the teaching RouteConfigError from the loader.
|
|
94
|
+
*/
|
|
95
|
+
function effectiveRoutesDir(config) {
|
|
96
|
+
const dir = config.routesDir;
|
|
97
|
+
if (dir === null)
|
|
98
|
+
return null;
|
|
99
|
+
if (config.extraRoutesDir === undefined && (0, config_1.isDefaultRoutesDir)(config) && !(0, node_fs_1.existsSync)(dir)) {
|
|
100
|
+
console.warn(exports.ABSENT_DEFAULT_ROUTES_WARNING);
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
return dir;
|
|
104
|
+
}
|
|
74
105
|
/** The empty route store for a runRaw-only rig: `list()` is `[]` (so
|
|
75
106
|
* serveability, capture-detection, and the exported route mirror are all no-ops)
|
|
76
107
|
* and `get()` teaches, though no public surface reaches it — every route method
|
|
@@ -100,21 +131,25 @@ function loadRoutesOrReport(config, routesDir) {
|
|
|
100
131
|
throw err;
|
|
101
132
|
}
|
|
102
133
|
}
|
|
103
|
-
/** The throwing overrides for a runRaw-only rig (spec §2 R1).
|
|
134
|
+
/** The throwing overrides for a runRaw-only rig (spec §2 R1).
|
|
135
|
+
* `bakeoffFromEvalCases` is deliberately NOT overridden (SURF-1, audit #12): it
|
|
136
|
+
* reads eval cases from the local store first, so with none promoted (or none
|
|
137
|
+
* rehydratable) it reports honestly without any routes dir; only a case that
|
|
138
|
+
* actually replays reaches `routes.get`, which teaches the same failure. Each names the
|
|
104
139
|
* surface + the route/task the caller asked for, so the terminal error points at
|
|
105
140
|
* exactly the call that needs a `routesDir`. Promise-returning surfaces REJECT
|
|
106
141
|
* (matching the routed lane, where errors surface as rejections); the two
|
|
107
142
|
* streaming surfaces return a handle synchronously, so they throw synchronously
|
|
108
143
|
* — the caller never reaches an event to await. */
|
|
109
|
-
function rawOnlyRouteMethods() {
|
|
144
|
+
function rawOnlyRouteMethods(reason = "routesDir-null") {
|
|
145
|
+
const fail = (surface) => Promise.reject(rawOnlyRouteFailure(surface, reason));
|
|
110
146
|
return {
|
|
111
|
-
run: (route) =>
|
|
112
|
-
runStream: (route) => rawOnlyStreamHandle(`runStream("${route}")
|
|
113
|
-
runBundle: (bundle) =>
|
|
114
|
-
runBundleStream: (bundle) => rawOnlyStreamHandle(`runBundleStream("${bundle.route}")
|
|
115
|
-
replay: (route) =>
|
|
116
|
-
bakeoff: (route) =>
|
|
117
|
-
|
|
118
|
-
verifySwap: (route) => Promise.reject(rawOnlyRouteFailure(`verifySwap("${route}")`)),
|
|
147
|
+
run: (route) => fail(`run("${route}")`),
|
|
148
|
+
runStream: (route) => rawOnlyStreamHandle(`runStream("${route}")`, reason),
|
|
149
|
+
runBundle: (bundle) => fail(`runBundle("${bundle.route}")`),
|
|
150
|
+
runBundleStream: (bundle) => rawOnlyStreamHandle(`runBundleStream("${bundle.route}")`, reason),
|
|
151
|
+
replay: (route) => fail(`replay("${route}")`),
|
|
152
|
+
bakeoff: (route) => fail(`bakeoff("${route}")`),
|
|
153
|
+
verifySwap: (route) => fail(`verifySwap("${route}")`),
|
|
119
154
|
};
|
|
120
155
|
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OBS-1 — `rig.recordCall()`: record a provider call ModelRig did NOT dispatch
|
|
3
|
+
* (acceptance ruling M12, Option A). The customer's own code made the call with
|
|
4
|
+
* its own key; it reports what the provider returned (tokens, tier, latency,
|
|
5
|
+
* failure) and ModelRig writes ONE telemetry row for it, exactly the shape the
|
|
6
|
+
* raw lane writes — same builder, same cost function, same step emission — so
|
|
7
|
+
* the call shows on /inferences, joins its run and step, and can be graded.
|
|
8
|
+
*
|
|
9
|
+
* WHAT IS DIFFERENT FROM THE RAW LANE (and nothing else is):
|
|
10
|
+
* - `tags.lane = "external"` (SDK-owned, stamped LAST — a caller tag cannot
|
|
11
|
+
* override it). Named `external`, not `direct`: ruling D-2 already uses
|
|
12
|
+
* "direct" for `destination`.
|
|
13
|
+
* - `tags.key_ownership = "byok"` (SDK-owned). Gate B-0 (usage-aggregation.ts):
|
|
14
|
+
* a BYOK seam must stamp the tag at that seam. A recorded call is always the
|
|
15
|
+
* customer's key — ModelRig bore no provider cost — so it can never settle as
|
|
16
|
+
* a managed charge.
|
|
17
|
+
* - `meta.provenance = "caller-reported"`: the numbers are the caller's, not a
|
|
18
|
+
* measurement. Nothing was validated (`validated` is never claimed).
|
|
19
|
+
*
|
|
20
|
+
* CONTRACT: synchronous (the row is written to the local store; the exporter
|
|
21
|
+
* ships it on its normal cadence) and NEVER throws into the caller. Bad input is
|
|
22
|
+
* warned with a teaching fix (P4) and returns `{ inferenceId: null, … }` with no
|
|
23
|
+
* row written.
|
|
24
|
+
*
|
|
25
|
+
* No route/registry/capability-gate dependency beyond the raw lane's own (cost
|
|
26
|
+
* + telemetry + run context) — the raw lane's no-hard-hot-path invariant holds.
|
|
27
|
+
*/
|
|
28
|
+
import type { FailureClass } from "../types/failures";
|
|
29
|
+
import type { ProviderId, ServiceTier } from "../types/provider";
|
|
30
|
+
import type { RunTags } from "../types/route";
|
|
31
|
+
import type { RawRunDeps } from "./raw";
|
|
32
|
+
/** The lane tag a recorded call carries (SDK-owned) — the shared classifier's constant. */
|
|
33
|
+
export { EXTERNAL_LANE } from "../telemetry/lanes";
|
|
34
|
+
/** Where the warning's fix points (the "record them" section). */
|
|
35
|
+
export declare const RECORD_CALL_DOCS_URL = "https://modelrig.dev/not-captured.html";
|
|
36
|
+
/** Usage the caller's provider response reported. Same convention as every
|
|
37
|
+
* adapter (`AdapterUsage`): `tokensIn` is the TOTAL input and INCLUDES
|
|
38
|
+
* `tokensCached` and `tokensCacheWrite` as subsets. Token counts (and
|
|
39
|
+
* `latencyMs`/`ttfbMs`) are ROUNDED to the nearest integer and clamped
|
|
40
|
+
* (tokens ≤ Number.MAX_SAFE_INTEGER, milliseconds ≤ 2³¹−1) before recording. */
|
|
41
|
+
export interface RecordCallUsage {
|
|
42
|
+
readonly tokensIn: number;
|
|
43
|
+
readonly tokensOut: number;
|
|
44
|
+
readonly tokensCached?: number;
|
|
45
|
+
readonly tokensCacheWrite?: number;
|
|
46
|
+
readonly reasoningTokens?: number;
|
|
47
|
+
}
|
|
48
|
+
export interface RecordCallInput {
|
|
49
|
+
/** A ModelRig `ProviderId` is priced; any other non-empty provider id is
|
|
50
|
+
* recorded at $0 with `pricingMissing: true`. */
|
|
51
|
+
readonly provider: ProviderId | (string & {});
|
|
52
|
+
readonly model: string;
|
|
53
|
+
readonly usage: RecordCallUsage;
|
|
54
|
+
/** The tier that SERVED the call — the row is priced at it (flex is cheaper),
|
|
55
|
+
* exactly as the raw lane prices. Default `"standard"`. */
|
|
56
|
+
readonly servedTier?: ServiceTier;
|
|
57
|
+
/** The tier your code asked for. Default: `servedTier`. */
|
|
58
|
+
readonly requestedTier?: ServiceTier;
|
|
59
|
+
readonly latencyMs: number;
|
|
60
|
+
readonly ttfbMs?: number;
|
|
61
|
+
/** A failed attempt that the provider still billed is still a row. */
|
|
62
|
+
readonly failureClass?: FailureClass | null;
|
|
63
|
+
/** Native web-search tool calls billed on this call — writes the separate
|
|
64
|
+
* `rung:"search"` cost row, as the raw lane does for native grounding. */
|
|
65
|
+
readonly webSearchCalls?: number;
|
|
66
|
+
/** The ground-truth step inside an active run (same rule as `runRaw`). */
|
|
67
|
+
readonly stepKey?: string;
|
|
68
|
+
readonly tags?: RunTags;
|
|
69
|
+
readonly providerRequestId?: string;
|
|
70
|
+
/** The provider's native finish reason (`STOP`, `end_turn`, `stop`, …) or a
|
|
71
|
+
* normalized one; mapped to the envelope enum. */
|
|
72
|
+
readonly finishReason?: string;
|
|
73
|
+
}
|
|
74
|
+
export interface RecordCallResult {
|
|
75
|
+
/** The recorded inference id (pass it to `rig.grade`), or null when the input
|
|
76
|
+
* was rejected and nothing was recorded. */
|
|
77
|
+
readonly inferenceId: string | null;
|
|
78
|
+
readonly callId: string | null;
|
|
79
|
+
/** The token row's list-price cost (a `webSearchCalls` surcharge rides its own
|
|
80
|
+
* row, as on the raw lane). 0 when unpriced or rejected. */
|
|
81
|
+
readonly costEstimateUsd: number;
|
|
82
|
+
readonly pricingMissing: boolean;
|
|
83
|
+
}
|
|
84
|
+
/** Server ingest's route cap (`modelrig-server` ingest.ts `isNonEmptyString(raw.route, 200)`). */
|
|
85
|
+
export declare const MAX_ROUTE_CHARS = 200;
|
|
86
|
+
/** Test-only: clear the once-per-rule warning latch. */
|
|
87
|
+
export declare function resetRecordCallWarningsForTest(): void;
|
|
88
|
+
/** The ceiling for latency/ttfb (Postgres `integer`, 2³¹−1 ms ≈ 24.8 days). */
|
|
89
|
+
export declare const MAX_MS = 2147483647;
|
|
90
|
+
/**
|
|
91
|
+
* L6 MUST-3 integer hygiene. Telemetry columns are integers, but a caller's
|
|
92
|
+
* `performance.now()` delta is fractional — so ROUND to the nearest integer
|
|
93
|
+
* (friendlier than rejecting) and CLAMP: latency/ttfb to ≤ 2³¹−1 ms, token
|
|
94
|
+
* counts to ≤ Number.MAX_SAFE_INTEGER. Applied before any check that compares
|
|
95
|
+
* token counts, so validation and the recorded row see the same numbers.
|
|
96
|
+
*/
|
|
97
|
+
export declare function toCount(v: number, max: number): number;
|
|
98
|
+
/**
|
|
99
|
+
* Record one call the caller made itself. See the module header for the
|
|
100
|
+
* contract. `deps` is the rig's raw-lane deps (telemetry + unpriced rate +
|
|
101
|
+
* manifest) — the same object `runRaw` receives.
|
|
102
|
+
*/
|
|
103
|
+
export declare function recordCall(deps: RawRunDeps, input: RecordCallInput): RecordCallResult;
|
|
104
|
+
//# sourceMappingURL=record-call.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"record-call.d.ts","sourceRoot":"","sources":["../../src/run/record-call.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAYH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAgB,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AAExC,2FAA2F;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,kEAAkE;AAClE,eAAO,MAAM,oBAAoB,2CAA2C,CAAC;AAE7E;;;;gFAIgF;AAChF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B;qDACiD;IACjD,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;+DAC2D;IAC3D,QAAQ,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC;IAClC,2DAA2D;IAC3D,QAAQ,CAAC,aAAa,CAAC,EAAE,WAAW,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,sEAAsE;IACtE,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5C;8EAC0E;IAC1E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;sDACkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B;gEAC4D;IAC5D,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAQD,kGAAkG;AAClG,eAAO,MAAM,eAAe,MAAM,CAAC;AAcnC,wDAAwD;AACxD,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AAMD,+EAA+E;AAC/E,eAAO,MAAM,MAAM,aAAgB,CAAC;AAEpC;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAEtD;AA6GD;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,GAAG,gBAAgB,CAarF"}
|