modelrig 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/dist/doctor-cli.d.ts.map +1 -1
- package/dist/doctor-cli.js +2 -1
- package/dist/doctor-preflight.d.ts +14 -2
- package/dist/doctor-preflight.d.ts.map +1 -1
- package/dist/doctor-preflight.js +60 -5
- package/dist/feedback/coverage.d.ts.map +1 -1
- package/dist/feedback/coverage.js +9 -2
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -5
- package/dist/run/raw-finalize.d.ts +49 -3
- package/dist/run/raw-finalize.d.ts.map +1 -1
- package/dist/run/raw-finalize.js +31 -14
- package/dist/run/raw-search-cost.d.ts +3 -2
- package/dist/run/raw-search-cost.d.ts.map +1 -1
- package/dist/run/raw-search-cost.js +1 -0
- package/dist/run/record-call.d.ts +104 -0
- package/dist/run/record-call.d.ts.map +1 -0
- package/dist/run/record-call.js +277 -0
- package/dist/telemetry/attempt-meta.d.ts +11 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +3 -0
- package/dist/telemetry/lanes.d.ts +31 -0
- package/dist/telemetry/lanes.d.ts.map +1 -0
- package/dist/telemetry/lanes.js +41 -0
- package/dist/telemetry/org-identity.d.ts +5 -2
- package/dist/telemetry/org-identity.d.ts.map +1 -1
- package/dist/telemetry/org-identity.js +10 -8
- package/docs/agent-skill/SKILL.md +3 -1
- package/docs/instrumentation.md +17 -0
- package/docs/llms.txt +3 -2
- package/package.json +1 -1
|
@@ -36,6 +36,7 @@ exports.NATIVE_WEB_SEARCH_MODEL = "web_search";
|
|
|
36
36
|
*/
|
|
37
37
|
function recordRawSearchCostRow(telemetry, args) {
|
|
38
38
|
const searchCalls = Math.max(1, args.searchCalls);
|
|
39
|
+
// An unknown provider has no WEB_SEARCH_PRICING entry ⇒ webSearchPerCallUsd's `?.` yields 0.
|
|
39
40
|
const costEstimateUsd = (0, web_search_pricing_1.webSearchPerCallUsd)(args.provider) * searchCalls;
|
|
40
41
|
// U4: assembled by the ONE shared builder (./telemetry-row.ts). `rung:"search"`
|
|
41
42
|
// and `attemptSeq: 2` (so it never collides with the primary raw row's
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OBS-1 — `rig.recordCall()`: record a provider call ModelRig did NOT dispatch
|
|
3
|
+
* (acceptance ruling M12, Option A). The customer's own code made the call with
|
|
4
|
+
* its own key; it reports what the provider returned (tokens, tier, latency,
|
|
5
|
+
* failure) and ModelRig writes ONE telemetry row for it, exactly the shape the
|
|
6
|
+
* raw lane writes — same builder, same cost function, same step emission — so
|
|
7
|
+
* the call shows on /inferences, joins its run and step, and can be graded.
|
|
8
|
+
*
|
|
9
|
+
* WHAT IS DIFFERENT FROM THE RAW LANE (and nothing else is):
|
|
10
|
+
* - `tags.lane = "external"` (SDK-owned, stamped LAST — a caller tag cannot
|
|
11
|
+
* override it). Named `external`, not `direct`: ruling D-2 already uses
|
|
12
|
+
* "direct" for `destination`.
|
|
13
|
+
* - `tags.key_ownership = "byok"` (SDK-owned). Gate B-0 (usage-aggregation.ts):
|
|
14
|
+
* a BYOK seam must stamp the tag at that seam. A recorded call is always the
|
|
15
|
+
* customer's key — ModelRig bore no provider cost — so it can never settle as
|
|
16
|
+
* a managed charge.
|
|
17
|
+
* - `meta.provenance = "caller-reported"`: the numbers are the caller's, not a
|
|
18
|
+
* measurement. Nothing was validated (`validated` is never claimed).
|
|
19
|
+
*
|
|
20
|
+
* CONTRACT: synchronous (the row is written to the local store; the exporter
|
|
21
|
+
* ships it on its normal cadence) and NEVER throws into the caller. Bad input is
|
|
22
|
+
* warned with a teaching fix (P4) and returns `{ inferenceId: null, … }` with no
|
|
23
|
+
* row written.
|
|
24
|
+
*
|
|
25
|
+
* No route/registry/capability-gate dependency beyond the raw lane's own (cost
|
|
26
|
+
* + telemetry + run context) — the raw lane's no-hard-hot-path invariant holds.
|
|
27
|
+
*/
|
|
28
|
+
import type { FailureClass } from "../types/failures";
|
|
29
|
+
import type { ProviderId, ServiceTier } from "../types/provider";
|
|
30
|
+
import type { RunTags } from "../types/route";
|
|
31
|
+
import type { RawRunDeps } from "./raw";
|
|
32
|
+
/** The lane tag a recorded call carries (SDK-owned) — the shared classifier's constant. */
|
|
33
|
+
export { EXTERNAL_LANE } from "../telemetry/lanes";
|
|
34
|
+
/** Where the warning's fix points (the "record them" section). */
|
|
35
|
+
export declare const RECORD_CALL_DOCS_URL = "https://modelrig.dev/not-captured.html";
|
|
36
|
+
/** Usage the caller's provider response reported. Same convention as every
|
|
37
|
+
* adapter (`AdapterUsage`): `tokensIn` is the TOTAL input and INCLUDES
|
|
38
|
+
* `tokensCached` and `tokensCacheWrite` as subsets. Token counts (and
|
|
39
|
+
* `latencyMs`/`ttfbMs`) are ROUNDED to the nearest integer and clamped
|
|
40
|
+
* (tokens ≤ Number.MAX_SAFE_INTEGER, milliseconds ≤ 2³¹−1) before recording. */
|
|
41
|
+
export interface RecordCallUsage {
|
|
42
|
+
readonly tokensIn: number;
|
|
43
|
+
readonly tokensOut: number;
|
|
44
|
+
readonly tokensCached?: number;
|
|
45
|
+
readonly tokensCacheWrite?: number;
|
|
46
|
+
readonly reasoningTokens?: number;
|
|
47
|
+
}
|
|
48
|
+
export interface RecordCallInput {
|
|
49
|
+
/** A ModelRig `ProviderId` is priced; any other non-empty provider id is
|
|
50
|
+
* recorded at $0 with `pricingMissing: true`. */
|
|
51
|
+
readonly provider: ProviderId | (string & {});
|
|
52
|
+
readonly model: string;
|
|
53
|
+
readonly usage: RecordCallUsage;
|
|
54
|
+
/** The tier that SERVED the call — the row is priced at it (flex is cheaper),
|
|
55
|
+
* exactly as the raw lane prices. Default `"standard"`. */
|
|
56
|
+
readonly servedTier?: ServiceTier;
|
|
57
|
+
/** The tier your code asked for. Default: `servedTier`. */
|
|
58
|
+
readonly requestedTier?: ServiceTier;
|
|
59
|
+
readonly latencyMs: number;
|
|
60
|
+
readonly ttfbMs?: number;
|
|
61
|
+
/** A failed attempt that the provider still billed is still a row. */
|
|
62
|
+
readonly failureClass?: FailureClass | null;
|
|
63
|
+
/** Native web-search tool calls billed on this call — writes the separate
|
|
64
|
+
* `rung:"search"` cost row, as the raw lane does for native grounding. */
|
|
65
|
+
readonly webSearchCalls?: number;
|
|
66
|
+
/** The ground-truth step inside an active run (same rule as `runRaw`). */
|
|
67
|
+
readonly stepKey?: string;
|
|
68
|
+
readonly tags?: RunTags;
|
|
69
|
+
readonly providerRequestId?: string;
|
|
70
|
+
/** The provider's native finish reason (`STOP`, `end_turn`, `stop`, …) or a
|
|
71
|
+
* normalized one; mapped to the envelope enum. */
|
|
72
|
+
readonly finishReason?: string;
|
|
73
|
+
}
|
|
74
|
+
export interface RecordCallResult {
|
|
75
|
+
/** The recorded inference id (pass it to `rig.grade`), or null when the input
|
|
76
|
+
* was rejected and nothing was recorded. */
|
|
77
|
+
readonly inferenceId: string | null;
|
|
78
|
+
readonly callId: string | null;
|
|
79
|
+
/** The token row's list-price cost (a `webSearchCalls` surcharge rides its own
|
|
80
|
+
* row, as on the raw lane). 0 when unpriced or rejected. */
|
|
81
|
+
readonly costEstimateUsd: number;
|
|
82
|
+
readonly pricingMissing: boolean;
|
|
83
|
+
}
|
|
84
|
+
/** Server ingest's route cap (`modelrig-server` ingest.ts `isNonEmptyString(raw.route, 200)`). */
|
|
85
|
+
export declare const MAX_ROUTE_CHARS = 200;
|
|
86
|
+
/** Test-only: clear the once-per-rule warning latch. */
|
|
87
|
+
export declare function resetRecordCallWarningsForTest(): void;
|
|
88
|
+
/** The ceiling for latency/ttfb (Postgres `integer`, 2³¹−1 ms ≈ 24.8 days). */
|
|
89
|
+
export declare const MAX_MS = 2147483647;
|
|
90
|
+
/**
|
|
91
|
+
* L6 MUST-3 integer hygiene. Telemetry columns are integers, but a caller's
|
|
92
|
+
* `performance.now()` delta is fractional — so ROUND to the nearest integer
|
|
93
|
+
* (friendlier than rejecting) and CLAMP: latency/ttfb to ≤ 2³¹−1 ms, token
|
|
94
|
+
* counts to ≤ Number.MAX_SAFE_INTEGER. Applied before any check that compares
|
|
95
|
+
* token counts, so validation and the recorded row see the same numbers.
|
|
96
|
+
*/
|
|
97
|
+
export declare function toCount(v: number, max: number): number;
|
|
98
|
+
/**
|
|
99
|
+
* Record one call the caller made itself. See the module header for the
|
|
100
|
+
* contract. `deps` is the rig's raw-lane deps (telemetry + unpriced rate +
|
|
101
|
+
* manifest) — the same object `runRaw` receives.
|
|
102
|
+
*/
|
|
103
|
+
export declare function recordCall(deps: RawRunDeps, input: RecordCallInput): RecordCallResult;
|
|
104
|
+
//# sourceMappingURL=record-call.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"record-call.d.ts","sourceRoot":"","sources":["../../src/run/record-call.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAYH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAgB,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AAExC,2FAA2F;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,kEAAkE;AAClE,eAAO,MAAM,oBAAoB,2CAA2C,CAAC;AAE7E;;;;gFAIgF;AAChF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B;qDACiD;IACjD,QAAQ,CAAC,QAAQ,EAAE,UAAU,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;+DAC2D;IAC3D,QAAQ,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC;IAClC,2DAA2D;IAC3D,QAAQ,CAAC,aAAa,CAAC,EAAE,WAAW,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,sEAAsE;IACtE,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IAC5C;8EAC0E;IAC1E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;sDACkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B;gEAC4D;IAC5D,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAQD,kGAAkG;AAClG,eAAO,MAAM,eAAe,MAAM,CAAC;AAcnC,wDAAwD;AACxD,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AAMD,+EAA+E;AAC/E,eAAO,MAAM,MAAM,aAAgB,CAAC;AAEpC;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAEtD;AA6GD;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,GAAG,gBAAgB,CAarF"}
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* OBS-1 — `rig.recordCall()`: record a provider call ModelRig did NOT dispatch
|
|
4
|
+
* (acceptance ruling M12, Option A). The customer's own code made the call with
|
|
5
|
+
* its own key; it reports what the provider returned (tokens, tier, latency,
|
|
6
|
+
* failure) and ModelRig writes ONE telemetry row for it, exactly the shape the
|
|
7
|
+
* raw lane writes — same builder, same cost function, same step emission — so
|
|
8
|
+
* the call shows on /inferences, joins its run and step, and can be graded.
|
|
9
|
+
*
|
|
10
|
+
* WHAT IS DIFFERENT FROM THE RAW LANE (and nothing else is):
|
|
11
|
+
* - `tags.lane = "external"` (SDK-owned, stamped LAST — a caller tag cannot
|
|
12
|
+
* override it). Named `external`, not `direct`: ruling D-2 already uses
|
|
13
|
+
* "direct" for `destination`.
|
|
14
|
+
* - `tags.key_ownership = "byok"` (SDK-owned). Gate B-0 (usage-aggregation.ts):
|
|
15
|
+
* a BYOK seam must stamp the tag at that seam. A recorded call is always the
|
|
16
|
+
* customer's key — ModelRig bore no provider cost — so it can never settle as
|
|
17
|
+
* a managed charge.
|
|
18
|
+
* - `meta.provenance = "caller-reported"`: the numbers are the caller's, not a
|
|
19
|
+
* measurement. Nothing was validated (`validated` is never claimed).
|
|
20
|
+
*
|
|
21
|
+
* CONTRACT: synchronous (the row is written to the local store; the exporter
|
|
22
|
+
* ships it on its normal cadence) and NEVER throws into the caller. Bad input is
|
|
23
|
+
* warned with a teaching fix (P4) and returns `{ inferenceId: null, … }` with no
|
|
24
|
+
* row written.
|
|
25
|
+
*
|
|
26
|
+
* No route/registry/capability-gate dependency beyond the raw lane's own (cost
|
|
27
|
+
* + telemetry + run context) — the raw lane's no-hard-hot-path invariant holds.
|
|
28
|
+
*/
|
|
29
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
30
|
+
exports.MAX_MS = exports.MAX_ROUTE_CHARS = exports.RECORD_CALL_DOCS_URL = exports.EXTERNAL_LANE = void 0;
|
|
31
|
+
exports.resetRecordCallWarningsForTest = resetRecordCallWarningsForTest;
|
|
32
|
+
exports.toCount = toCount;
|
|
33
|
+
exports.recordCall = recordCall;
|
|
34
|
+
const node_crypto_1 = require("node:crypto");
|
|
35
|
+
const cost_1 = require("./cost");
|
|
36
|
+
const raw_finalize_1 = require("./raw-finalize");
|
|
37
|
+
const raw_search_cost_1 = require("./raw-search-cost");
|
|
38
|
+
const attempt_meta_map_1 = require("../adapters/attempt-meta-map");
|
|
39
|
+
const run_context_1 = require("../artifacts/run-context");
|
|
40
|
+
const lanes_1 = require("../telemetry/lanes");
|
|
41
|
+
const attempt_meta_1 = require("../telemetry/attempt-meta");
|
|
42
|
+
const failures_teaching_1 = require("../types/failures-teaching");
|
|
43
|
+
const provider_1 = require("../types/provider");
|
|
44
|
+
/** The lane tag a recorded call carries (SDK-owned) — the shared classifier's constant. */
|
|
45
|
+
var lanes_2 = require("../telemetry/lanes");
|
|
46
|
+
Object.defineProperty(exports, "EXTERNAL_LANE", { enumerable: true, get: function () { return lanes_2.EXTERNAL_LANE; } });
|
|
47
|
+
/** Where the warning's fix points (the "record them" section). */
|
|
48
|
+
exports.RECORD_CALL_DOCS_URL = "https://modelrig.dev/not-captured.html";
|
|
49
|
+
const REJECTED = { inferenceId: null, callId: null, costEstimateUsd: 0, pricingMissing: false };
|
|
50
|
+
const SERVICE_TIERS = new Set(["standard", "flex", "priority"]);
|
|
51
|
+
const FINISH_REASONS = new Set([
|
|
52
|
+
"stop", "length", "tool_calls", "content_filter", "refusal", "error", "other",
|
|
53
|
+
]);
|
|
54
|
+
const KNOWN_PROVIDERS = new Set(provider_1.PROVIDER_IDS);
|
|
55
|
+
/** Server ingest's route cap (`modelrig-server` ingest.ts `isNonEmptyString(raw.route, 200)`). */
|
|
56
|
+
exports.MAX_ROUTE_CHARS = 200;
|
|
57
|
+
/** Max distinct rules the once-per-rule warning latch remembers (L6 NIT-3). */
|
|
58
|
+
const WARN_LATCH_MAX = 100;
|
|
59
|
+
const warned = new Set();
|
|
60
|
+
/** Warn once per validation RULE. The latch key is the rule id — never a
|
|
61
|
+
* caller-supplied string — and the set is bounded, so a caller looping over bad
|
|
62
|
+
* input with varying values cannot grow it. */
|
|
63
|
+
function warnOnce(rule, problem, fix) {
|
|
64
|
+
if (warned.has(rule))
|
|
65
|
+
return;
|
|
66
|
+
if (warned.size < WARN_LATCH_MAX)
|
|
67
|
+
warned.add(rule);
|
|
68
|
+
console.warn(`[modelrig recordCall] ${problem} — nothing was recorded. fix: ${fix} (${exports.RECORD_CALL_DOCS_URL})`);
|
|
69
|
+
}
|
|
70
|
+
/** Test-only: clear the once-per-rule warning latch. */
|
|
71
|
+
function resetRecordCallWarningsForTest() {
|
|
72
|
+
warned.clear();
|
|
73
|
+
}
|
|
74
|
+
function isCount(v) {
|
|
75
|
+
return typeof v === "number" && Number.isFinite(v) && v >= 0;
|
|
76
|
+
}
|
|
77
|
+
/** The ceiling for latency/ttfb (Postgres `integer`, 2³¹−1 ms ≈ 24.8 days). */
|
|
78
|
+
exports.MAX_MS = 2147483647;
|
|
79
|
+
/**
|
|
80
|
+
* L6 MUST-3 integer hygiene. Telemetry columns are integers, but a caller's
|
|
81
|
+
* `performance.now()` delta is fractional — so ROUND to the nearest integer
|
|
82
|
+
* (friendlier than rejecting) and CLAMP: latency/ttfb to ≤ 2³¹−1 ms, token
|
|
83
|
+
* counts to ≤ Number.MAX_SAFE_INTEGER. Applied before any check that compares
|
|
84
|
+
* token counts, so validation and the recorded row see the same numbers.
|
|
85
|
+
*/
|
|
86
|
+
function toCount(v, max) {
|
|
87
|
+
return Math.min(Math.round(v), max);
|
|
88
|
+
}
|
|
89
|
+
const tok = (v) => (typeof v === "number" ? toCount(v, Number.MAX_SAFE_INTEGER) : 0);
|
|
90
|
+
function isNonEmpty(v, max) {
|
|
91
|
+
return typeof v === "string" && v.length > 0 && v.length <= max;
|
|
92
|
+
}
|
|
93
|
+
/** Validate the caller's input. Returns the failed rule, or null. */
|
|
94
|
+
function validate(input) {
|
|
95
|
+
const p = validateShape(input);
|
|
96
|
+
return p === null ? null : { rule: p.fix, ...p };
|
|
97
|
+
}
|
|
98
|
+
/** The rule checks. Every `fix` string is built only from SDK constants and
|
|
99
|
+
* field names (never from a caller value), so it doubles as the rule id. */
|
|
100
|
+
function validateShape(input) {
|
|
101
|
+
if (typeof input !== "object" || input === null) {
|
|
102
|
+
return { problem: "input is not an object", fix: "call rig.recordCall({ provider, model, usage, latencyMs })" };
|
|
103
|
+
}
|
|
104
|
+
const i = input;
|
|
105
|
+
if (!isNonEmpty(i.provider, 100)) {
|
|
106
|
+
return { problem: "provider is missing", fix: 'pass the provider id, e.g. provider: "gemini"' };
|
|
107
|
+
}
|
|
108
|
+
if (!isNonEmpty(i.model, 200)) {
|
|
109
|
+
return { problem: "model is missing", fix: 'pass the model id your code called, e.g. model: "gemini-3.1-pro-preview"' };
|
|
110
|
+
}
|
|
111
|
+
// L6 SHOULD-2: the row's route is `provider/model`; server ingest drops a
|
|
112
|
+
// route over 200 chars (ingest.ts), so refuse here instead of writing a row
|
|
113
|
+
// that would never arrive.
|
|
114
|
+
if (i.provider.length + 1 + i.model.length > exports.MAX_ROUTE_CHARS) {
|
|
115
|
+
return {
|
|
116
|
+
problem: `provider/model is longer than ${exports.MAX_ROUTE_CHARS} characters`,
|
|
117
|
+
fix: `shorten the provider or model id so "provider/model" is at most ${exports.MAX_ROUTE_CHARS} characters`,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const u = i.usage;
|
|
121
|
+
if (typeof u !== "object" || u === null || !isCount(u.tokensIn) || !isCount(u.tokensOut)) {
|
|
122
|
+
return {
|
|
123
|
+
problem: "usage.tokensIn / usage.tokensOut are missing or not non-negative numbers",
|
|
124
|
+
fix: "copy them from the provider response's usage block (Gemini usageMetadata.promptTokenCount / candidatesTokenCount; OpenAI usage.prompt_tokens / completion_tokens)",
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
for (const k of ["tokensCached", "tokensCacheWrite", "reasoningTokens"]) {
|
|
128
|
+
if (u[k] !== undefined && !isCount(u[k])) {
|
|
129
|
+
return { problem: `usage.${k} is not a non-negative number`, fix: `omit usage.${k} or pass a number ≥ 0` };
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
if (tok(u.tokensCached) + tok(u.tokensCacheWrite) > tok(u.tokensIn)) {
|
|
133
|
+
return {
|
|
134
|
+
problem: "usage.tokensCached + usage.tokensCacheWrite exceed usage.tokensIn",
|
|
135
|
+
fix: "tokensIn is the TOTAL input and includes cached and cache-write tokens (Anthropic reports input_tokens without them — add cache_read_input_tokens and cache_creation_input_tokens back)",
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
if (!isCount(i.latencyMs)) {
|
|
139
|
+
return { problem: "latencyMs is missing or not a non-negative number", fix: "measure the call's wall time in ms and pass latencyMs" };
|
|
140
|
+
}
|
|
141
|
+
if (i.ttfbMs !== undefined && !isCount(i.ttfbMs)) {
|
|
142
|
+
return { problem: "ttfbMs is not a non-negative number", fix: "omit ttfbMs or pass a number ≥ 0" };
|
|
143
|
+
}
|
|
144
|
+
for (const k of ["servedTier", "requestedTier"]) {
|
|
145
|
+
if (i[k] !== undefined && !(typeof i[k] === "string" && SERVICE_TIERS.has(i[k]))) {
|
|
146
|
+
return { problem: `${k} is not a service tier`, fix: `pass "standard", "flex" or "priority", or omit ${k}` };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (i.failureClass !== undefined && i.failureClass !== null) {
|
|
150
|
+
if (typeof i.failureClass !== "string" || !Object.prototype.hasOwnProperty.call(failures_teaching_1.FAILURE_TEACHING, i.failureClass)) {
|
|
151
|
+
return {
|
|
152
|
+
problem: `failureClass "${String(i.failureClass)}" is not a ModelRig failure class`,
|
|
153
|
+
fix: `use one of ${Object.keys(failures_teaching_1.FAILURE_TEACHING).join(", ")}, or null for a success`,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (i.webSearchCalls !== undefined && !(isCount(i.webSearchCalls) && Number.isInteger(i.webSearchCalls))) {
|
|
158
|
+
return { problem: "webSearchCalls is not a whole number ≥ 0", fix: "pass the count of billed web_search tool calls, or omit it" };
|
|
159
|
+
}
|
|
160
|
+
if (i.tags !== undefined) {
|
|
161
|
+
if (typeof i.tags !== "object" || i.tags === null || Array.isArray(i.tags)) {
|
|
162
|
+
return { problem: "tags is not an object", fix: 'pass tags as { key: "value" } strings' };
|
|
163
|
+
}
|
|
164
|
+
for (const [k, v] of Object.entries(i.tags)) {
|
|
165
|
+
if (v !== undefined && typeof v !== "string") {
|
|
166
|
+
return { problem: `tags.${k} is not a string`, fix: "tag values are strings — String() the value" };
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
for (const k of ["stepKey", "providerRequestId", "finishReason"]) {
|
|
171
|
+
if (i[k] !== undefined && typeof i[k] !== "string") {
|
|
172
|
+
return { problem: `${k} is not a string`, fix: `omit ${k} or pass a string` };
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
/** Normalize a caller finish reason: an enum value passes; a provider-native one
|
|
178
|
+
* maps through the adapters' own table; empty ⇒ absent. */
|
|
179
|
+
function finishReasonOf(provider, raw) {
|
|
180
|
+
if (raw === undefined || raw === "")
|
|
181
|
+
return undefined;
|
|
182
|
+
if (FINISH_REASONS.has(raw))
|
|
183
|
+
return raw;
|
|
184
|
+
return (0, attempt_meta_map_1.mapFinishReason)((KNOWN_PROVIDERS.has(provider) ? provider : "openai"), raw);
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Record one call the caller made itself. See the module header for the
|
|
188
|
+
* contract. `deps` is the rig's raw-lane deps (telemetry + unpriced rate +
|
|
189
|
+
* manifest) — the same object `runRaw` receives.
|
|
190
|
+
*/
|
|
191
|
+
function recordCall(deps, input) {
|
|
192
|
+
try {
|
|
193
|
+
const bad = validate(input);
|
|
194
|
+
if (bad !== null) {
|
|
195
|
+
warnOnce(bad.rule, bad.problem, bad.fix);
|
|
196
|
+
return REJECTED;
|
|
197
|
+
}
|
|
198
|
+
return record(deps, input);
|
|
199
|
+
}
|
|
200
|
+
catch (err) {
|
|
201
|
+
// Instrumentation never throws into customer code (P4 graceful degradation).
|
|
202
|
+
warnOnce("unexpected", `unexpected error (${err instanceof Error ? err.message : String(err)})`, "report this to ModelRig");
|
|
203
|
+
return REJECTED;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
function record(deps, input) {
|
|
207
|
+
// Same ALS-boundary read + step key resolution + dimension hygiene as runRaw.
|
|
208
|
+
const step = (0, raw_finalize_1.prepareRawStep)(deps, input);
|
|
209
|
+
const provider = input.provider;
|
|
210
|
+
const known = KNOWN_PROVIDERS.has(provider);
|
|
211
|
+
const rawModelId = `${provider}/${input.model}`;
|
|
212
|
+
const servedTier = input.servedTier ?? "standard";
|
|
213
|
+
const requestedTier = input.requestedTier ?? servedTier;
|
|
214
|
+
// MUST-3: integers only on the row (rounded + clamped — see toCount).
|
|
215
|
+
const usage = {
|
|
216
|
+
tokensIn: tok(input.usage.tokensIn),
|
|
217
|
+
tokensOut: tok(input.usage.tokensOut),
|
|
218
|
+
tokensCached: tok(input.usage.tokensCached),
|
|
219
|
+
...(input.usage.tokensCacheWrite !== undefined ? { tokensCacheWrite: tok(input.usage.tokensCacheWrite) } : {}),
|
|
220
|
+
servedTier,
|
|
221
|
+
...(input.ttfbMs !== undefined ? { ttfbMs: toCount(input.ttfbMs, exports.MAX_MS) } : {}),
|
|
222
|
+
};
|
|
223
|
+
const latencyMs = toCount(input.latencyMs, exports.MAX_MS);
|
|
224
|
+
// Priced exactly like the raw lane: list price at the SERVED tier. A provider
|
|
225
|
+
// ModelRig has no price table for is recorded honestly at $0, flagged.
|
|
226
|
+
const unpricedRatePerTok = (deps.unpricedRateUsdPerMTok ?? cost_1.DEFAULT_UNPRICED_RATE_USD_PER_MTOK) / 1000000;
|
|
227
|
+
const { costUsd, pricingMissing } = known
|
|
228
|
+
? (0, cost_1.attemptCost)(provider, input.model, usage, unpricedRatePerTok)
|
|
229
|
+
: { costUsd: 0, pricingMissing: true };
|
|
230
|
+
// Tags: caller tags + the ambient step/run_id (same order as finalizeRaw), then
|
|
231
|
+
// the SDK-owned markers LAST so a caller value cannot override them.
|
|
232
|
+
const stepTag = step !== null ? { step: step.stepKey } : {};
|
|
233
|
+
const tags = {
|
|
234
|
+
...stepTag,
|
|
235
|
+
...(0, run_context_1.withRunIdTag)(input.tags ?? {}, step?.runCtx ?? null),
|
|
236
|
+
key_ownership: "byok",
|
|
237
|
+
lane: lanes_1.EXTERNAL_LANE,
|
|
238
|
+
};
|
|
239
|
+
const finishReason = finishReasonOf(provider, input.finishReason);
|
|
240
|
+
const adapterMeta = {
|
|
241
|
+
...(finishReason !== undefined ? { finish_reason: finishReason } : {}),
|
|
242
|
+
...(input.providerRequestId !== undefined ? { provider_request_id: input.providerRequestId } : {}),
|
|
243
|
+
...(input.usage.reasoningTokens !== undefined ? { reasoning_tokens: tok(input.usage.reasoningTokens) } : {}),
|
|
244
|
+
};
|
|
245
|
+
// No rendered request and no output text: ModelRig never saw the content, so
|
|
246
|
+
// there is no input/output hash. Cost components only for a priced provider.
|
|
247
|
+
const meta = (0, attempt_meta_1.summarizeAttemptMeta)({
|
|
248
|
+
adapterMeta,
|
|
249
|
+
provenance: "caller-reported",
|
|
250
|
+
...(known ? { provider: provider, model: input.model } : {}),
|
|
251
|
+
usage,
|
|
252
|
+
});
|
|
253
|
+
const id = (0, node_crypto_1.randomUUID)();
|
|
254
|
+
const callId = (0, node_crypto_1.randomUUID)();
|
|
255
|
+
const failureClass = input.failureClass ?? null;
|
|
256
|
+
(0, raw_finalize_1.recordRawRow)(deps.telemetry, {
|
|
257
|
+
id,
|
|
258
|
+
rawModelId,
|
|
259
|
+
provider,
|
|
260
|
+
model: input.model,
|
|
261
|
+
usage,
|
|
262
|
+
costUsd,
|
|
263
|
+
latencyMs,
|
|
264
|
+
requestedTier,
|
|
265
|
+
failureClass,
|
|
266
|
+
tags,
|
|
267
|
+
pricingMissing,
|
|
268
|
+
callId,
|
|
269
|
+
meta,
|
|
270
|
+
});
|
|
271
|
+
(0, raw_finalize_1.emitRawStep)(step, input.tags?.step ?? rawModelId, id, failureClass);
|
|
272
|
+
const searchCalls = input.webSearchCalls ?? 0;
|
|
273
|
+
if (searchCalls >= 1) {
|
|
274
|
+
(0, raw_search_cost_1.recordRawSearchCostRow)(deps.telemetry, { provider, rawModelId, tags, callId, searchCalls });
|
|
275
|
+
}
|
|
276
|
+
return { inferenceId: id, callId, costEstimateUsd: costUsd, pricingMissing };
|
|
277
|
+
}
|
|
@@ -124,7 +124,15 @@ export interface AttemptMetaV1 {
|
|
|
124
124
|
/** QA-S7 (issue #1, plan §7.3): the route's output contract on this attempt —
|
|
125
125
|
* `json` | `text`. Route lane only; absent on raw rows. */
|
|
126
126
|
readonly output_kind?: OutputKind;
|
|
127
|
+
/** OBS-1: where this row's numbers came from. `caller-reported` ⇒ the
|
|
128
|
+
* customer's own code made the provider call and reported its usage through
|
|
129
|
+
* `rig.recordCall()`; ModelRig neither dispatched nor validated it. Absent ⇒
|
|
130
|
+
* ModelRig dispatched the attempt and measured it (every other row). Additive
|
|
131
|
+
* — no version bump. */
|
|
132
|
+
readonly provenance?: AttemptProvenance;
|
|
127
133
|
}
|
|
134
|
+
/** OBS-1: the one non-default provenance value (see {@link AttemptMetaV1.provenance}). */
|
|
135
|
+
export type AttemptProvenance = "caller-reported";
|
|
128
136
|
export interface RunMetaV1 {
|
|
129
137
|
readonly v: 1;
|
|
130
138
|
readonly pipeline_sha?: string;
|
|
@@ -194,6 +202,9 @@ export interface AttemptMetaInputs {
|
|
|
194
202
|
* through verbatim (already counts/enums/hashes — no free text). Absent ⇒ no
|
|
195
203
|
* `screen` key ⇒ byte-identical to a non-screened row. */
|
|
196
204
|
readonly screen?: ScreenReceipt;
|
|
205
|
+
/** OBS-1: stamp `meta.provenance` (`rig.recordCall` rows only). Absent ⇒ no
|
|
206
|
+
* key ⇒ byte-identical to every dispatched row. */
|
|
207
|
+
readonly provenance?: AttemptProvenance;
|
|
197
208
|
}
|
|
198
209
|
/** The classification-rule string test (spec §2, §5 wire backstop): a value is
|
|
199
210
|
* an ENUM / IDENTIFIER / HASH only if it is at most 128 chars AND contains no
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attempt-meta.d.ts","sourceRoot":"","sources":["../../src/telemetry/attempt-meta.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAOH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,QAAQ,GACR,YAAY,GACZ,gBAAgB,GAChB,SAAS,GACT,OAAO,GACP,OAAO,CAAC;AAEZ,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAExE;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,MAAM,GAAG,YAAY,CAAC;AAC9D,eAAO,MAAM,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAA6D,CAAC;AAEhH;;;;yDAIyD;AACzD,eAAO,MAAM,YAAY,EAAE,WAAW,CAAC,MAAM,CAAyC,CAAC;AAEvF,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,CAAC,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACnG,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,oBAAoB,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC;IACd,QAAQ,CAAC,aAAa,CAAC,EAAE,YAAY,CAAC;IACtC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACpH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,8EAA8E;IAC9E,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACnF,QAAQ,CAAC,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;iBAKa;IACb,QAAQ,CAAC,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IAC/C,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE;QACd,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,8DAA8D;IAC9D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;oFAGgF;IAChF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC;IACxC,QAAQ,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC;IAClC;;;qFAGiF;IACjF,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;mBAMe;IACf,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;IAChC;yEACqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC;IAC1C;+DAC2D;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"attempt-meta.d.ts","sourceRoot":"","sources":["../../src/telemetry/attempt-meta.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAOH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErD,MAAM,MAAM,YAAY,GACpB,MAAM,GACN,QAAQ,GACR,YAAY,GACZ,gBAAgB,GAChB,SAAS,GACT,OAAO,GACP,OAAO,CAAC;AAEZ,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAExE;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,MAAM,GAAG,YAAY,CAAC;AAC9D,eAAO,MAAM,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAA6D,CAAC;AAEhH;;;;yDAIyD;AACzD,eAAO,MAAM,YAAY,EAAE,WAAW,CAAC,MAAM,CAAyC,CAAC;AAEvF,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,CAAC,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACnG,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,oBAAoB,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC;IACd,QAAQ,CAAC,aAAa,CAAC,EAAE,YAAY,CAAC;IACtC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACpH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,8EAA8E;IAC9E,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACnF,QAAQ,CAAC,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;iBAKa;IACb,QAAQ,CAAC,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IAC/C,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE;QACd,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,8DAA8D;IAC9D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;oFAGgF;IAChF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC;IACxC,QAAQ,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC;IAClC;;;qFAGiF;IACjF,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;mBAMe;IACf,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;IAChC;yEACqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,cAAc,CAAC;IAC1C;+DAC2D;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,UAAU,CAAC;IAClC;;;;4BAIwB;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,iBAAiB,CAAC;CACzC;AAED,0FAA0F;AAC1F,MAAM,MAAM,iBAAiB,GAAG,iBAAiB,CAAC;AAElD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC;IACd,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IAClE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC9C,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/C,4DAA4D;IAC5D,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAED;;;;uEAIuE;AACvE,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC,GAAG,MAAM,CAMT;AAED;;uEAEuE;AACvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC9C,4EAA4E;IAC5E,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,8DAA8D;IAC9D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/C;;gEAE4D;IAC5D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;0DACsD;IACtD,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC,qDAAqD;IACrD,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC;IAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,CACnB,YAAY,EACZ,UAAU,GAAG,WAAW,GAAG,cAAc,GAAG,kBAAkB,GAAG,oBAAoB,CACtF;IACC,+EAA+E;IAC/E,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC,CAAC;IAC5C,mEAAmE;IACnE,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,yDAAyD;IACzD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;qFACiF;IACjF,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB;;8DAE0D;IAC1D,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;IAChC;uDACmD;IACnD,QAAQ,CAAC,UAAU,CAAC,EAAE,iBAAiB,CAAC;CACzC;AAID;;;;+BAI+B;AAC/B,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,CAExD;AAOD;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAgCrG;AAID;gEACgE;AAChE,eAAO,MAAM,iBAAiB,mEAAoE,CAAC;AACnG,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AA4BhE;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,cAAc,CAAA;CAAE,EAAE,CAS5G;AAID;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAInE;AAED;;;2CAG2C;AAC3C,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAkBrE;AAwCD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,iBAAiB,GAAG,aAAa,GAAG,IAAI,CA6GpF"}
|
|
@@ -270,6 +270,9 @@ function summarizeAttemptMeta(inputs) {
|
|
|
270
270
|
if (typeof inputs.outputKind === "string" && exports.OUTPUT_KINDS.has(inputs.outputKind)) {
|
|
271
271
|
meta.output_kind = inputs.outputKind;
|
|
272
272
|
}
|
|
273
|
+
// OBS-1: caller-reported provenance — an enum; anything else is dropped.
|
|
274
|
+
if (inputs.provenance === "caller-reported")
|
|
275
|
+
meta.provenance = inputs.provenance;
|
|
273
276
|
// Externally-sourced identifiers — gated by the classification rule.
|
|
274
277
|
if (isMetaSafeString(a.model_snapshot))
|
|
275
278
|
meta.model_snapshot = a.model_snapshot;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ONE lane classifier every read surface uses (OBS-1). A telemetry row's
|
|
3
|
+
* `tags.lane` names how ModelRig came to hold it:
|
|
4
|
+
* • absent / anything else — a ROUTED call (it walked a route's ladder; its
|
|
5
|
+
* quality signal is schema conformance).
|
|
6
|
+
* • `"raw"` — a raw-lane BYOK passthrough ModelRig dispatched but never
|
|
7
|
+
* validated (`rig.runRaw`).
|
|
8
|
+
* • `"external"` — a RECORDED call: the customer's own code made it and
|
|
9
|
+
* reported it through `rig.recordCall`; ModelRig neither dispatched nor
|
|
10
|
+
* validated it.
|
|
11
|
+
* Raw and external rows are never routed: they never enter ladder or
|
|
12
|
+
* conformance math, and their quality signal is `rig.grade`.
|
|
13
|
+
*
|
|
14
|
+
* Pure, dependency-free (the console imports it through `@modelrig-core`).
|
|
15
|
+
* The billing path (`modelrig-server` usage-aggregation `laneOf`) deliberately
|
|
16
|
+
* keeps its own literal so a money decision imports nothing from here.
|
|
17
|
+
*/
|
|
18
|
+
export declare const RAW_LANE = "raw";
|
|
19
|
+
export declare const EXTERNAL_LANE = "external";
|
|
20
|
+
/** The call lane a read surface renders. */
|
|
21
|
+
export type CallLane = "routes" | "raw" | "external";
|
|
22
|
+
/** True when a row with this `tags.lane` walked a route (enters ladder and
|
|
23
|
+
* conformance math). False for `raw` and `external`. */
|
|
24
|
+
export declare function isRoutedLane(lane: unknown): boolean;
|
|
25
|
+
/** Three-way classification of a `tags.lane` value. */
|
|
26
|
+
export declare function callLaneOf(lane: unknown): CallLane;
|
|
27
|
+
/** Console badge copy for a recorded (external) call. User-facing copy says
|
|
28
|
+
* "recorded call", never "lane". */
|
|
29
|
+
export declare const RECORDED_BADGE = "recorded \u00B7 unverified";
|
|
30
|
+
export declare const RECORDED_TOOLTIP = "Your code made this call; ModelRig recorded what you reported (tokens, tier, latency). It did not dispatch or validate it.";
|
|
31
|
+
//# sourceMappingURL=lanes.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lanes.d.ts","sourceRoot":"","sources":["../../src/telemetry/lanes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,eAAO,MAAM,QAAQ,QAAQ,CAAC;AAC9B,eAAO,MAAM,aAAa,aAAa,CAAC;AAExC,4CAA4C;AAC5C,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,KAAK,GAAG,UAAU,CAAC;AAErD;wDACwD;AACxD,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAEnD;AAED,uDAAuD;AACvD,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,QAAQ,CAIlD;AAED;oCACoC;AACpC,eAAO,MAAM,cAAc,+BAA0B,CAAC;AACtD,eAAO,MAAM,gBAAgB,+HACiG,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The ONE lane classifier every read surface uses (OBS-1). A telemetry row's
|
|
4
|
+
* `tags.lane` names how ModelRig came to hold it:
|
|
5
|
+
* • absent / anything else — a ROUTED call (it walked a route's ladder; its
|
|
6
|
+
* quality signal is schema conformance).
|
|
7
|
+
* • `"raw"` — a raw-lane BYOK passthrough ModelRig dispatched but never
|
|
8
|
+
* validated (`rig.runRaw`).
|
|
9
|
+
* • `"external"` — a RECORDED call: the customer's own code made it and
|
|
10
|
+
* reported it through `rig.recordCall`; ModelRig neither dispatched nor
|
|
11
|
+
* validated it.
|
|
12
|
+
* Raw and external rows are never routed: they never enter ladder or
|
|
13
|
+
* conformance math, and their quality signal is `rig.grade`.
|
|
14
|
+
*
|
|
15
|
+
* Pure, dependency-free (the console imports it through `@modelrig-core`).
|
|
16
|
+
* The billing path (`modelrig-server` usage-aggregation `laneOf`) deliberately
|
|
17
|
+
* keeps its own literal so a money decision imports nothing from here.
|
|
18
|
+
*/
|
|
19
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
|
+
exports.RECORDED_TOOLTIP = exports.RECORDED_BADGE = exports.EXTERNAL_LANE = exports.RAW_LANE = void 0;
|
|
21
|
+
exports.isRoutedLane = isRoutedLane;
|
|
22
|
+
exports.callLaneOf = callLaneOf;
|
|
23
|
+
exports.RAW_LANE = "raw";
|
|
24
|
+
exports.EXTERNAL_LANE = "external";
|
|
25
|
+
/** True when a row with this `tags.lane` walked a route (enters ladder and
|
|
26
|
+
* conformance math). False for `raw` and `external`. */
|
|
27
|
+
function isRoutedLane(lane) {
|
|
28
|
+
return lane !== exports.RAW_LANE && lane !== exports.EXTERNAL_LANE;
|
|
29
|
+
}
|
|
30
|
+
/** Three-way classification of a `tags.lane` value. */
|
|
31
|
+
function callLaneOf(lane) {
|
|
32
|
+
if (lane === exports.RAW_LANE)
|
|
33
|
+
return "raw";
|
|
34
|
+
if (lane === exports.EXTERNAL_LANE)
|
|
35
|
+
return "external";
|
|
36
|
+
return "routes";
|
|
37
|
+
}
|
|
38
|
+
/** Console badge copy for a recorded (external) call. User-facing copy says
|
|
39
|
+
* "recorded call", never "lane". */
|
|
40
|
+
exports.RECORDED_BADGE = "recorded · unverified";
|
|
41
|
+
exports.RECORDED_TOOLTIP = "Your code made this call; ModelRig recorded what you reported (tokens, tier, latency). It did not dispatch or validate it.";
|
|
@@ -33,8 +33,11 @@ export interface OrgIdentityInput {
|
|
|
33
33
|
/** Override WHOAMI_TIMEOUT_MS (tests). */
|
|
34
34
|
readonly timeoutMs?: number;
|
|
35
35
|
}
|
|
36
|
-
/** Build the one-line org banner from a successful whoami.
|
|
37
|
-
|
|
36
|
+
/** Build the one-line org banner from a successful whoami. Only an ok result
|
|
37
|
+
* reaches here, and an ok result always NAMES its org (QA N14: a 2xx without
|
|
38
|
+
* one is not ok), so `orgName` is required. Never prints a placeholder glyph:
|
|
39
|
+
* the slug is shown only when the server sent one that differs from the name. */
|
|
40
|
+
export declare function formatOrgLine(orgName: string, result: PreflightResult, apiKey: string): string;
|
|
38
41
|
/**
|
|
39
42
|
* Record one successful flush. Returns immediately; the whoami (first flush
|
|
40
43
|
* only) runs in the background. Exposed as a promise-returning helper ONLY so
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"org-identity.d.ts","sourceRoot":"","sources":["../../src/telemetry/org-identity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAAwC,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGjG;oEACoE;AACpE,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAYtC,kFAAkF;AAClF,wBAAgB,yBAAyB,IAAI,IAAI,CAEhD;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,KAAK,CAAC;IAC/B,0CAA0C;IAC1C,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED
|
|
1
|
+
{"version":3,"file":"org-identity.d.ts","sourceRoot":"","sources":["../../src/telemetry/org-identity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAAwC,KAAK,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGjG;oEACoE;AACpE,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAYtC,kFAAkF;AAClF,wBAAgB,yBAAyB,IAAI,IAAI,CAEhD;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,KAAK,CAAC;IAC/B,0CAA0C;IAC1C,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;iFAGiF;AACjF,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAM9F;AAgBD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+B1E"}
|
|
@@ -36,14 +36,16 @@ const states = new Map();
|
|
|
36
36
|
function __resetOrgIdentityForTest() {
|
|
37
37
|
states.clear();
|
|
38
38
|
}
|
|
39
|
-
/** Build the one-line org banner from a successful whoami.
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
/** Build the one-line org banner from a successful whoami. Only an ok result
|
|
40
|
+
* reaches here, and an ok result always NAMES its org (QA N14: a 2xx without
|
|
41
|
+
* one is not ok), so `orgName` is required. Never prints a placeholder glyph:
|
|
42
|
+
* the slug is shown only when the server sent one that differs from the name. */
|
|
43
|
+
function formatOrgLine(orgName, result, apiKey) {
|
|
44
|
+
const slugRaw = result.whoami?.org?.slug;
|
|
45
|
+
const slug = typeof slugRaw === "string" && slugRaw !== "" && slugRaw !== orgName ? ` (${slugRaw})` : "";
|
|
44
46
|
const keyName = result.whoami?.key?.name;
|
|
45
47
|
const keyLabel = typeof keyName === "string" && keyName !== "" ? `«${keyName}» ` : "";
|
|
46
|
-
return `[modelrig] exporting to org «${
|
|
48
|
+
return `[modelrig] exporting to org «${orgName}»${slug} as key ${keyLabel}${(0, api_key_format_1.redactApiKey)(apiKey)}`;
|
|
47
49
|
}
|
|
48
50
|
function mismatchLine(actual, expectOrg) {
|
|
49
51
|
return (`[modelrig] ORG MISMATCH: key resolves to «${actual}», expectOrg is «${expectOrg}» — ` +
|
|
@@ -81,10 +83,10 @@ function noteSuccessfulFlush(input) {
|
|
|
81
83
|
if (s.flushes === 1) {
|
|
82
84
|
return (0, doctor_preflight_1.whoamiPreflight)(input.ingestUrl, input.apiKey, input.fetchFn, AbortSignal.timeout(input.timeoutMs ?? exports.WHOAMI_TIMEOUT_MS))
|
|
83
85
|
.then((result) => {
|
|
84
|
-
if (!result.ok)
|
|
86
|
+
if (!result.ok || result.orgName === undefined)
|
|
85
87
|
return; // silent: a 401 has its own loud line
|
|
86
88
|
s.resolved = result;
|
|
87
|
-
console.log(formatOrgLine(result, input.apiKey));
|
|
89
|
+
console.log(formatOrgLine(result.orgName, result, input.apiKey));
|
|
88
90
|
warnIfMismatch(result, input.expectOrg);
|
|
89
91
|
})
|
|
90
92
|
.catch(() => {
|
|
@@ -109,9 +109,11 @@ await rig.close(); // flushes the exporter — call once at shutdown
|
|
|
109
109
|
<!-- /BLOCK:RUN_WRAP -->
|
|
110
110
|
|
|
111
111
|
<!-- BLOCK:NOT_CAPTURED -->
|
|
112
|
-
**What ModelRig does not see.** ModelRig records the calls your code makes through the SDK (`rig.run`, `rig.runBundle`, `rig.runRaw`, `rig.runRawStream`) or the gateway, and
|
|
112
|
+
**What ModelRig does not see unless you record it.** ModelRig records the calls your code makes through the SDK (`rig.run`, `rig.runBundle`, `rig.runRaw`, `rig.runRawStream`) or the gateway, plus any call your code makes itself and reports with `rig.recordCall`. Nothing else is seen. Five things stay invisible or unattached unless you record them: (1) direct provider SDK calls, with or without a JSON schema; (2) streams you read from a provider client yourself; (3) a fallback your code makes after the raw lane refuses a `responseSchema` it cannot enforce (deepseek, deepinfra, fireworks and zai fail closed with `invariant_violation`); (4) calls made outside `run.scope` / `run.start`, which land on /inferences but on no run; (5) embeddings. A recorded call shows as "recorded · unverified": ModelRig neither dispatched nor validated it, and its tokens are the numbers you reported. Details and the console tells: https://modelrig.dev/not-captured.html.
|
|
113
113
|
<!-- /BLOCK:NOT_CAPTURED -->
|
|
114
114
|
|
|
115
|
+
A provider call you can't move onto a lane (an unsupported client, a stream you read yourself, a fallback): record it with `rig.recordCall({ provider, model, usage, servedTier, latencyMs, stepKey })` right after it returns, never leave it invisible (modelrig ≥ 0.11.0; https://modelrig.dev/not-captured.html).
|
|
116
|
+
|
|
115
117
|
Which lane fits a codebase (observe, the `runRaw` seam, or named routes):
|
|
116
118
|
<https://modelrig.dev/choosing-a-lane.html>. Hosted workers:
|
|
117
119
|
<https://modelrig.dev/deploying.html>.
|
package/docs/instrumentation.md
CHANGED
|
@@ -115,6 +115,23 @@ right for one run at a time on a request/async path.
|
|
|
115
115
|
and honest. Never route a stage through the rig just to mint a step row; the
|
|
116
116
|
console renders "cost n/a" for un-routed steps rather than inventing a number.
|
|
117
117
|
|
|
118
|
+
### Record a call your code made itself (`rig.recordCall`)
|
|
119
|
+
|
|
120
|
+
_Needs `modelrig` 0.11.0 or newer._
|
|
121
|
+
|
|
122
|
+
If a stage's model call has to stay a direct provider call, report it instead:
|
|
123
|
+
`rig.recordCall({ provider, model, usage: { tokensIn, tokensOut, tokensCached }, servedTier, latencyMs, stepKey })`
|
|
124
|
+
right after the call returns. Inside a run context it records a step row for
|
|
125
|
+
that `stepKey` (stored as `ground-truth`, the only joined-inference value),
|
|
126
|
+
joined to the recorded inference, so the step gets a cost instead of "cost n/a".
|
|
127
|
+
The console shows that step and its call as **recorded · unverified**, never as
|
|
128
|
+
measured:
|
|
129
|
+
ModelRig neither dispatched nor validated the call, and the tokens are the ones
|
|
130
|
+
you reported. It is synchronous and never throws (bad input warns and returns
|
|
131
|
+
`inferenceId: null`); the returned `inferenceId` is gradeable with `rig.grade`.
|
|
132
|
+
Full contract, usage convention and cost notes:
|
|
133
|
+
[Not captured → record them](not-captured.md#calls-your-code-makes-itself-record-them).
|
|
134
|
+
|
|
118
135
|
## Tags, dimensions, and the graph
|
|
119
136
|
|
|
120
137
|
ModelRig has **two** grouping mechanisms, and they compose:
|