@wildorder/nightshift 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -2
- package/dist/agent-runner.d.ts +56 -4
- package/dist/agent-runner.d.ts.map +1 -1
- package/dist/agent-runner.js +253 -28
- package/dist/agent-runner.js.map +1 -1
- package/dist/author.d.ts +12 -0
- package/dist/author.d.ts.map +1 -1
- package/dist/author.js +183 -88
- package/dist/author.js.map +1 -1
- package/dist/causal-analysis.d.ts +212 -0
- package/dist/causal-analysis.d.ts.map +1 -0
- package/dist/causal-analysis.js +733 -0
- package/dist/causal-analysis.js.map +1 -0
- package/dist/decider-review.d.ts +6 -1
- package/dist/decider-review.d.ts.map +1 -1
- package/dist/decider-review.js +19 -7
- package/dist/decider-review.js.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/preflight.d.ts +3 -0
- package/dist/preflight.d.ts.map +1 -1
- package/dist/preflight.js +73 -57
- package/dist/preflight.js.map +1 -1
- package/dist/prompt-telemetry.d.ts +64 -0
- package/dist/prompt-telemetry.d.ts.map +1 -0
- package/dist/prompt-telemetry.js +112 -0
- package/dist/prompt-telemetry.js.map +1 -0
- package/dist/provider-telemetry.d.ts +106 -0
- package/dist/provider-telemetry.d.ts.map +1 -0
- package/dist/provider-telemetry.js +423 -0
- package/dist/provider-telemetry.js.map +1 -0
- package/dist/run-analytics-report.d.ts +173 -0
- package/dist/run-analytics-report.d.ts.map +1 -0
- package/dist/run-analytics-report.js +650 -0
- package/dist/run-analytics-report.js.map +1 -0
- package/dist/run-analytics.d.ts +738 -0
- package/dist/run-analytics.d.ts.map +1 -0
- package/dist/run-analytics.js +545 -0
- package/dist/run-analytics.js.map +1 -0
- package/dist/run-program.d.ts +69 -1
- package/dist/run-program.d.ts.map +1 -1
- package/dist/run-program.js +1263 -716
- package/dist/run-program.js.map +1 -1
- package/dist/whole-program-review.d.ts +3 -0
- package/dist/whole-program-review.d.ts.map +1 -1
- package/dist/whole-program-review.js +8 -1
- package/dist/whole-program-review.js.map +1 -1
- package/package.json +3 -3
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic, model-free measurement of what a brief actually asked a
|
|
3
|
+
* provider to read (WS-03, SC-07) — taken at assembly time, before any
|
|
4
|
+
* provider is involved. Pure and side-effect-free throughout.
|
|
5
|
+
*/
|
|
6
|
+
export const PROMPT_COMPONENT_SIZE_KIND = "prompt-component-size";
|
|
7
|
+
/**
|
|
8
|
+
* A coarse, deterministic bytes-to-tokens heuristic (~4 UTF-8 bytes per
|
|
9
|
+
* token of English prose) — an estimate, never mistaken for a
|
|
10
|
+
* provider-reported count, hence always emitted with `coverage: "estimated"`.
|
|
11
|
+
*/
|
|
12
|
+
function estimateTokensFromBytes(bytes) {
|
|
13
|
+
return Math.ceil(bytes / 4);
|
|
14
|
+
}
|
|
15
|
+
function sizeOf(label, bytes) {
|
|
16
|
+
return { label, bytes, estimatedTokens: estimateTokensFromBytes(bytes) };
|
|
17
|
+
}
|
|
18
|
+
export function createBriefBuilder() {
|
|
19
|
+
const lines = [];
|
|
20
|
+
const segments = [];
|
|
21
|
+
return {
|
|
22
|
+
push(label, ...newLines) {
|
|
23
|
+
if (newLines.length === 0)
|
|
24
|
+
return;
|
|
25
|
+
segments.push({ label, start: lines.length });
|
|
26
|
+
lines.push(...newLines);
|
|
27
|
+
},
|
|
28
|
+
join() {
|
|
29
|
+
return lines.join("\n");
|
|
30
|
+
},
|
|
31
|
+
components() {
|
|
32
|
+
const bytesByLabel = new Map();
|
|
33
|
+
segments.forEach((segment, index) => {
|
|
34
|
+
const end = index + 1 < segments.length ? segments[index + 1].start : lines.length;
|
|
35
|
+
const slice = lines.slice(segment.start, end);
|
|
36
|
+
const text = (segment.start === 0 ? "" : "\n") + slice.join("\n");
|
|
37
|
+
const bytes = Buffer.byteLength(text, "utf8");
|
|
38
|
+
bytesByLabel.set(segment.label, (bytesByLabel.get(segment.label) ?? 0) + bytes);
|
|
39
|
+
});
|
|
40
|
+
return [...bytesByLabel.entries()].map(([label, bytes]) => sizeOf(label, bytes));
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Two points per component — exact bytes (`observed`) and the derived
|
|
46
|
+
* estimated-token figure (`estimated`) — never a compound object on one
|
|
47
|
+
* point, per WS-03's scalar-per-fact vocabulary.
|
|
48
|
+
*/
|
|
49
|
+
export function promptComponentSizePoints(sizes, dimensions) {
|
|
50
|
+
return sizes.flatMap((size) => [
|
|
51
|
+
{
|
|
52
|
+
kind: PROMPT_COMPONENT_SIZE_KIND,
|
|
53
|
+
label: size.label,
|
|
54
|
+
coverage: "observed",
|
|
55
|
+
value: size.bytes,
|
|
56
|
+
unit: "bytes",
|
|
57
|
+
dimensions,
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
kind: PROMPT_COMPONENT_SIZE_KIND,
|
|
61
|
+
label: size.label,
|
|
62
|
+
coverage: "estimated",
|
|
63
|
+
value: size.estimatedTokens,
|
|
64
|
+
unit: "tokens",
|
|
65
|
+
dimensions,
|
|
66
|
+
},
|
|
67
|
+
]);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The reclaimed-bytes point for dependency specs demoted to a one-line stub
|
|
71
|
+
* because the included set exceeded the author's dependency-spec budget
|
|
72
|
+
* (§2.3) — the bytes a demoted spec would have cost, not bytes the brief
|
|
73
|
+
* actually spent, so it is measured and emitted separately from
|
|
74
|
+
* `promptComponentSizePoints`'s totality-checked set rather than folded into
|
|
75
|
+
* it. `undefined` when nothing was demoted — an unexercised seam emits
|
|
76
|
+
* nothing, never a fabricated zero.
|
|
77
|
+
*/
|
|
78
|
+
export function demotedDependenciesPoint(demotedIds, reclaimedBytes, dimensions) {
|
|
79
|
+
if (demotedIds.length === 0)
|
|
80
|
+
return undefined;
|
|
81
|
+
return {
|
|
82
|
+
kind: PROMPT_COMPONENT_SIZE_KIND,
|
|
83
|
+
label: "demoted-dependencies",
|
|
84
|
+
coverage: "observed",
|
|
85
|
+
value: reclaimedBytes,
|
|
86
|
+
unit: "bytes",
|
|
87
|
+
detail: demotedIds.join(","),
|
|
88
|
+
dimensions,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* The exact UTF-8 bytes a reviewer did not see at one of the three
|
|
93
|
+
* char-based clip seams — measured from the actual full and shown strings,
|
|
94
|
+
* never inferred from the char limit or a boolean. `seam` names the call
|
|
95
|
+
* site (e.g. `"test-critique-diff"`, `"decider-diff"`,
|
|
96
|
+
* `"whole-program-diff"`) and becomes `clipped-input:<seam>`'s label.
|
|
97
|
+
*/
|
|
98
|
+
export function clippedInputPoint(seam, full, shown, dimensions) {
|
|
99
|
+
const fullBytes = Buffer.byteLength(full, "utf8");
|
|
100
|
+
const shownBytes = Buffer.byteLength(shown, "utf8");
|
|
101
|
+
const droppedBytes = Math.max(0, fullBytes - shownBytes);
|
|
102
|
+
return {
|
|
103
|
+
kind: PROMPT_COMPONENT_SIZE_KIND,
|
|
104
|
+
label: `clipped-input:${seam}`,
|
|
105
|
+
coverage: "observed",
|
|
106
|
+
value: droppedBytes,
|
|
107
|
+
unit: "bytes",
|
|
108
|
+
...(droppedBytes > 0 ? { detail: "clipped" } : {}),
|
|
109
|
+
dimensions,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=prompt-telemetry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prompt-telemetry.js","sourceRoot":"","sources":["../src/prompt-telemetry.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,uBAAgC,CAAC;AAS3E;;;;GAIG;AACH,SAAS,uBAAuB,CAAC,KAAa;IAC5C,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,KAAa;IAC1C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,uBAAuB,CAAC,KAAK,CAAC,EAAE,CAAC;AAC3E,CAAC;AA4BD,MAAM,UAAU,kBAAkB;IAChC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAA4C,EAAE,CAAC;IAE7D,OAAO;QACL,IAAI,CAAC,KAAK,EAAE,GAAG,QAAQ;YACrB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAClC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9C,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC1B,CAAC;QACD,IAAI;YACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QACD,UAAU;YACR,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC/C,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;gBAClC,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;gBACpF,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAC9C,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClE,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC9C,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;YAClF,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QACnF,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CACvC,KAAqC,EACrC,UAA+B;IAE/B,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC7B;YACE,IAAI,EAAE,0BAA0B;YAChC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,UAAmB;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,OAAO;YACb,UAAU;SACX;QACD;YACE,IAAI,EAAE,0BAA0B;YAChC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,WAAoB;YAC9B,KAAK,EAAE,IAAI,CAAC,eAAe;YAC3B,IAAI,EAAE,QAAQ;YACd,UAAU;SACX;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,wBAAwB,CACtC,UAA6B,EAC7B,cAAsB,EACtB,UAA+B;IAE/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,OAAO;QACL,IAAI,EAAE,0BAA0B;QAChC,KAAK,EAAE,sBAAsB;QAC7B,QAAQ,EAAE,UAAU;QACpB,KAAK,EAAE,cAAc;QACrB,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAC5B,UAAU;KACX,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAAY,EACZ,IAAY,EACZ,KAAa,EACb,UAA+B;IAE/B,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,UAAU,CAAC,CAAC;IACzD,OAAO;QACL,IAAI,EAAE,0BAA0B;QAChC,KAAK,EAAE,iBAAiB,IAAI,EAAE;QAC9B,QAAQ,EAAE,UAAU;QACpB,KAAK,EAAE,YAAY;QACnB,IAAI,EAAE,OAAO;QACb,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,UAAU;KACX,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { Dimensions, Coverage, EvidenceReference, PointObservationInput } from "./run-analytics.js";
|
|
2
|
+
/**
|
|
3
|
+
* Provider-neutral normalization of a Claude `--output-format stream-json`
|
|
4
|
+
* transcript into coverage-labelled facts (WS-03, SC-08). Kept apart from
|
|
5
|
+
* `agent-runner.ts` so the normalizer is unit-testable against fixtures
|
|
6
|
+
* without spawning anything; `agent-runner.ts`'s demultiplexer feeds this
|
|
7
|
+
* module's accumulator the same already-parsed events it routes for
|
|
8
|
+
* commands/denials, so production never parses the stream twice.
|
|
9
|
+
*
|
|
10
|
+
* The one rule every field here answers to: absent or unsupported telemetry
|
|
11
|
+
* is `{ coverage: "unavailable" }` with no `value` — never a measured zero.
|
|
12
|
+
*/
|
|
13
|
+
export declare const PROVIDER_TELEMETRY_KIND: "provider-telemetry";
|
|
14
|
+
/** A numeric fact with its observability label. `value` is absent whenever `coverage` is `"unavailable"`. */
|
|
15
|
+
export interface Measured {
|
|
16
|
+
coverage: Coverage;
|
|
17
|
+
value?: number;
|
|
18
|
+
unit?: string;
|
|
19
|
+
}
|
|
20
|
+
/** A string-valued fact (model identity, provider label) with its observability label. */
|
|
21
|
+
export interface MeasuredString {
|
|
22
|
+
coverage: Coverage;
|
|
23
|
+
value?: string;
|
|
24
|
+
}
|
|
25
|
+
/** One `compact_boundary` event, when the stream exposes one. */
|
|
26
|
+
export interface CompactionOccurrence {
|
|
27
|
+
tokensBefore: Measured;
|
|
28
|
+
at: Measured;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The internal, in-memory normalization of one spawn's provider telemetry.
|
|
32
|
+
* Never persisted as-is — `providerTelemetryPoints` fans it out into one
|
|
33
|
+
* scalar `PointObservationInput` per measured fact.
|
|
34
|
+
*/
|
|
35
|
+
export interface NormalizedTelemetry {
|
|
36
|
+
provider: MeasuredString;
|
|
37
|
+
model: MeasuredString;
|
|
38
|
+
configuredModel: MeasuredString;
|
|
39
|
+
inputTokens: Measured;
|
|
40
|
+
outputTokens: Measured;
|
|
41
|
+
cacheReadTokens: Measured;
|
|
42
|
+
cacheCreationTokens: Measured;
|
|
43
|
+
reasoningTokens: Measured;
|
|
44
|
+
contextCapacityTokens: Measured;
|
|
45
|
+
peakContextOccupancyTokens: Measured;
|
|
46
|
+
compactions: CompactionOccurrence[];
|
|
47
|
+
compactionCount: Measured;
|
|
48
|
+
continuationTurns: Measured;
|
|
49
|
+
toolCalls: Measured;
|
|
50
|
+
fileActivity: {
|
|
51
|
+
reads: Measured;
|
|
52
|
+
writes: Measured;
|
|
53
|
+
edits: Measured;
|
|
54
|
+
};
|
|
55
|
+
commandOutputBytes: Measured;
|
|
56
|
+
costUsd?: Measured;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Parses `--model <value>` and `--model=<value>` out of a derived
|
|
60
|
+
* invocation's argv — free operator-authored args routinely carry an
|
|
61
|
+
* explicit model override even though `AgentConfig` has no dedicated model
|
|
62
|
+
* property (SC-05).
|
|
63
|
+
*/
|
|
64
|
+
export declare function parseConfiguredModel(args: readonly string[]): MeasuredString;
|
|
65
|
+
/**
|
|
66
|
+
* A coarse provider label derived from the configured command when the
|
|
67
|
+
* stream itself exposes no model — an inference from the invocation, never
|
|
68
|
+
* a provider-reported fact, hence `"estimated"`.
|
|
69
|
+
*/
|
|
70
|
+
export declare function providerLabelFromCommand(command: string | undefined): MeasuredString;
|
|
71
|
+
interface TelemetryAccumulatorOptions {
|
|
72
|
+
/** Precomputed from the derived invocation's argv (SC-05). */
|
|
73
|
+
configuredModel: MeasuredString;
|
|
74
|
+
/** Precomputed from the derived invocation's command, for the provider fallback. */
|
|
75
|
+
commandProviderGuess: MeasuredString;
|
|
76
|
+
}
|
|
77
|
+
export interface TelemetryAccumulator {
|
|
78
|
+
observeEvent(event: unknown): void;
|
|
79
|
+
finish(flags: {
|
|
80
|
+
observed: boolean;
|
|
81
|
+
corrupted: boolean;
|
|
82
|
+
}): NormalizedTelemetry;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* A pure, non-throwing, stateful accumulator fed one already-parsed
|
|
86
|
+
* stream-json event at a time. `agent-runner.ts`'s demultiplexer routes each
|
|
87
|
+
* event to both its own command/denial routing and this accumulator's
|
|
88
|
+
* `observeEvent`, so the underlying text is parsed exactly once in
|
|
89
|
+
* production; `normalizeClaudeTelemetry` below wraps the same accumulator
|
|
90
|
+
* with its own line-splitting for standalone fixture-driven tests.
|
|
91
|
+
*/
|
|
92
|
+
export declare function createTelemetryAccumulator(options: TelemetryAccumulatorOptions): TelemetryAccumulator;
|
|
93
|
+
/**
|
|
94
|
+
* Demultiplexes a whole captured stream-json string in one call, purely for
|
|
95
|
+
* fixture-driven tests — production feeds `createTelemetryAccumulator`
|
|
96
|
+
* directly from `agent-runner.ts`'s single event-parsing pass so the raw
|
|
97
|
+
* text is never parsed twice.
|
|
98
|
+
*/
|
|
99
|
+
export declare function normalizeClaudeTelemetry(raw: string, options?: {
|
|
100
|
+
configuredArgs?: readonly string[];
|
|
101
|
+
command?: string;
|
|
102
|
+
}): NormalizedTelemetry;
|
|
103
|
+
/** One point per SC-08-enumerated metric, scalar and coverage-labelled, never a fabricated zero for a metric the stream did not expose. */
|
|
104
|
+
export declare function providerTelemetryPoints(telemetry: NormalizedTelemetry, dimensions: Partial<Dimensions>, evidence: EvidenceReference[]): PointObservationInput[];
|
|
105
|
+
export {};
|
|
106
|
+
//# sourceMappingURL=provider-telemetry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider-telemetry.d.ts","sourceRoot":"","sources":["../src/provider-telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEzG;;;;;;;;;;GAUG;AAEH,eAAO,MAAM,uBAAuB,EAAG,oBAA6B,CAAC;AAErE,6GAA6G;AAC7G,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,0FAA0F;AAC1F,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,iEAAiE;AACjE,MAAM,WAAW,oBAAoB;IACnC,YAAY,EAAE,QAAQ,CAAC;IACvB,EAAE,EAAE,QAAQ,CAAC;CACd;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,cAAc,CAAC;IACzB,KAAK,EAAE,cAAc,CAAC;IACtB,eAAe,EAAE,cAAc,CAAC;IAChC,WAAW,EAAE,QAAQ,CAAC;IACtB,YAAY,EAAE,QAAQ,CAAC;IACvB,eAAe,EAAE,QAAQ,CAAC;IAC1B,mBAAmB,EAAE,QAAQ,CAAC;IAC9B,eAAe,EAAE,QAAQ,CAAC;IAC1B,qBAAqB,EAAE,QAAQ,CAAC;IAChC,0BAA0B,EAAE,QAAQ,CAAC;IACrC,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,eAAe,EAAE,QAAQ,CAAC;IAC1B,iBAAiB,EAAE,QAAQ,CAAC;IAC5B,SAAS,EAAE,QAAQ,CAAC;IACpB,YAAY,EAAE;QAAE,KAAK,EAAE,QAAQ,CAAC;QAAC,MAAM,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,QAAQ,CAAA;KAAE,CAAC;IACrE,kBAAkB,EAAE,QAAQ,CAAC;IAC7B,OAAO,CAAC,EAAE,QAAQ,CAAC;CACpB;AAuBD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,cAAc,CAa5E;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,cAAc,CASpF;AAiBD,UAAU,2BAA2B;IACnC,8DAA8D;IAC9D,eAAe,EAAE,cAAc,CAAC;IAChC,oFAAoF;IACpF,oBAAoB,EAAE,cAAc,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IACnC,MAAM,CAAC,KAAK,EAAE;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,GAAG,mBAAmB,CAAC;CAC/E;AAED;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,2BAA2B,GAAG,oBAAoB,CAqOrG;AAMD;;;;;GAKG;AACH,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GACrE,mBAAmB,CA+BrB;AAED,2IAA2I;AAC3I,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,mBAAmB,EAC9B,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,EAC/B,QAAQ,EAAE,iBAAiB,EAAE,GAC5B,qBAAqB,EAAE,CAkEzB"}
|
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider-neutral normalization of a Claude `--output-format stream-json`
|
|
3
|
+
* transcript into coverage-labelled facts (WS-03, SC-08). Kept apart from
|
|
4
|
+
* `agent-runner.ts` so the normalizer is unit-testable against fixtures
|
|
5
|
+
* without spawning anything; `agent-runner.ts`'s demultiplexer feeds this
|
|
6
|
+
* module's accumulator the same already-parsed events it routes for
|
|
7
|
+
* commands/denials, so production never parses the stream twice.
|
|
8
|
+
*
|
|
9
|
+
* The one rule every field here answers to: absent or unsupported telemetry
|
|
10
|
+
* is `{ coverage: "unavailable" }` with no `value` — never a measured zero.
|
|
11
|
+
*/
|
|
12
|
+
export const PROVIDER_TELEMETRY_KIND = "provider-telemetry";
|
|
13
|
+
const UNAVAILABLE = { coverage: "unavailable" };
|
|
14
|
+
const UNAVAILABLE_STRING = { coverage: "unavailable" };
|
|
15
|
+
function observedNumber(value, unit) {
|
|
16
|
+
return { coverage: "observed", value, unit };
|
|
17
|
+
}
|
|
18
|
+
function estimatedNumber(value, unit) {
|
|
19
|
+
return { coverage: "estimated", value, unit };
|
|
20
|
+
}
|
|
21
|
+
function incompleteNumber(value, unit) {
|
|
22
|
+
return { coverage: "incomplete", value, unit };
|
|
23
|
+
}
|
|
24
|
+
/** Downgrades an observed/estimated tally to `incomplete` when the stream lost a mid-stream line; leaves anything already unavailable alone. */
|
|
25
|
+
function maybeIncomplete(measured, corrupted) {
|
|
26
|
+
if (!corrupted || measured.coverage === "unavailable")
|
|
27
|
+
return measured;
|
|
28
|
+
return { ...measured, coverage: "incomplete" };
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Parses `--model <value>` and `--model=<value>` out of a derived
|
|
32
|
+
* invocation's argv — free operator-authored args routinely carry an
|
|
33
|
+
* explicit model override even though `AgentConfig` has no dedicated model
|
|
34
|
+
* property (SC-05).
|
|
35
|
+
*/
|
|
36
|
+
export function parseConfiguredModel(args) {
|
|
37
|
+
for (let i = 0; i < args.length; i++) {
|
|
38
|
+
const arg = args[i];
|
|
39
|
+
if (arg === undefined)
|
|
40
|
+
continue;
|
|
41
|
+
const next = args[i + 1];
|
|
42
|
+
if (arg === "--model" && typeof next === "string") {
|
|
43
|
+
return { coverage: "observed", value: next };
|
|
44
|
+
}
|
|
45
|
+
if (arg.startsWith("--model=")) {
|
|
46
|
+
return { coverage: "observed", value: arg.slice("--model=".length) };
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return UNAVAILABLE_STRING;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* A coarse provider label derived from the configured command when the
|
|
53
|
+
* stream itself exposes no model — an inference from the invocation, never
|
|
54
|
+
* a provider-reported fact, hence `"estimated"`.
|
|
55
|
+
*/
|
|
56
|
+
export function providerLabelFromCommand(command) {
|
|
57
|
+
if (command === undefined || command.trim() === "")
|
|
58
|
+
return UNAVAILABLE_STRING;
|
|
59
|
+
const base = command
|
|
60
|
+
.replaceAll("\\", "/")
|
|
61
|
+
.split("/")
|
|
62
|
+
.pop()
|
|
63
|
+
?.replace(/\.(cmd|exe|bat|sh)$/iu, "");
|
|
64
|
+
if (base === undefined || base.trim() === "")
|
|
65
|
+
return UNAVAILABLE_STRING;
|
|
66
|
+
return { coverage: "estimated", value: base };
|
|
67
|
+
}
|
|
68
|
+
/** The text of a `tool_result` block, whether the CLI sent it as a string or as content blocks. */
|
|
69
|
+
function toolResultBytes(block) {
|
|
70
|
+
if (typeof block.content === "string")
|
|
71
|
+
return Buffer.byteLength(block.content, "utf8");
|
|
72
|
+
if (Array.isArray(block.content)) {
|
|
73
|
+
const texts = block.content
|
|
74
|
+
.filter((part) => part?.type === "text" && typeof part.text === "string")
|
|
75
|
+
.map((part) => part.text);
|
|
76
|
+
if (texts.length === 0)
|
|
77
|
+
return undefined;
|
|
78
|
+
return Buffer.byteLength(texts.join("\n"), "utf8");
|
|
79
|
+
}
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
const FILE_TOOL_KEYS = { Read: "reads", Write: "writes", Edit: "edits" };
|
|
83
|
+
/**
|
|
84
|
+
* A pure, non-throwing, stateful accumulator fed one already-parsed
|
|
85
|
+
* stream-json event at a time. `agent-runner.ts`'s demultiplexer routes each
|
|
86
|
+
* event to both its own command/denial routing and this accumulator's
|
|
87
|
+
* `observeEvent`, so the underlying text is parsed exactly once in
|
|
88
|
+
* production; `normalizeClaudeTelemetry` below wraps the same accumulator
|
|
89
|
+
* with its own line-splitting for standalone fixture-driven tests.
|
|
90
|
+
*/
|
|
91
|
+
export function createTelemetryAccumulator(options) {
|
|
92
|
+
let streamModel;
|
|
93
|
+
let toolCallCount = 0;
|
|
94
|
+
let reads = 0;
|
|
95
|
+
let writes = 0;
|
|
96
|
+
let edits = 0;
|
|
97
|
+
let commandOutputBytes = 0;
|
|
98
|
+
let hadOrphanToolResult = false;
|
|
99
|
+
const toolNameById = new Map();
|
|
100
|
+
let reasoningTokens;
|
|
101
|
+
const compactions = [];
|
|
102
|
+
// Final `result`-event figures, captured once.
|
|
103
|
+
let sawResultEvent = false;
|
|
104
|
+
let inputTokens;
|
|
105
|
+
let outputTokens;
|
|
106
|
+
let cacheReadTokens;
|
|
107
|
+
let cacheCreationTokens;
|
|
108
|
+
let numTurns;
|
|
109
|
+
let costUsd;
|
|
110
|
+
let contextCapacityTokens;
|
|
111
|
+
let peakContextOccupancy;
|
|
112
|
+
function observeEvent(event) {
|
|
113
|
+
try {
|
|
114
|
+
if (event === null || typeof event !== "object")
|
|
115
|
+
return;
|
|
116
|
+
const record = event;
|
|
117
|
+
if (record.type === "system" && record.subtype === "init") {
|
|
118
|
+
if (typeof record.model === "string")
|
|
119
|
+
streamModel = record.model;
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
if (record.type === "system" && record.subtype === "thinking_tokens") {
|
|
123
|
+
if (typeof record.estimated_tokens === "number")
|
|
124
|
+
reasoningTokens = record.estimated_tokens;
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
if (record.type === "system" && record.subtype === "compact_boundary") {
|
|
128
|
+
const preTokens = typeof record.pre_tokens === "number" ? observedNumber(record.pre_tokens, "tokens") : UNAVAILABLE;
|
|
129
|
+
let at = UNAVAILABLE;
|
|
130
|
+
if (typeof record.timestamp === "string") {
|
|
131
|
+
const parsed = Date.parse(record.timestamp);
|
|
132
|
+
if (!Number.isNaN(parsed))
|
|
133
|
+
at = observedNumber(parsed, "ms-since-epoch");
|
|
134
|
+
}
|
|
135
|
+
else if (typeof record.turn === "number") {
|
|
136
|
+
at = observedNumber(record.turn, "turn");
|
|
137
|
+
}
|
|
138
|
+
compactions.push({ tokensBefore: preTokens, at });
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
if (record.type === "assistant") {
|
|
142
|
+
const content = record.message?.content;
|
|
143
|
+
if (Array.isArray(content)) {
|
|
144
|
+
for (const block of content) {
|
|
145
|
+
if (block?.type !== "tool_use")
|
|
146
|
+
continue;
|
|
147
|
+
toolCallCount += 1;
|
|
148
|
+
const name = typeof block.name === "string" ? block.name : undefined;
|
|
149
|
+
const id = typeof block.id === "string" ? block.id : undefined;
|
|
150
|
+
if (name !== undefined && id !== undefined)
|
|
151
|
+
toolNameById.set(id, name);
|
|
152
|
+
if (name !== undefined && name in FILE_TOOL_KEYS) {
|
|
153
|
+
const key = FILE_TOOL_KEYS[name];
|
|
154
|
+
if (key === "reads")
|
|
155
|
+
reads += 1;
|
|
156
|
+
else if (key === "writes")
|
|
157
|
+
writes += 1;
|
|
158
|
+
else
|
|
159
|
+
edits += 1;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
if (record.type === "user") {
|
|
166
|
+
const content = record.message?.content;
|
|
167
|
+
if (Array.isArray(content)) {
|
|
168
|
+
for (const block of content) {
|
|
169
|
+
if (block?.type !== "tool_result")
|
|
170
|
+
continue;
|
|
171
|
+
const toolUseId = typeof block.tool_use_id === "string" ? block.tool_use_id : undefined;
|
|
172
|
+
const toolName = toolUseId === undefined ? undefined : toolNameById.get(toolUseId);
|
|
173
|
+
if (toolName === undefined) {
|
|
174
|
+
hadOrphanToolResult = true;
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
if (toolName !== "Bash")
|
|
178
|
+
continue;
|
|
179
|
+
const bytes = toolResultBytes(block);
|
|
180
|
+
if (bytes !== undefined)
|
|
181
|
+
commandOutputBytes += bytes;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
if (record.type === "result") {
|
|
187
|
+
sawResultEvent = true;
|
|
188
|
+
if (typeof record.num_turns === "number")
|
|
189
|
+
numTurns = record.num_turns;
|
|
190
|
+
if (typeof record.total_cost_usd === "number")
|
|
191
|
+
costUsd = record.total_cost_usd;
|
|
192
|
+
const usage = record.usage;
|
|
193
|
+
if (usage) {
|
|
194
|
+
if (typeof usage.input_tokens === "number")
|
|
195
|
+
inputTokens = usage.input_tokens;
|
|
196
|
+
if (typeof usage.output_tokens === "number")
|
|
197
|
+
outputTokens = usage.output_tokens;
|
|
198
|
+
if (typeof usage.cache_read_input_tokens === "number")
|
|
199
|
+
cacheReadTokens = usage.cache_read_input_tokens;
|
|
200
|
+
if (typeof usage.cache_creation_input_tokens === "number") {
|
|
201
|
+
cacheCreationTokens = usage.cache_creation_input_tokens;
|
|
202
|
+
}
|
|
203
|
+
const iterations = usage.iterations;
|
|
204
|
+
if (Array.isArray(iterations)) {
|
|
205
|
+
let peak;
|
|
206
|
+
let anyIterationPartial = false;
|
|
207
|
+
for (const iteration of iterations) {
|
|
208
|
+
const fields = [
|
|
209
|
+
iteration.input_tokens,
|
|
210
|
+
iteration.output_tokens,
|
|
211
|
+
iteration.cache_read_input_tokens,
|
|
212
|
+
iteration.cache_creation_input_tokens,
|
|
213
|
+
];
|
|
214
|
+
// An iteration exposing none of the four occupancy fields (e.g.
|
|
215
|
+
// `{}`) contributes no usable measurement at all — folding it
|
|
216
|
+
// in via numberOr0 would invent a fabricated 0 that could win
|
|
217
|
+
// the max, misreporting "the provider observed zero occupancy"
|
|
218
|
+
// for an iteration the provider told us nothing about. Skip it
|
|
219
|
+
// rather than let it participate in the max.
|
|
220
|
+
if (!fields.some((field) => typeof field === "number")) {
|
|
221
|
+
anyIterationPartial = true;
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
// An iteration with only *some* of the four fields present is
|
|
225
|
+
// still counted (using 0 for the fields it does lack), but the
|
|
226
|
+
// result can undercount that iteration's true occupancy, so the
|
|
227
|
+
// overall peak is downgraded to "incomplete" rather than
|
|
228
|
+
// "observed" below.
|
|
229
|
+
if (!fields.every((field) => typeof field === "number"))
|
|
230
|
+
anyIterationPartial = true;
|
|
231
|
+
const occupancy = fields.reduce((sum, field) => sum + numberOr0(field), 0);
|
|
232
|
+
if (peak === undefined || occupancy > peak)
|
|
233
|
+
peak = occupancy;
|
|
234
|
+
}
|
|
235
|
+
if (peak !== undefined)
|
|
236
|
+
peakContextOccupancy = { value: peak, incomplete: anyIterationPartial };
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
const modelUsage = record.modelUsage;
|
|
240
|
+
if (modelUsage) {
|
|
241
|
+
const key = streamModel !== undefined && modelUsage[streamModel] ? streamModel : Object.keys(modelUsage)[0];
|
|
242
|
+
const entry = key === undefined ? undefined : modelUsage[key];
|
|
243
|
+
if (entry && typeof entry.contextWindow === "number")
|
|
244
|
+
contextCapacityTokens = entry.contextWindow;
|
|
245
|
+
if (costUsd === undefined && entry && typeof entry.costUSD === "number")
|
|
246
|
+
costUsd = entry.costUSD;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
catch {
|
|
251
|
+
// The accumulator is total and non-throwing (SC-13): one malformed
|
|
252
|
+
// event never poisons every fact collected so far.
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
function finish(flags) {
|
|
256
|
+
// Model/provider precedence (SC-05) does not gate on whether the stream
|
|
257
|
+
// itself parsed: `streamModel` is only ever set from an observed stream,
|
|
258
|
+
// so this is a no-op widening for the observed branch and, for a
|
|
259
|
+
// non-NDJSON or crashed spawn, still honors an explicit `--model` in the
|
|
260
|
+
// invocation args and a command-derived provider guess rather than
|
|
261
|
+
// discarding facts the invocation itself already gave us.
|
|
262
|
+
const model = streamModel !== undefined
|
|
263
|
+
? { coverage: "observed", value: streamModel }
|
|
264
|
+
: options.configuredModel.value !== undefined
|
|
265
|
+
? { coverage: "observed", value: options.configuredModel.value }
|
|
266
|
+
: UNAVAILABLE_STRING;
|
|
267
|
+
const provider = streamModel !== undefined ? { coverage: "observed", value: "claude" } : options.commandProviderGuess;
|
|
268
|
+
if (!flags.observed) {
|
|
269
|
+
return {
|
|
270
|
+
provider,
|
|
271
|
+
model,
|
|
272
|
+
configuredModel: options.configuredModel,
|
|
273
|
+
inputTokens: UNAVAILABLE,
|
|
274
|
+
outputTokens: UNAVAILABLE,
|
|
275
|
+
cacheReadTokens: UNAVAILABLE,
|
|
276
|
+
cacheCreationTokens: UNAVAILABLE,
|
|
277
|
+
reasoningTokens: UNAVAILABLE,
|
|
278
|
+
contextCapacityTokens: UNAVAILABLE,
|
|
279
|
+
peakContextOccupancyTokens: UNAVAILABLE,
|
|
280
|
+
compactions: [],
|
|
281
|
+
compactionCount: UNAVAILABLE,
|
|
282
|
+
continuationTurns: UNAVAILABLE,
|
|
283
|
+
toolCalls: UNAVAILABLE,
|
|
284
|
+
fileActivity: { reads: UNAVAILABLE, writes: UNAVAILABLE, edits: UNAVAILABLE },
|
|
285
|
+
commandOutputBytes: UNAVAILABLE,
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
const commandOutputIncomplete = flags.corrupted && hadOrphanToolResult;
|
|
289
|
+
return {
|
|
290
|
+
provider,
|
|
291
|
+
model,
|
|
292
|
+
configuredModel: options.configuredModel,
|
|
293
|
+
inputTokens: sawResultEvent && inputTokens !== undefined ? observedNumber(inputTokens, "tokens") : UNAVAILABLE,
|
|
294
|
+
outputTokens: sawResultEvent && outputTokens !== undefined ? observedNumber(outputTokens, "tokens") : UNAVAILABLE,
|
|
295
|
+
cacheReadTokens: sawResultEvent && cacheReadTokens !== undefined ? observedNumber(cacheReadTokens, "tokens") : UNAVAILABLE,
|
|
296
|
+
cacheCreationTokens: sawResultEvent && cacheCreationTokens !== undefined
|
|
297
|
+
? observedNumber(cacheCreationTokens, "tokens")
|
|
298
|
+
: UNAVAILABLE,
|
|
299
|
+
reasoningTokens: reasoningTokens !== undefined ? estimatedNumber(reasoningTokens, "tokens") : UNAVAILABLE,
|
|
300
|
+
contextCapacityTokens: contextCapacityTokens !== undefined ? observedNumber(contextCapacityTokens, "tokens") : UNAVAILABLE,
|
|
301
|
+
peakContextOccupancyTokens: peakContextOccupancy === undefined
|
|
302
|
+
? UNAVAILABLE
|
|
303
|
+
: peakContextOccupancy.incomplete
|
|
304
|
+
? incompleteNumber(peakContextOccupancy.value, "tokens")
|
|
305
|
+
: observedNumber(peakContextOccupancy.value, "tokens"),
|
|
306
|
+
compactions,
|
|
307
|
+
compactionCount: compactions.length > 0 ? observedNumber(compactions.length, "count") : UNAVAILABLE,
|
|
308
|
+
continuationTurns: sawResultEvent && numTurns !== undefined ? observedNumber(numTurns, "count") : UNAVAILABLE,
|
|
309
|
+
toolCalls: maybeIncomplete(observedNumber(toolCallCount, "count"), flags.corrupted),
|
|
310
|
+
fileActivity: {
|
|
311
|
+
reads: maybeIncomplete(observedNumber(reads, "count"), flags.corrupted),
|
|
312
|
+
writes: maybeIncomplete(observedNumber(writes, "count"), flags.corrupted),
|
|
313
|
+
edits: maybeIncomplete(observedNumber(edits, "count"), flags.corrupted),
|
|
314
|
+
},
|
|
315
|
+
commandOutputBytes: commandOutputIncomplete
|
|
316
|
+
? incompleteNumber(commandOutputBytes, "bytes")
|
|
317
|
+
: observedNumber(commandOutputBytes, "bytes"),
|
|
318
|
+
...(costUsd !== undefined ? { costUsd: observedNumber(costUsd, "usd") } : {}),
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
return { observeEvent, finish };
|
|
322
|
+
}
|
|
323
|
+
function numberOr0(value) {
|
|
324
|
+
return typeof value === "number" ? value : 0;
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Demultiplexes a whole captured stream-json string in one call, purely for
|
|
328
|
+
* fixture-driven tests — production feeds `createTelemetryAccumulator`
|
|
329
|
+
* directly from `agent-runner.ts`'s single event-parsing pass so the raw
|
|
330
|
+
* text is never parsed twice.
|
|
331
|
+
*/
|
|
332
|
+
export function normalizeClaudeTelemetry(raw, options = {}) {
|
|
333
|
+
const accumulator = createTelemetryAccumulator({
|
|
334
|
+
configuredModel: parseConfiguredModel(options.configuredArgs ?? []),
|
|
335
|
+
commandProviderGuess: providerLabelFromCommand(options.command),
|
|
336
|
+
});
|
|
337
|
+
let firstLineSeen = false;
|
|
338
|
+
let isNdjson = true;
|
|
339
|
+
let corrupted = false;
|
|
340
|
+
for (const rawLine of raw.split("\n")) {
|
|
341
|
+
const line = rawLine.trim();
|
|
342
|
+
if (line === "")
|
|
343
|
+
continue;
|
|
344
|
+
if (!firstLineSeen) {
|
|
345
|
+
firstLineSeen = true;
|
|
346
|
+
try {
|
|
347
|
+
accumulator.observeEvent(JSON.parse(line));
|
|
348
|
+
}
|
|
349
|
+
catch {
|
|
350
|
+
isNdjson = false;
|
|
351
|
+
}
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
if (!isNdjson)
|
|
355
|
+
break;
|
|
356
|
+
try {
|
|
357
|
+
accumulator.observeEvent(JSON.parse(line));
|
|
358
|
+
}
|
|
359
|
+
catch {
|
|
360
|
+
corrupted = true;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return accumulator.finish({ observed: firstLineSeen && isNdjson, corrupted });
|
|
364
|
+
}
|
|
365
|
+
/** One point per SC-08-enumerated metric, scalar and coverage-labelled, never a fabricated zero for a metric the stream did not expose. */
|
|
366
|
+
export function providerTelemetryPoints(telemetry, dimensions, evidence) {
|
|
367
|
+
const points = [];
|
|
368
|
+
const stringPoint = (label, fact, detailOverride) => {
|
|
369
|
+
points.push({
|
|
370
|
+
kind: PROVIDER_TELEMETRY_KIND,
|
|
371
|
+
label,
|
|
372
|
+
coverage: fact.coverage,
|
|
373
|
+
...(detailOverride !== undefined
|
|
374
|
+
? { detail: detailOverride }
|
|
375
|
+
: fact.value !== undefined
|
|
376
|
+
? { detail: fact.value }
|
|
377
|
+
: {}),
|
|
378
|
+
dimensions,
|
|
379
|
+
evidence,
|
|
380
|
+
});
|
|
381
|
+
};
|
|
382
|
+
const numberPoint = (label, fact) => {
|
|
383
|
+
points.push({
|
|
384
|
+
kind: PROVIDER_TELEMETRY_KIND,
|
|
385
|
+
label,
|
|
386
|
+
coverage: fact.coverage,
|
|
387
|
+
...(fact.value === undefined ? {} : { value: fact.value }),
|
|
388
|
+
...(fact.unit === undefined ? {} : { unit: fact.unit }),
|
|
389
|
+
dimensions,
|
|
390
|
+
evidence,
|
|
391
|
+
});
|
|
392
|
+
};
|
|
393
|
+
stringPoint("provider", telemetry.provider);
|
|
394
|
+
const modelMismatch = telemetry.model.value !== undefined &&
|
|
395
|
+
telemetry.configuredModel.value !== undefined &&
|
|
396
|
+
telemetry.model.value !== telemetry.configuredModel.value;
|
|
397
|
+
stringPoint("model", telemetry.model, modelMismatch
|
|
398
|
+
? `${telemetry.model.value} (configured invocation specified ${telemetry.configuredModel.value})`
|
|
399
|
+
: undefined);
|
|
400
|
+
stringPoint("configured-model", telemetry.configuredModel);
|
|
401
|
+
numberPoint("input-tokens", telemetry.inputTokens);
|
|
402
|
+
numberPoint("output-tokens", telemetry.outputTokens);
|
|
403
|
+
numberPoint("cache-read-tokens", telemetry.cacheReadTokens);
|
|
404
|
+
numberPoint("cache-creation-tokens", telemetry.cacheCreationTokens);
|
|
405
|
+
numberPoint("reasoning-tokens", telemetry.reasoningTokens);
|
|
406
|
+
numberPoint("context-capacity-tokens", telemetry.contextCapacityTokens);
|
|
407
|
+
numberPoint("peak-context-occupancy-tokens", telemetry.peakContextOccupancyTokens);
|
|
408
|
+
numberPoint("continuation-turns", telemetry.continuationTurns);
|
|
409
|
+
numberPoint("tool-calls", telemetry.toolCalls);
|
|
410
|
+
numberPoint("file-activity-reads", telemetry.fileActivity.reads);
|
|
411
|
+
numberPoint("file-activity-writes", telemetry.fileActivity.writes);
|
|
412
|
+
numberPoint("file-activity-edits", telemetry.fileActivity.edits);
|
|
413
|
+
numberPoint("command-output-bytes", telemetry.commandOutputBytes);
|
|
414
|
+
numberPoint("compaction-count", telemetry.compactionCount);
|
|
415
|
+
if (telemetry.costUsd !== undefined)
|
|
416
|
+
numberPoint("cost-usd", telemetry.costUsd);
|
|
417
|
+
for (const occurrence of telemetry.compactions) {
|
|
418
|
+
numberPoint("compaction-tokens-before", occurrence.tokensBefore);
|
|
419
|
+
numberPoint("compaction-at", occurrence.at);
|
|
420
|
+
}
|
|
421
|
+
return points;
|
|
422
|
+
}
|
|
423
|
+
//# sourceMappingURL=provider-telemetry.js.map
|