context-lens 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -17
- package/dist/cli-utils.d.ts +1 -1
- package/dist/cli-utils.d.ts.map +1 -1
- package/dist/cli-utils.js +28 -13
- package/dist/cli-utils.js.map +1 -1
- package/dist/cli.js +77 -65
- package/dist/cli.js.map +1 -1
- package/dist/core/conversation.d.ts +54 -0
- package/dist/core/conversation.d.ts.map +1 -0
- package/dist/core/conversation.js +188 -0
- package/dist/core/conversation.js.map +1 -0
- package/dist/core/models.d.ts +30 -0
- package/dist/core/models.d.ts.map +1 -0
- package/dist/core/models.js +96 -0
- package/dist/core/models.js.map +1 -0
- package/dist/core/parse.d.ts +17 -0
- package/dist/core/parse.d.ts.map +1 -0
- package/dist/core/parse.js +349 -0
- package/dist/core/parse.js.map +1 -0
- package/dist/core/routing.d.ts +47 -0
- package/dist/core/routing.d.ts.map +1 -0
- package/dist/core/routing.js +132 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/source.d.ts +22 -0
- package/dist/core/source.d.ts.map +1 -0
- package/dist/core/source.js +56 -0
- package/dist/core/source.js.map +1 -0
- package/dist/core/tokens.d.ts +11 -0
- package/dist/core/tokens.d.ts.map +1 -0
- package/dist/core/tokens.js +16 -0
- package/dist/core/tokens.js.map +1 -0
- package/dist/core.d.ts +12 -22
- package/dist/core.d.ts.map +1 -1
- package/dist/core.js +12 -471
- package/dist/core.js.map +1 -1
- package/dist/http/headers.d.ts +25 -0
- package/dist/http/headers.d.ts.map +1 -0
- package/dist/http/headers.js +54 -0
- package/dist/http/headers.js.map +1 -0
- package/dist/lhar-types.generated.d.ts +1 -1
- package/dist/lhar.d.ts +4 -4
- package/dist/lhar.d.ts.map +1 -1
- package/dist/lhar.js +190 -106
- package/dist/lhar.js.map +1 -1
- package/dist/server/config.d.ts +12 -0
- package/dist/server/config.d.ts.map +1 -0
- package/dist/server/config.js +33 -0
- package/dist/server/config.js.map +1 -0
- package/dist/server/projection.d.ts +9 -0
- package/dist/server/projection.d.ts.map +1 -0
- package/dist/server/projection.js +39 -0
- package/dist/server/projection.js.map +1 -0
- package/dist/server/proxy.d.ts +13 -0
- package/dist/server/proxy.d.ts.map +1 -0
- package/dist/server/proxy.js +232 -0
- package/dist/server/proxy.js.map +1 -0
- package/dist/server/store.d.ts +33 -0
- package/dist/server/store.d.ts.map +1 -0
- package/dist/server/store.js +350 -0
- package/dist/server/store.js.map +1 -0
- package/dist/server/webui.d.ts +5 -0
- package/dist/server/webui.d.ts.map +1 -0
- package/dist/server/webui.js +170 -0
- package/dist/server/webui.js.map +1 -0
- package/dist/server-utils.d.ts +2 -2
- package/dist/server-utils.d.ts.map +1 -1
- package/dist/server-utils.js +12 -21
- package/dist/server-utils.js.map +1 -1
- package/dist/server.js +30 -697
- package/dist/server.js.map +1 -1
- package/dist/types.d.ts +50 -10
- package/dist/types.d.ts.map +1 -1
- package/dist/version.generated.d.ts +2 -0
- package/dist/version.generated.d.ts.map +1 -0
- package/dist/version.generated.js +2 -0
- package/dist/version.generated.js.map +1 -0
- package/package.json +18 -6
- package/public/index.html +39 -12
- package/schema/lhar.schema.json +1 -1
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared header redaction utilities.
|
|
3
|
+
*
|
|
4
|
+
* Context Lens captures and exports some headers for debugging and provenance,
|
|
5
|
+
* but must never persist secrets (API keys, cookies, auth challenges, etc.).
|
|
6
|
+
*
|
|
7
|
+
* Keep this as the single source of truth to avoid drift between "capture" and "export".
|
|
8
|
+
*/
|
|
9
|
+
/** Case-insensitive set of header names that must never be persisted/exported. */
|
|
10
|
+
export const SENSITIVE_HEADERS = new Set([
|
|
11
|
+
"authorization",
|
|
12
|
+
"x-api-key",
|
|
13
|
+
"cookie",
|
|
14
|
+
"set-cookie",
|
|
15
|
+
"x-target-url",
|
|
16
|
+
"proxy-authorization",
|
|
17
|
+
"x-auth-token",
|
|
18
|
+
"x-forwarded-authorization",
|
|
19
|
+
"www-authenticate",
|
|
20
|
+
"proxy-authenticate",
|
|
21
|
+
"x-goog-api-key",
|
|
22
|
+
]);
|
|
23
|
+
/**
|
|
24
|
+
* Remove sensitive headers from a header map.
|
|
25
|
+
*
|
|
26
|
+
* @param headers - Header map (string -> string)
|
|
27
|
+
* @returns A new object with sensitive headers removed.
|
|
28
|
+
*/
|
|
29
|
+
export function redactHeaders(headers) {
|
|
30
|
+
const result = {};
|
|
31
|
+
for (const [key, val] of Object.entries(headers)) {
|
|
32
|
+
if (SENSITIVE_HEADERS.has(key.toLowerCase()))
|
|
33
|
+
continue;
|
|
34
|
+
result[key] = val;
|
|
35
|
+
}
|
|
36
|
+
return result;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Select a safe subset of request/response headers for capture.
|
|
40
|
+
*
|
|
41
|
+
* - Drops sensitive headers (see `SENSITIVE_HEADERS`)
|
|
42
|
+
* - Keeps only string-valued headers (Node can represent multi-valued headers as arrays)
|
|
43
|
+
*/
|
|
44
|
+
export function selectHeaders(headers) {
|
|
45
|
+
const result = {};
|
|
46
|
+
for (const [key, val] of Object.entries(headers)) {
|
|
47
|
+
if (SENSITIVE_HEADERS.has(key.toLowerCase()))
|
|
48
|
+
continue;
|
|
49
|
+
if (typeof val === "string")
|
|
50
|
+
result[key] = val;
|
|
51
|
+
}
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=headers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../src/http/headers.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,kFAAkF;AAClF,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IACvC,eAAe;IACf,WAAW;IACX,QAAQ;IACR,YAAY;IACZ,cAAc;IACd,qBAAqB;IACrB,cAAc;IACd,2BAA2B;IAC3B,kBAAkB;IAClB,oBAAoB;IACpB,gBAAgB;CACjB,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,OAA+B;IAE/B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACvD,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,OAA4B;IAE5B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACvD,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
export type CompositionCategory = "system_prompt" | "tool_definitions" | "tool_results" | "tool_calls" | "assistant_text" | "user_text" | "thinking" | "system_injections" | "images" | "cache_markers" | "other";
|
|
11
11
|
/**
|
|
12
|
-
* JSON Schema for the LHAR format
|
|
12
|
+
* JSON Schema for the LHAR format. What HAR is for web traffic, but for LLM API calls.
|
|
13
13
|
*/
|
|
14
14
|
export interface LHARLLMHTTPArchiveFormat {
|
|
15
15
|
[k: string]: unknown;
|
package/dist/lhar.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import
|
|
2
|
-
import type {
|
|
3
|
-
|
|
4
|
-
export
|
|
1
|
+
import { redactHeaders, SENSITIVE_HEADERS } from "./http/headers.js";
|
|
2
|
+
import type { CompositionEntry, LharJsonWrapper, LharRecord, LharSessionLine } from "./lhar-types.generated.js";
|
|
3
|
+
import type { CapturedEntry, ContextInfo, Conversation } from "./types.js";
|
|
4
|
+
export { redactHeaders, SENSITIVE_HEADERS };
|
|
5
5
|
export declare function analyzeComposition(contextInfo: ContextInfo, rawBody: Record<string, any> | undefined): CompositionEntry[];
|
|
6
6
|
export interface ParsedResponseUsage {
|
|
7
7
|
inputTokens: number;
|
package/dist/lhar.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lhar.d.ts","sourceRoot":"","sources":["../src/lhar.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"lhar.d.ts","sourceRoot":"","sources":["../src/lhar.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,KAAK,EAEV,gBAAgB,EAChB,eAAe,EACf,UAAU,EACV,eAAe,EAChB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAS3E,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,CAAC;AAI5C,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,GACvC,gBAAgB,EAAE,CA0EpB;AAgLD,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,GAAG,GAAG,mBAAmB,CAoEzE;AAsFD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,aAAa,EACpB,WAAW,EAAE,aAAa,EAAE,GAC3B,UAAU,CAkJZ;AAID,wBAAgB,gBAAgB,CAC9B,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,eAAe,CAQjB;AAID,wBAAgB,WAAW,CACzB,OAAO,EAAE,aAAa,EAAE,EACxB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,GACvC,MAAM,CAmCR;AAED,wBAAgB,UAAU,CACxB,OAAO,EAAE,aAAa,EAAE,EACxB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,GACvC,eAAe,CAsCjB"}
|
package/dist/lhar.js
CHANGED
|
@@ -1,23 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { estimateTokens } from
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const
|
|
1
|
+
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
2
|
+
import { estimateTokens } from "./core.js";
|
|
3
|
+
import { redactHeaders, SENSITIVE_HEADERS } from "./http/headers.js";
|
|
4
|
+
import { VERSION } from "./version.generated.js";
|
|
5
|
+
const COLLECTOR_NAME = "context-lens";
|
|
6
|
+
const COLLECTOR_VERSION = VERSION;
|
|
7
|
+
const LHAR_VERSION = "0.1.0";
|
|
6
8
|
// --- Header Redaction ---
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
'x-target-url', 'proxy-authorization', 'x-auth-token',
|
|
10
|
-
'x-forwarded-authorization', 'www-authenticate', 'proxy-authenticate',
|
|
11
|
-
]);
|
|
12
|
-
export function redactHeaders(headers) {
|
|
13
|
-
const result = {};
|
|
14
|
-
for (const [key, val] of Object.entries(headers)) {
|
|
15
|
-
if (SENSITIVE_HEADERS.has(key.toLowerCase()))
|
|
16
|
-
continue;
|
|
17
|
-
result[key] = val;
|
|
18
|
-
}
|
|
19
|
-
return result;
|
|
20
|
-
}
|
|
9
|
+
// Kept as exports for backward compatibility, but implemented in `src/http/headers.ts`.
|
|
10
|
+
export { redactHeaders, SENSITIVE_HEADERS };
|
|
21
11
|
// --- Composition Analysis ---
|
|
22
12
|
export function analyzeComposition(contextInfo, rawBody) {
|
|
23
13
|
const counts = new Map();
|
|
@@ -34,90 +24,104 @@ export function analyzeComposition(contextInfo, rawBody) {
|
|
|
34
24
|
if (!rawBody) {
|
|
35
25
|
// Fallback to contextInfo aggregates
|
|
36
26
|
if (contextInfo.systemTokens > 0)
|
|
37
|
-
add(
|
|
27
|
+
add("system_prompt", contextInfo.systemTokens);
|
|
38
28
|
if (contextInfo.toolsTokens > 0)
|
|
39
|
-
add(
|
|
29
|
+
add("tool_definitions", contextInfo.toolsTokens);
|
|
40
30
|
if (contextInfo.messagesTokens > 0)
|
|
41
|
-
add(
|
|
31
|
+
add("other", contextInfo.messagesTokens);
|
|
42
32
|
return buildCompositionArray(counts, contextInfo.totalTokens);
|
|
43
33
|
}
|
|
44
34
|
// System prompt(s)
|
|
45
35
|
if (rawBody.system) {
|
|
46
|
-
if (typeof rawBody.system ===
|
|
47
|
-
add(
|
|
36
|
+
if (typeof rawBody.system === "string") {
|
|
37
|
+
add("system_prompt", estimateTokens(rawBody.system));
|
|
48
38
|
}
|
|
49
39
|
else if (Array.isArray(rawBody.system)) {
|
|
50
40
|
for (const block of rawBody.system) {
|
|
51
41
|
if (block.cache_control) {
|
|
52
|
-
add(
|
|
42
|
+
add("cache_markers", estimateTokens(block.cache_control));
|
|
53
43
|
}
|
|
54
|
-
add(
|
|
44
|
+
add("system_prompt", estimateTokens(block.text || block));
|
|
55
45
|
}
|
|
56
46
|
}
|
|
57
47
|
}
|
|
58
48
|
// Instructions (OpenAI Responses API / ChatGPT)
|
|
59
49
|
if (rawBody.instructions) {
|
|
60
|
-
add(
|
|
50
|
+
add("system_prompt", estimateTokens(rawBody.instructions));
|
|
61
51
|
}
|
|
62
52
|
// Tool definitions
|
|
63
53
|
if (rawBody.tools && Array.isArray(rawBody.tools)) {
|
|
64
|
-
add(
|
|
54
|
+
add("tool_definitions", estimateTokens(JSON.stringify(rawBody.tools)));
|
|
65
55
|
}
|
|
66
|
-
//
|
|
67
|
-
const
|
|
56
|
+
// Gemini/Code Assist: unwrap .request wrapper if present
|
|
57
|
+
const geminiBody = rawBody.request || rawBody;
|
|
58
|
+
// Gemini systemInstruction
|
|
59
|
+
if (geminiBody.systemInstruction) {
|
|
60
|
+
const parts = geminiBody.systemInstruction.parts || [];
|
|
61
|
+
add("system_prompt", estimateTokens(parts.map((p) => p.text || "").join("\n")));
|
|
62
|
+
}
|
|
63
|
+
// Gemini contents[] or standard messages[]/input[]
|
|
64
|
+
const messages = geminiBody.contents || rawBody.messages || rawBody.input;
|
|
68
65
|
if (Array.isArray(messages)) {
|
|
69
66
|
for (const msg of messages) {
|
|
70
67
|
classifyMessage(msg, add);
|
|
71
68
|
}
|
|
72
69
|
}
|
|
73
|
-
else if (typeof messages ===
|
|
74
|
-
add(
|
|
70
|
+
else if (typeof messages === "string") {
|
|
71
|
+
add("user_text", estimateTokens(messages));
|
|
75
72
|
}
|
|
76
73
|
const total = Array.from(counts.values()).reduce((s, e) => s + e.tokens, 0);
|
|
77
74
|
return buildCompositionArray(counts, total);
|
|
78
75
|
}
|
|
79
76
|
function classifyMessage(msg, add) {
|
|
80
|
-
const type = msg.type ||
|
|
77
|
+
const type = msg.type || "";
|
|
81
78
|
// OpenAI Responses API typed items (no role field)
|
|
82
79
|
if (!msg.role && type) {
|
|
83
|
-
if (type ===
|
|
84
|
-
add(
|
|
80
|
+
if (type === "function_call" || type === "custom_tool_call") {
|
|
81
|
+
add("tool_calls", estimateTokens(msg));
|
|
85
82
|
return;
|
|
86
83
|
}
|
|
87
|
-
if (type ===
|
|
88
|
-
add(
|
|
84
|
+
if (type === "function_call_output" || type === "custom_tool_call_output") {
|
|
85
|
+
add("tool_results", estimateTokens(msg.output || ""));
|
|
89
86
|
return;
|
|
90
87
|
}
|
|
91
|
-
if (type ===
|
|
92
|
-
add(
|
|
88
|
+
if (type === "reasoning") {
|
|
89
|
+
add("thinking", estimateTokens(msg));
|
|
93
90
|
return;
|
|
94
91
|
}
|
|
95
|
-
if (type ===
|
|
96
|
-
add(
|
|
92
|
+
if (type === "output_text") {
|
|
93
|
+
add("assistant_text", estimateTokens(msg.text || ""));
|
|
97
94
|
return;
|
|
98
95
|
}
|
|
99
|
-
if (type ===
|
|
100
|
-
add(
|
|
96
|
+
if (type === "input_text") {
|
|
97
|
+
add("user_text", estimateTokens(msg.text || ""));
|
|
101
98
|
return;
|
|
102
99
|
}
|
|
103
100
|
}
|
|
104
|
-
const role = msg.role ||
|
|
101
|
+
const role = msg.role || "user";
|
|
105
102
|
const content = msg.content;
|
|
106
103
|
// System / developer messages
|
|
107
|
-
if (role ===
|
|
108
|
-
add(
|
|
104
|
+
if (role === "system" || role === "developer") {
|
|
105
|
+
add("system_prompt", estimateTokens(content));
|
|
109
106
|
return;
|
|
110
107
|
}
|
|
111
108
|
// String content
|
|
112
|
-
if (typeof content ===
|
|
113
|
-
if (content.includes(
|
|
114
|
-
add(
|
|
109
|
+
if (typeof content === "string") {
|
|
110
|
+
if (content.includes("<system-reminder>")) {
|
|
111
|
+
add("system_injections", estimateTokens(content));
|
|
115
112
|
}
|
|
116
|
-
else if (role ===
|
|
117
|
-
add(
|
|
113
|
+
else if (role === "assistant") {
|
|
114
|
+
add("assistant_text", estimateTokens(content));
|
|
118
115
|
}
|
|
119
116
|
else {
|
|
120
|
-
add(
|
|
117
|
+
add("user_text", estimateTokens(content));
|
|
118
|
+
}
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
// Gemini parts array (role + parts instead of role + content)
|
|
122
|
+
if (msg.parts && Array.isArray(msg.parts)) {
|
|
123
|
+
for (const part of msg.parts) {
|
|
124
|
+
classifyGeminiPart(part, role, add);
|
|
121
125
|
}
|
|
122
126
|
return;
|
|
123
127
|
}
|
|
@@ -130,52 +134,80 @@ function classifyMessage(msg, add) {
|
|
|
130
134
|
}
|
|
131
135
|
// Fallback
|
|
132
136
|
if (content) {
|
|
133
|
-
add(
|
|
137
|
+
add("other", estimateTokens(content));
|
|
134
138
|
}
|
|
135
139
|
}
|
|
136
140
|
function classifyBlock(block, role, add) {
|
|
137
|
-
const type = block.type ||
|
|
138
|
-
if (type ===
|
|
139
|
-
add(
|
|
141
|
+
const type = block.type || "";
|
|
142
|
+
if (type === "tool_use") {
|
|
143
|
+
add("tool_calls", estimateTokens(block));
|
|
140
144
|
return;
|
|
141
145
|
}
|
|
142
|
-
if (type ===
|
|
143
|
-
add(
|
|
146
|
+
if (type === "tool_result") {
|
|
147
|
+
add("tool_results", estimateTokens(block.content || ""));
|
|
144
148
|
return;
|
|
145
149
|
}
|
|
146
|
-
if (type ===
|
|
147
|
-
add(
|
|
150
|
+
if (type === "thinking") {
|
|
151
|
+
add("thinking", estimateTokens(block.thinking || block.text || ""));
|
|
148
152
|
return;
|
|
149
153
|
}
|
|
150
|
-
if (type ===
|
|
151
|
-
add(
|
|
154
|
+
if (type === "image" || type === "image_url") {
|
|
155
|
+
add("images", estimateTokens(block));
|
|
152
156
|
return;
|
|
153
157
|
}
|
|
154
158
|
// Text blocks
|
|
155
|
-
const text = block.text ||
|
|
156
|
-
if (type ===
|
|
157
|
-
if (text.includes(
|
|
158
|
-
add(
|
|
159
|
+
const text = block.text || "";
|
|
160
|
+
if (type === "text" || type === "input_text" || !type) {
|
|
161
|
+
if (text.includes("<system-reminder>")) {
|
|
162
|
+
add("system_injections", estimateTokens(text));
|
|
159
163
|
}
|
|
160
164
|
else if (block.cache_control) {
|
|
161
|
-
add(
|
|
165
|
+
add("cache_markers", estimateTokens(block.cache_control));
|
|
162
166
|
// Still count the text content in its natural category
|
|
163
|
-
if (role ===
|
|
164
|
-
add(
|
|
167
|
+
if (role === "assistant") {
|
|
168
|
+
add("assistant_text", estimateTokens(text));
|
|
165
169
|
}
|
|
166
170
|
else {
|
|
167
|
-
add(
|
|
171
|
+
add("user_text", estimateTokens(text));
|
|
168
172
|
}
|
|
169
173
|
}
|
|
170
|
-
else if (role ===
|
|
171
|
-
add(
|
|
174
|
+
else if (role === "assistant") {
|
|
175
|
+
add("assistant_text", estimateTokens(text));
|
|
172
176
|
}
|
|
173
177
|
else {
|
|
174
|
-
add(
|
|
178
|
+
add("user_text", estimateTokens(text));
|
|
175
179
|
}
|
|
176
180
|
return;
|
|
177
181
|
}
|
|
178
|
-
add(
|
|
182
|
+
add("other", estimateTokens(block));
|
|
183
|
+
}
|
|
184
|
+
function classifyGeminiPart(part, role, add) {
|
|
185
|
+
if (part.text) {
|
|
186
|
+
if (role === "model") {
|
|
187
|
+
add("assistant_text", estimateTokens(part.text));
|
|
188
|
+
}
|
|
189
|
+
else {
|
|
190
|
+
add("user_text", estimateTokens(part.text));
|
|
191
|
+
}
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
if (part.functionCall) {
|
|
195
|
+
add("tool_calls", estimateTokens(part.functionCall));
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (part.functionResponse) {
|
|
199
|
+
add("tool_results", estimateTokens(part.functionResponse));
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
if (part.inlineData || part.fileData) {
|
|
203
|
+
add("images", estimateTokens(part));
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (part.executableCode || part.codeExecutionResult) {
|
|
207
|
+
add("assistant_text", estimateTokens(part));
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
add("other", estimateTokens(part));
|
|
179
211
|
}
|
|
180
212
|
function buildCompositionArray(counts, total) {
|
|
181
213
|
const result = [];
|
|
@@ -205,8 +237,8 @@ export function parseResponseUsage(responseData) {
|
|
|
205
237
|
};
|
|
206
238
|
if (!responseData)
|
|
207
239
|
return result;
|
|
208
|
-
// Streaming response
|
|
209
|
-
if (responseData.streaming && typeof responseData.chunks ===
|
|
240
|
+
// Streaming response: scan SSE chunks for usage
|
|
241
|
+
if (responseData.streaming && typeof responseData.chunks === "string") {
|
|
210
242
|
result.stream = true;
|
|
211
243
|
return parseStreamingUsage(responseData.chunks, result);
|
|
212
244
|
}
|
|
@@ -218,7 +250,24 @@ export function parseResponseUsage(responseData) {
|
|
|
218
250
|
result.cacheReadTokens = u.cache_read_input_tokens || 0;
|
|
219
251
|
result.cacheWriteTokens = u.cache_creation_input_tokens || 0;
|
|
220
252
|
}
|
|
221
|
-
|
|
253
|
+
// Gemini usageMetadata (direct or inside Code Assist wrapper .response)
|
|
254
|
+
const geminiResp = responseData.usageMetadata
|
|
255
|
+
? responseData
|
|
256
|
+
: responseData.response;
|
|
257
|
+
if (geminiResp?.usageMetadata) {
|
|
258
|
+
const u = geminiResp.usageMetadata;
|
|
259
|
+
result.inputTokens = u.promptTokenCount || 0;
|
|
260
|
+
result.outputTokens =
|
|
261
|
+
u.candidatesTokenCount ||
|
|
262
|
+
u.totalTokenCount - (u.promptTokenCount || 0) ||
|
|
263
|
+
0;
|
|
264
|
+
result.cacheReadTokens = u.cachedContentTokenCount || 0;
|
|
265
|
+
}
|
|
266
|
+
result.model =
|
|
267
|
+
responseData.model ||
|
|
268
|
+
responseData.modelVersion ||
|
|
269
|
+
geminiResp?.modelVersion ||
|
|
270
|
+
null;
|
|
222
271
|
if (responseData.stop_reason) {
|
|
223
272
|
result.finishReasons = [responseData.stop_reason];
|
|
224
273
|
}
|
|
@@ -227,45 +276,76 @@ export function parseResponseUsage(responseData) {
|
|
|
227
276
|
.map((c) => c.finish_reason)
|
|
228
277
|
.filter(Boolean);
|
|
229
278
|
}
|
|
279
|
+
else if (responseData.candidates &&
|
|
280
|
+
Array.isArray(responseData.candidates)) {
|
|
281
|
+
result.finishReasons = responseData.candidates
|
|
282
|
+
.map((c) => c.finishReason)
|
|
283
|
+
.filter(Boolean);
|
|
284
|
+
}
|
|
285
|
+
else if (geminiResp?.candidates && Array.isArray(geminiResp.candidates)) {
|
|
286
|
+
result.finishReasons = geminiResp.candidates
|
|
287
|
+
.map((c) => c.finishReason)
|
|
288
|
+
.filter(Boolean);
|
|
289
|
+
}
|
|
230
290
|
return result;
|
|
231
291
|
}
|
|
232
292
|
function parseStreamingUsage(chunks, result) {
|
|
233
293
|
// Parse SSE events looking for usage data
|
|
234
|
-
const lines = chunks.split(
|
|
294
|
+
const lines = chunks.split("\n");
|
|
235
295
|
for (const line of lines) {
|
|
236
|
-
if (!line.startsWith(
|
|
296
|
+
if (!line.startsWith("data: "))
|
|
237
297
|
continue;
|
|
238
298
|
const data = line.slice(6).trim();
|
|
239
|
-
if (data ===
|
|
299
|
+
if (data === "[DONE]")
|
|
240
300
|
continue;
|
|
241
301
|
try {
|
|
242
302
|
const parsed = JSON.parse(data);
|
|
243
303
|
// Anthropic message_start: contains model
|
|
244
|
-
if (parsed.type ===
|
|
304
|
+
if (parsed.type === "message_start" && parsed.message) {
|
|
245
305
|
result.model = parsed.message.model || result.model;
|
|
246
306
|
if (parsed.message.usage) {
|
|
247
307
|
result.inputTokens = parsed.message.usage.input_tokens || 0;
|
|
248
|
-
result.cacheReadTokens =
|
|
249
|
-
|
|
308
|
+
result.cacheReadTokens =
|
|
309
|
+
parsed.message.usage.cache_read_input_tokens || 0;
|
|
310
|
+
result.cacheWriteTokens =
|
|
311
|
+
parsed.message.usage.cache_creation_input_tokens || 0;
|
|
250
312
|
}
|
|
251
313
|
}
|
|
252
314
|
// Anthropic message_delta: contains stop_reason and output token count
|
|
253
|
-
if (parsed.type ===
|
|
315
|
+
if (parsed.type === "message_delta") {
|
|
254
316
|
if (parsed.delta?.stop_reason) {
|
|
255
317
|
result.finishReasons = [parsed.delta.stop_reason];
|
|
256
318
|
}
|
|
257
319
|
if (parsed.usage) {
|
|
258
|
-
result.outputTokens =
|
|
320
|
+
result.outputTokens =
|
|
321
|
+
parsed.usage.output_tokens || result.outputTokens;
|
|
259
322
|
}
|
|
260
323
|
}
|
|
261
324
|
// OpenAI streaming: final chunk with usage
|
|
262
325
|
if (parsed.usage && parsed.choices) {
|
|
263
326
|
result.inputTokens = parsed.usage.prompt_tokens || result.inputTokens;
|
|
264
|
-
result.outputTokens =
|
|
327
|
+
result.outputTokens =
|
|
328
|
+
parsed.usage.completion_tokens || result.outputTokens;
|
|
265
329
|
}
|
|
266
330
|
if (parsed.choices?.[0]?.finish_reason) {
|
|
267
331
|
result.finishReasons = [parsed.choices[0].finish_reason];
|
|
268
332
|
}
|
|
333
|
+
// Gemini streaming: usageMetadata in chunks
|
|
334
|
+
if (parsed.usageMetadata) {
|
|
335
|
+
result.inputTokens =
|
|
336
|
+
parsed.usageMetadata.promptTokenCount || result.inputTokens;
|
|
337
|
+
result.outputTokens =
|
|
338
|
+
parsed.usageMetadata.candidatesTokenCount || result.outputTokens;
|
|
339
|
+
result.cacheReadTokens =
|
|
340
|
+
parsed.usageMetadata.cachedContentTokenCount ||
|
|
341
|
+
result.cacheReadTokens;
|
|
342
|
+
}
|
|
343
|
+
if (parsed.candidates?.[0]?.finishReason) {
|
|
344
|
+
result.finishReasons = [parsed.candidates[0].finishReason];
|
|
345
|
+
}
|
|
346
|
+
if (parsed.modelVersion) {
|
|
347
|
+
result.model = parsed.modelVersion;
|
|
348
|
+
}
|
|
269
349
|
if (parsed.model) {
|
|
270
350
|
result.model = parsed.model;
|
|
271
351
|
}
|
|
@@ -278,13 +358,13 @@ function parseStreamingUsage(chunks, result) {
|
|
|
278
358
|
}
|
|
279
359
|
// --- LHAR Record Builder ---
|
|
280
360
|
function hexId(bytes) {
|
|
281
|
-
return randomBytes(bytes).toString(
|
|
361
|
+
return randomBytes(bytes).toString("hex");
|
|
282
362
|
}
|
|
283
363
|
function traceIdFromConversation(conversationId) {
|
|
284
364
|
if (!conversationId)
|
|
285
365
|
return hexId(16);
|
|
286
366
|
// Deterministic: hash the conversationId to a 32-hex-char trace ID
|
|
287
|
-
return createHash(
|
|
367
|
+
return createHash("sha256").update(conversationId).digest("hex").slice(0, 32);
|
|
288
368
|
}
|
|
289
369
|
export function buildLharRecord(entry, prevEntries) {
|
|
290
370
|
const ci = entry.contextInfo;
|
|
@@ -296,11 +376,11 @@ export function buildLharRecord(entry, prevEntries) {
|
|
|
296
376
|
// Sequence + growth must be derived from a stable ordering.
|
|
297
377
|
// Use oldest-first timestamp ordering within the conversation; tie-break by id.
|
|
298
378
|
let convoEntries = entry.conversationId
|
|
299
|
-
? prevEntries.filter(e => e.conversationId === entry.conversationId)
|
|
379
|
+
? prevEntries.filter((e) => e.conversationId === entry.conversationId)
|
|
300
380
|
: [entry];
|
|
301
|
-
// Make buildLharRecord
|
|
381
|
+
// Make buildLharRecord work even if the caller doesn't include `entry` in `prevEntries`.
|
|
302
382
|
if (entry.conversationId) {
|
|
303
|
-
const found = convoEntries.some(e => e.id === entry.id && e.timestamp === entry.timestamp);
|
|
383
|
+
const found = convoEntries.some((e) => e.id === entry.id && e.timestamp === entry.timestamp);
|
|
304
384
|
if (!found)
|
|
305
385
|
convoEntries = [...convoEntries, entry];
|
|
306
386
|
}
|
|
@@ -310,9 +390,9 @@ export function buildLharRecord(entry, prevEntries) {
|
|
|
310
390
|
return dt;
|
|
311
391
|
return a.id - b.id;
|
|
312
392
|
});
|
|
313
|
-
let convoIndex = convoEntries.findIndex(e => e.id === entry.id && e.timestamp === entry.timestamp);
|
|
393
|
+
let convoIndex = convoEntries.findIndex((e) => e.id === entry.id && e.timestamp === entry.timestamp);
|
|
314
394
|
if (convoIndex < 0)
|
|
315
|
-
convoIndex = convoEntries.findIndex(e => e.id === entry.id);
|
|
395
|
+
convoIndex = convoEntries.findIndex((e) => e.id === entry.id);
|
|
316
396
|
if (convoIndex < 0)
|
|
317
397
|
convoIndex = 0;
|
|
318
398
|
const sequence = convoIndex + 1;
|
|
@@ -322,18 +402,22 @@ export function buildLharRecord(entry, prevEntries) {
|
|
|
322
402
|
const tokensAdded = prevEntry ? ci.totalTokens - prevTokens : null;
|
|
323
403
|
const compactionDetected = tokensAdded !== null && tokensAdded < 0;
|
|
324
404
|
// Agent role
|
|
325
|
-
const agentRole = entry.agentKey ?
|
|
405
|
+
const agentRole = entry.agentKey ? "subagent" : "main";
|
|
326
406
|
// Tokens per second
|
|
327
407
|
let tokensPerSecond = null;
|
|
328
408
|
if (entry.timings && entry.timings.receive_ms > 0 && usage.outputTokens > 0) {
|
|
329
|
-
tokensPerSecond =
|
|
409
|
+
tokensPerSecond =
|
|
410
|
+
Math.round((usage.outputTokens / entry.timings.receive_ms) * 1000 * 10) /
|
|
411
|
+
10;
|
|
330
412
|
}
|
|
331
|
-
const timings = entry.timings
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
413
|
+
const timings = entry.timings
|
|
414
|
+
? {
|
|
415
|
+
...entry.timings,
|
|
416
|
+
tokens_per_second: tokensPerSecond,
|
|
417
|
+
}
|
|
418
|
+
: null;
|
|
335
419
|
return {
|
|
336
|
-
type:
|
|
420
|
+
type: "entry",
|
|
337
421
|
id: randomUUID(),
|
|
338
422
|
trace_id: traceIdFromConversation(entry.conversationId),
|
|
339
423
|
span_id: hexId(8),
|
|
@@ -341,7 +425,7 @@ export function buildLharRecord(entry, prevEntries) {
|
|
|
341
425
|
timestamp: entry.timestamp,
|
|
342
426
|
sequence,
|
|
343
427
|
source: {
|
|
344
|
-
tool: entry.source ||
|
|
428
|
+
tool: entry.source || "unknown",
|
|
345
429
|
tool_version: null,
|
|
346
430
|
agent_role: agentRole,
|
|
347
431
|
collector: COLLECTOR_NAME,
|
|
@@ -372,7 +456,7 @@ export function buildLharRecord(entry, prevEntries) {
|
|
|
372
456
|
cost_usd: entry.costUsd,
|
|
373
457
|
},
|
|
374
458
|
http: {
|
|
375
|
-
method:
|
|
459
|
+
method: "POST",
|
|
376
460
|
url: entry.targetUrl,
|
|
377
461
|
status_code: entry.httpStatus,
|
|
378
462
|
api_format: ci.apiFormat,
|
|
@@ -410,7 +494,7 @@ export function buildLharRecord(entry, prevEntries) {
|
|
|
410
494
|
// --- Session Line ---
|
|
411
495
|
export function buildSessionLine(conversationId, conversation, model) {
|
|
412
496
|
return {
|
|
413
|
-
type:
|
|
497
|
+
type: "session",
|
|
414
498
|
trace_id: traceIdFromConversation(conversationId),
|
|
415
499
|
started_at: conversation.firstSeen,
|
|
416
500
|
tool: conversation.source,
|
|
@@ -437,11 +521,11 @@ export function toLharJsonl(entries, conversations) {
|
|
|
437
521
|
}
|
|
438
522
|
lines.push(JSON.stringify(record));
|
|
439
523
|
}
|
|
440
|
-
return lines.join(
|
|
524
|
+
return `${lines.join("\n")}\n`;
|
|
441
525
|
}
|
|
442
526
|
export function toLharJson(entries, conversations) {
|
|
443
527
|
const sorted = [...entries].sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
|
|
444
|
-
const records = sorted.map(entry => buildLharRecord(entry, entries));
|
|
528
|
+
const records = sorted.map((entry) => buildLharRecord(entry, entries));
|
|
445
529
|
// Build sessions from conversations map
|
|
446
530
|
const sessions = [];
|
|
447
531
|
const seenTraces = new Set();
|
|
@@ -449,7 +533,7 @@ export function toLharJson(entries, conversations) {
|
|
|
449
533
|
if (!seenTraces.has(record.trace_id)) {
|
|
450
534
|
seenTraces.add(record.trace_id);
|
|
451
535
|
const convo = record.trace_id
|
|
452
|
-
? Array.from(conversations.values()).find(c => traceIdFromConversation(c.id) === record.trace_id)
|
|
536
|
+
? Array.from(conversations.values()).find((c) => traceIdFromConversation(c.id) === record.trace_id)
|
|
453
537
|
: undefined;
|
|
454
538
|
sessions.push({
|
|
455
539
|
trace_id: record.trace_id,
|