@tangle-network/traces 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -29
- package/dist/chunk-YYVPAOOD.js +1935 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +129 -1640
- package/dist/index.d.ts +659 -0
- package/dist/index.js +126 -0
- package/package.json +13 -3
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,659 @@
|
|
|
1
|
+
import { ToolUseMetrics, InMemoryTraceStore } from '@tangle-network/agent-eval';
|
|
2
|
+
import { StuckLoopReport } from '@tangle-network/agent-eval/pipelines';
|
|
3
|
+
import { AxAIService } from '@ax-llm/ax';
|
|
4
|
+
import { AnalystRegistry, buildDefaultAnalystRegistry } from '@tangle-network/agent-eval/analyst';
|
|
5
|
+
export { Analyst, AnalystContext, AnalystFinding, AnalystRegistry, buildDefaultAnalystRegistry, makeFinding } from '@tangle-network/agent-eval/analyst';
|
|
6
|
+
import { RedactionRule, RedactionReport } from '@tangle-network/agent-eval/traces';
|
|
7
|
+
export { DEFAULT_REDACTION_RULES, RedactionReport, RedactionRule, redactString, redactValue } from '@tangle-network/agent-eval/traces';
|
|
8
|
+
import { TraceSpanEvent } from '@tangle-network/agent-eval/hosted';
|
|
9
|
+
export { HostedClient, createHostedClient, hostedClientFromEnv } from '@tangle-network/agent-eval/hosted';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* OTLP-flat-line emitter — the canonical normalized trace shape.
|
|
13
|
+
*
|
|
14
|
+
* Every harness adapter projects its native session log onto `OtlpSpan[]`
|
|
15
|
+
* and we serialize one span per JSONL line. This is the exact wire shape
|
|
16
|
+
* `@tangle-network/agent-eval`'s `OtlpFileTraceStore` indexes
|
|
17
|
+
* (`projectOtlpFlatLine` + `extractOtlpAttributes`) AND the OpenInference
|
|
18
|
+
* OTLP form HALO consumes — so one emitted file feeds both engines.
|
|
19
|
+
*
|
|
20
|
+
* Attribute vocabulary the downstream analysts key off:
|
|
21
|
+
* - `openinference.span.kind` → span kind (AGENT | LLM | TOOL | CHAIN)
|
|
22
|
+
* - `service.name` → harness id (claude-code, codex, …)
|
|
23
|
+
* - `agent.name` → agent / subagent name
|
|
24
|
+
* - `llm.model_name` → model
|
|
25
|
+
* - `llm.input_tokens` / `llm.output_tokens` → behavioral token trajectories
|
|
26
|
+
* - `tool.name` → tool histogram (monoculture / no-verify)
|
|
27
|
+
* - `step` → run order (falls back to start_time)
|
|
28
|
+
*/
|
|
29
|
+
type OtlpSpanKind = 'AGENT' | 'LLM' | 'TOOL' | 'CHAIN' | 'SPAN';
|
|
30
|
+
type OtlpStatusCode = 'OK' | 'ERROR' | 'UNSET';
|
|
31
|
+
interface OtlpSpan {
|
|
32
|
+
trace_id: string;
|
|
33
|
+
span_id: string;
|
|
34
|
+
parent_span_id: string | null;
|
|
35
|
+
name: string;
|
|
36
|
+
start_time: string;
|
|
37
|
+
end_time: string;
|
|
38
|
+
status: {
|
|
39
|
+
code: OtlpStatusCode;
|
|
40
|
+
message?: string;
|
|
41
|
+
};
|
|
42
|
+
attributes: Record<string, unknown>;
|
|
43
|
+
}
|
|
44
|
+
interface SpanInput {
|
|
45
|
+
traceId: string;
|
|
46
|
+
spanId: string;
|
|
47
|
+
parentSpanId?: string | null;
|
|
48
|
+
name: string;
|
|
49
|
+
kind: OtlpSpanKind;
|
|
50
|
+
startTime: string;
|
|
51
|
+
/** Defaults to startTime when the source has no explicit end. */
|
|
52
|
+
endTime?: string;
|
|
53
|
+
status?: OtlpStatusCode;
|
|
54
|
+
statusMessage?: string;
|
|
55
|
+
service?: string | null;
|
|
56
|
+
agent?: string | null;
|
|
57
|
+
model?: string | null;
|
|
58
|
+
tool?: string | null;
|
|
59
|
+
inputTokens?: number | null;
|
|
60
|
+
outputTokens?: number | null;
|
|
61
|
+
/** Run-order pivot; the behavioral analyst orders trajectories by this. */
|
|
62
|
+
step?: number;
|
|
63
|
+
/** Verbatim content the agentic analysts read via regex search. */
|
|
64
|
+
content?: string | null;
|
|
65
|
+
/** Extra attributes merged last (escape hatch for harness-specific signal). */
|
|
66
|
+
extra?: Record<string, unknown>;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Build one OTLP span line from a harness event. Only sets attribute keys
|
|
70
|
+
* that carry real signal — an absent token count is omitted, not zeroed,
|
|
71
|
+
* so the behavioral analyst's "no LLM data" case stays honest.
|
|
72
|
+
*/
|
|
73
|
+
declare function span(input: SpanInput): OtlpSpan;
|
|
74
|
+
/** Serialize spans to OTLP-JSONL (one span per line, trailing newline). */
|
|
75
|
+
declare function serializeSpans(spans: readonly OtlpSpan[]): string;
|
|
76
|
+
/** Write spans to an OTLP-JSONL file (a temp file when no path is given). */
|
|
77
|
+
declare function writeOtlpFile(spans: readonly OtlpSpan[], outPath?: string): Promise<string>;
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Harness trace adapter contract.
|
|
81
|
+
*
|
|
82
|
+
* Each coding-agent harness writes its session log in a different on-disk
|
|
83
|
+
* format. An adapter knows (a) where that harness stores sessions and
|
|
84
|
+
* (b) how to project one session file/dir onto normalized `OtlpSpan[]`.
|
|
85
|
+
*
|
|
86
|
+
* Adapters are the read counterpart of cli-bridge's existing per-harness
|
|
87
|
+
* `BackendModule` resume logic — they live next to the code that already
|
|
88
|
+
* locates these files.
|
|
89
|
+
*/
|
|
90
|
+
|
|
91
|
+
/** A single discovered session, before parsing. */
|
|
92
|
+
interface SessionRef {
|
|
93
|
+
/** Harness id (matches the nix profile / backend name). */
|
|
94
|
+
harness: string;
|
|
95
|
+
/** Stable session identifier (uuid, thread id, or derived from path). */
|
|
96
|
+
sessionId: string;
|
|
97
|
+
/** Absolute path to the session file (or session root dir for split formats). */
|
|
98
|
+
path: string;
|
|
99
|
+
/** Working directory the session ran in, when recoverable. */
|
|
100
|
+
cwd: string | null;
|
|
101
|
+
/** Last-modified epoch ms — used for `--last N` recency ordering. */
|
|
102
|
+
mtimeMs: number;
|
|
103
|
+
}
|
|
104
|
+
interface LocateOptions {
|
|
105
|
+
/** Limit discovery to sessions whose cwd matches (exact or prefix). */
|
|
106
|
+
cwd?: string;
|
|
107
|
+
/** Only sessions modified at/after this epoch ms. */
|
|
108
|
+
sinceMs?: number;
|
|
109
|
+
}
|
|
110
|
+
interface HarnessTraceAdapter {
|
|
111
|
+
/** Canonical harness id this adapter handles. */
|
|
112
|
+
readonly harness: string;
|
|
113
|
+
/** Aliases that resolve to this adapter (forks / variants / ACP wrappers). */
|
|
114
|
+
readonly aliases?: readonly string[];
|
|
115
|
+
/** Discover session files for this harness on disk. */
|
|
116
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
117
|
+
/** Parse one discovered session into normalized OTLP spans. */
|
|
118
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Harness adapter registry — resolves a harness id (or alias) to the
|
|
123
|
+
* adapter that knows its on-disk session format.
|
|
124
|
+
*
|
|
125
|
+
* Format families collapse many harnesses onto few adapters:
|
|
126
|
+
* - claude family (claude / claudish / openclaw / nanoclaw) share the
|
|
127
|
+
* `~/.claude`-style transcript JSONL
|
|
128
|
+
* - codex family (codex / codex-acp) share the rollout JSONL
|
|
129
|
+
* - gemini (gemini-cli) and its fork qwen (qwen-code) share the chat JSON
|
|
130
|
+
* under distinct home dirs
|
|
131
|
+
* - factory-droids (droid) has its own `~/.factory` rollout JSONL
|
|
132
|
+
*/
|
|
133
|
+
|
|
134
|
+
declare function listAdapters(): readonly HarnessTraceAdapter[];
|
|
135
|
+
declare function resolveAdapter(harness: string): HarnessTraceAdapter | undefined;
|
|
136
|
+
declare function knownHarnesses(): string[];
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Claude Code adapter — `~/.claude/projects/<encoded-cwd>/<session>.jsonl`.
|
|
140
|
+
*
|
|
141
|
+
* Each line is one transcript event: `user`, `assistant`, `attachment`,
|
|
142
|
+
* plus metadata events we ignore. Assistant events carry `message.usage`
|
|
143
|
+
* (token trajectory) and `message.content[]` tool_use blocks; tool results
|
|
144
|
+
* arrive as `tool_result` blocks in the following user message or as a
|
|
145
|
+
* `tool_result` attachment. Subagent runs live in a sibling
|
|
146
|
+
* `<session>/subagents/agent-*.jsonl` with a `.meta.json` carrying the
|
|
147
|
+
* spawning `toolUseId`, so we parent each subagent under its `Agent` call.
|
|
148
|
+
*
|
|
149
|
+
* Shared by the claudish / openclaw / nanoclaw forks via aliases — they
|
|
150
|
+
* write the same transcript shape.
|
|
151
|
+
*/
|
|
152
|
+
|
|
153
|
+
declare class ClaudeAdapter implements HarnessTraceAdapter {
|
|
154
|
+
readonly harness = "claude-code";
|
|
155
|
+
readonly aliases: readonly ["claude", "claudish", "openclaw", "nanoclaw"];
|
|
156
|
+
private root;
|
|
157
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
158
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
159
|
+
/** Parse `<session>/subagents/agent-*.jsonl`, parenting each under its Agent call. */
|
|
160
|
+
private foldSubagents;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* OpenAI Codex adapter — `~/.codex/sessions/<Y>/<M>/<D>/rollout-*.jsonl`.
|
|
165
|
+
*
|
|
166
|
+
* Line types: `session_meta` (id + cwd), `turn_context`, `event_msg`
|
|
167
|
+
* (carries `token_count` with per-turn `last_token_usage`), and
|
|
168
|
+
* `response_item` (the OpenAI Responses items: `message`, `reasoning`,
|
|
169
|
+
* `function_call`, `function_call_output`).
|
|
170
|
+
*
|
|
171
|
+
* Token trajectory comes from the `token_count` deltas (Codex puts usage
|
|
172
|
+
* on events, not on the message). Tools come from `function_call`, with
|
|
173
|
+
* status backfilled from the matching `function_call_output`.
|
|
174
|
+
*
|
|
175
|
+
* Shared by the codex-acp wrapper via alias (same rollout format).
|
|
176
|
+
*/
|
|
177
|
+
|
|
178
|
+
declare class CodexAdapter implements HarnessTraceAdapter {
|
|
179
|
+
readonly harness = "codex";
|
|
180
|
+
readonly aliases: readonly ["codex-acp"];
|
|
181
|
+
private root;
|
|
182
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
183
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* OpenCode adapter — `~/.local/share/opencode/storage/`.
|
|
188
|
+
*
|
|
189
|
+
* Split storage: `message/<sessionID>/<messageID>.json` carries role +
|
|
190
|
+
* `tokens` + `modelID` + `time`; `part/<messageID>/<partID>.json` carries
|
|
191
|
+
* the turn's parts (`text`, `reasoning`, `tool`, `step-*`). Tool parts
|
|
192
|
+
* hold `tool`, `callID`, and `state.{status,error}` — `status === 'error'`
|
|
193
|
+
* marks a failed call.
|
|
194
|
+
*/
|
|
195
|
+
|
|
196
|
+
declare class OpencodeAdapter implements HarnessTraceAdapter {
|
|
197
|
+
readonly harness = "opencode";
|
|
198
|
+
private storage;
|
|
199
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
200
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Gemini CLI family — `<home>/tmp/<projectHash>/chats/session-*.json`.
|
|
205
|
+
*
|
|
206
|
+
* One JSON object per chat: `{ sessionId, projectHash, messages[] }`.
|
|
207
|
+
* Each message carries `model`, `tokens.{input,output}`, `content`, and
|
|
208
|
+
* `toolCalls[]` with `{ name, args, status, result }`.
|
|
209
|
+
*
|
|
210
|
+
* Gemini CLI only — Qwen Code forked and diverged (flat Claude-style
|
|
211
|
+
* JSONL under `~/.qwen/projects/`); it has its own adapter.
|
|
212
|
+
*/
|
|
213
|
+
|
|
214
|
+
interface GeminiFamilyConfig {
|
|
215
|
+
harness: string;
|
|
216
|
+
service: string;
|
|
217
|
+
/** Directory under $HOME holding `tmp/<hash>/chats/` (e.g. '.gemini', '.qwen'). */
|
|
218
|
+
homeDirName: string;
|
|
219
|
+
aliases?: readonly string[];
|
|
220
|
+
}
|
|
221
|
+
/** Shared base for Gemini CLI and its forks (Qwen Code). */
|
|
222
|
+
declare class GeminiFamilyAdapter implements HarnessTraceAdapter {
|
|
223
|
+
readonly harness: string;
|
|
224
|
+
readonly aliases?: readonly string[];
|
|
225
|
+
private readonly service;
|
|
226
|
+
private readonly homeDirName;
|
|
227
|
+
constructor(cfg: GeminiFamilyConfig);
|
|
228
|
+
private chatsRoot;
|
|
229
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
230
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
231
|
+
}
|
|
232
|
+
declare class GeminiAdapter extends GeminiFamilyAdapter {
|
|
233
|
+
constructor();
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Qwen Code adapter — `~/.qwen/projects/<dashed-cwd>/chats/<sessionId>.jsonl`.
|
|
238
|
+
*
|
|
239
|
+
* Schema confirmed from QwenLM/qwen-code source (chatRecordingService.ts):
|
|
240
|
+
* flat Claude-Code-style `ChatRecord` per JSONL line — NOT the Gemini CLI
|
|
241
|
+
* wrapper. `type` ∈ user|assistant|tool_result|system; `message` is a raw
|
|
242
|
+
* `@google/genai` Content `{ role, parts[] }`; tokens use Gemini-API names
|
|
243
|
+
* in `usageMetadata`; tool calls/results are `functionCall`/`functionResponse`
|
|
244
|
+
* parts.
|
|
245
|
+
*
|
|
246
|
+
* Schema-from-source; parse unverified against local data (no `~/.qwen`
|
|
247
|
+
* sessions on this machine).
|
|
248
|
+
*/
|
|
249
|
+
|
|
250
|
+
declare class QwenAdapter implements HarnessTraceAdapter {
|
|
251
|
+
readonly harness = "qwen";
|
|
252
|
+
readonly aliases: readonly ["qwen-code"];
|
|
253
|
+
private root;
|
|
254
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
255
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Factory Droid adapter — `~/.factory/sessions/<encoded-cwd>/<uuid>.jsonl`
|
|
260
|
+
* plus a `<uuid>.settings.json` sidecar.
|
|
261
|
+
*
|
|
262
|
+
* Confirmed from disk + two parsers (droid-sync-plugin, tokscale): the JSONL
|
|
263
|
+
* has only `session_start` and `message` lines; `message.content[]` carries
|
|
264
|
+
* Anthropic-style blocks (`text`, `thinking`, `tool_use`, `tool_result`). The
|
|
265
|
+
* sidecar holds `model` and session-total `tokenUsage` — there is NO per-turn
|
|
266
|
+
* usage in the transcript, so token-growth can't be computed for Factory (the
|
|
267
|
+
* tool/loop signals still work). `<encoded-cwd>` is slash→dash. Tool-error
|
|
268
|
+
* flag (`is_error`) is inferred (Anthropic convention), not source-confirmed.
|
|
269
|
+
*/
|
|
270
|
+
|
|
271
|
+
declare class FactoryAdapter implements HarnessTraceAdapter {
|
|
272
|
+
readonly harness = "factory";
|
|
273
|
+
readonly aliases: readonly ["factory-droids", "droid"];
|
|
274
|
+
private root;
|
|
275
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
276
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Pi adapter — `~/.pi/agent/sessions/<encoded-cwd>/<ts>_<uuid>.jsonl`.
|
|
281
|
+
*
|
|
282
|
+
* Line types: `session` (id + cwd), `model_change`, `thinking_level_change`,
|
|
283
|
+
* and `message`. A `message` line wraps `message.{role, model, provider,
|
|
284
|
+
* content[], usage.{input,output}, stopReason, errorMessage}`. Tool calls
|
|
285
|
+
* ride inside `message.content[]` as tool blocks.
|
|
286
|
+
*/
|
|
287
|
+
|
|
288
|
+
declare class PiAdapter implements HarnessTraceAdapter {
|
|
289
|
+
readonly harness = "pi";
|
|
290
|
+
private root;
|
|
291
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
292
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Sourcegraph Amp adapter — `${XDG_DATA_HOME:-~/.local/share}/amp/threads/T-*.json`.
|
|
297
|
+
*
|
|
298
|
+
* One JSON object per thread: `{ id, created, messages[], usageLedger }`.
|
|
299
|
+
* Messages carry `role`, `usage` (camelCase on disk: `model`, `inputTokens`,
|
|
300
|
+
* `outputTokens`, `cacheReadInputTokens`, `cacheCreationInputTokens`) and
|
|
301
|
+
* Anthropic-style `content[]` blocks (`text`, `tool_use`, `tool_result` with
|
|
302
|
+
* `is_error`).
|
|
303
|
+
*
|
|
304
|
+
* Path/format/usage from the `tokscale` parser (high conf); content-block
|
|
305
|
+
* field names medium-conf; parse unverified against local data.
|
|
306
|
+
*/
|
|
307
|
+
|
|
308
|
+
declare class AmpAdapter implements HarnessTraceAdapter {
|
|
309
|
+
readonly harness = "amp";
|
|
310
|
+
private threadsRoot;
|
|
311
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
312
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* GitHub Copilot CLI adapter — `${COPILOT_HOME:-~/.copilot}/session-state/<id>/events.jsonl`.
|
|
317
|
+
*
|
|
318
|
+
* Event envelope per line: `{ id, timestamp, parentId, type, data }`. Relevant
|
|
319
|
+
* types: `assistant.message` (`outputTokens`, `toolRequests[]`),
|
|
320
|
+
* `tool.execution_start` (`toolCallId`, `toolName`, `arguments`),
|
|
321
|
+
* `tool.execution_complete` (`toolCallId`, `success`, `error{message}`, `model`),
|
|
322
|
+
* `assistant.usage` (ephemeral: `model`, `inputTokens`, `outputTokens`).
|
|
323
|
+
*
|
|
324
|
+
* Sub-agents interleave in the same stream — tool calls/results are joined by
|
|
325
|
+
* `toolCallId`, never adjacency. Lines may contain stray newlines/separators;
|
|
326
|
+
* parse line-tolerant. Schema from GitHub docs (high conf); parse unverified
|
|
327
|
+
* against local data.
|
|
328
|
+
*/
|
|
329
|
+
|
|
330
|
+
declare class CopilotAdapter implements HarnessTraceAdapter {
|
|
331
|
+
readonly harness = "github-copilot";
|
|
332
|
+
readonly aliases: readonly ["copilot"];
|
|
333
|
+
private root;
|
|
334
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
335
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/**
|
|
339
|
+
* Forge adapter (antinomyhq/forge) — `/dump` JSON exports.
|
|
340
|
+
*
|
|
341
|
+
* Forge's primary store is SQLite (`~/.forge/.forge.db`, `conversations.context`
|
|
342
|
+
* column). Reading it live needs a native sqlite dep, so v1 parses the
|
|
343
|
+
* dependency-free `/dump` artifact instead: `<cwd>/YYYY-MM-DD_HH-MM-SS-dump.json`,
|
|
344
|
+
* a single flattened `Context{ conversation_id, messages[] }`.
|
|
345
|
+
*
|
|
346
|
+
* Flattened message entry (from forge Rust serde, high conf): `{ text: {role,
|
|
347
|
+
* content, tool_calls?[], model?}, usage }` or `{ tool: {name, call_id?,
|
|
348
|
+
* output:{is_error, values}}, usage }`. Tokens are `TokenCount` = `{actual:N}`
|
|
349
|
+
* or `{approx:N}`. Parse unverified against local data.
|
|
350
|
+
*/
|
|
351
|
+
|
|
352
|
+
declare class ForgeAdapter implements HarnessTraceAdapter {
|
|
353
|
+
readonly harness = "forge";
|
|
354
|
+
readonly aliases: readonly ["forgecode"];
|
|
355
|
+
/** Directories scanned for `*-dump.json` exports. */
|
|
356
|
+
private dumpDirs;
|
|
357
|
+
locate(opts?: LocateOptions): Promise<SessionRef[]>;
|
|
358
|
+
parse(ref: SessionRef): Promise<OtlpSpan[]>;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Run agent-eval's shipped trace pipelines over normalized harness spans.
|
|
363
|
+
*
|
|
364
|
+
* These are the deterministic loop/thrash detectors that already exist in
|
|
365
|
+
* `@tangle-network/agent-eval`:
|
|
366
|
+
* - stuckLoopView — same tool + same args ≥ N times (the loop signal)
|
|
367
|
+
* - computeToolUseMetrics — duplicate-call / retry / error rates per run
|
|
368
|
+
*
|
|
369
|
+
* `toolWasteView` is intentionally NOT used: its default heuristic needs
|
|
370
|
+
* verbatim tool *results* + per-turn LLM `messages`, which traces
|
|
371
|
+
* doesn't capture from harness logs (we store tool args + error status). Its
|
|
372
|
+
* signal — repeated/failed calls — is already covered by the duplicate/retry/
|
|
373
|
+
* error rates above, so adding it would only emit a misleading waste %.
|
|
374
|
+
*
|
|
375
|
+
* Both are cheap ($0, deterministic), so they're safe to run continuously in
|
|
376
|
+
* `watch` mode, over the OTLP spans traces already produces.
|
|
377
|
+
*/
|
|
378
|
+
|
|
379
|
+
interface PipelineReport {
|
|
380
|
+
stuckLoops: StuckLoopReport;
|
|
381
|
+
toolUse: ToolUseMetrics[];
|
|
382
|
+
}
|
|
383
|
+
interface PipelineOptions {
|
|
384
|
+
/** Minimum repeated identical calls to flag a loop (default 3). */
|
|
385
|
+
minLoopOccurrences?: number;
|
|
386
|
+
}
|
|
387
|
+
declare function runPipelines(spans: readonly OtlpSpan[], opts?: PipelineOptions): Promise<PipelineReport>;
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* Project normalized `OtlpSpan[]` onto agent-eval's runtime trace store.
|
|
391
|
+
*
|
|
392
|
+
* The behavioral/agentic analysts consume the OTLP analyst store; the
|
|
393
|
+
* loop/waste pipelines (`stuckLoopView`, `toolWasteView`) and tool-use
|
|
394
|
+
* metrics consume the runtime `TraceStore` (`Span` with `toolName`/`args`/
|
|
395
|
+
* `startedAt`). This converter is the single bridge — one function, every
|
|
396
|
+
* harness — so traces reuses those shipped detectors instead of
|
|
397
|
+
* reimplementing loop/stall detection.
|
|
398
|
+
*
|
|
399
|
+
* Tool `args` is the verbatim tool input we stored on the span's `content`;
|
|
400
|
+
* `stuckLoopView` hashes it (`argHash`) so identical repeated calls collapse
|
|
401
|
+
* to one loop finding.
|
|
402
|
+
*/
|
|
403
|
+
|
|
404
|
+
interface RuntimeTrace {
|
|
405
|
+
store: InMemoryTraceStore;
|
|
406
|
+
runIds: string[];
|
|
407
|
+
}
|
|
408
|
+
declare function toRuntimeStore(spans: readonly OtlpSpan[]): Promise<RuntimeTrace>;
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* Run the agent-eval analyst suite over normalized spans.
|
|
412
|
+
*
|
|
413
|
+
* Spans → OTLP-JSONL file → `OtlpFileTraceStore` → `buildDefaultAnalystRegistry`.
|
|
414
|
+
* With no Ax service the deterministic behavioral analyst runs alone (zero
|
|
415
|
+
* LLM, model-agnostic). Supply `ai` to add the agentic RLM kinds
|
|
416
|
+
* (failure-mode / knowledge-gap / knowledge-poisoning / improvement).
|
|
417
|
+
*
|
|
418
|
+
* The written OTLP file is the canonical artifact — the same file feeds HALO
|
|
419
|
+
* (`halo <file> -p "diagnose"`), so analysis is never locked to one engine.
|
|
420
|
+
*/
|
|
421
|
+
|
|
422
|
+
interface AnalyzeOptions {
|
|
423
|
+
/** Ax service enabling the agentic RLM kinds. Omit → deterministic only. */
|
|
424
|
+
ai?: AxAIService;
|
|
425
|
+
model?: string;
|
|
426
|
+
/** USD cap across agentic analysts. */
|
|
427
|
+
budgetUsd?: number;
|
|
428
|
+
/**
|
|
429
|
+
* Bring your own analyst suite. When set, this registry runs over the trace
|
|
430
|
+
* store INSTEAD of the built-in deterministic suite — the seam for running
|
|
431
|
+
* your own agents/detectors over sessions. Register custom `Analyst`s with
|
|
432
|
+
* `@tangle-network/agent-eval`'s `AnalystRegistry`.
|
|
433
|
+
*/
|
|
434
|
+
registry?: AnalystRegistry;
|
|
435
|
+
/** Where to write the OTLP-JSONL artifact. Defaults to a temp file. */
|
|
436
|
+
otlpOutPath?: string;
|
|
437
|
+
runId?: string;
|
|
438
|
+
log?: (msg: string, fields?: Record<string, unknown>) => void;
|
|
439
|
+
}
|
|
440
|
+
interface AnalyzeResult {
|
|
441
|
+
/** Path to the OTLP-JSONL artifact (feeds HALO too). */
|
|
442
|
+
otlpPath: string;
|
|
443
|
+
result: Awaited<ReturnType<ReturnType<typeof buildDefaultAnalystRegistry>['run']>>;
|
|
444
|
+
}
|
|
445
|
+
declare function analyzeSpans(spans: readonly OtlpSpan[], opts?: AnalyzeOptions): Promise<AnalyzeResult>;
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* Live trace observer — the reusable, event-driven core behind `traces watch`.
|
|
449
|
+
*
|
|
450
|
+
* `watchSessions` tails active coding-agent sessions across harnesses and emits
|
|
451
|
+
* structured events as it goes: every session's pipeline report each tick
|
|
452
|
+
* (`onReport`), and a deduped signal whenever a stuck loop first appears or
|
|
453
|
+
* grows (`onLoop`). It's read-only (never touches the agent or its harness) and
|
|
454
|
+
* cancellable via an `AbortSignal`, so a third party can subscribe to the live
|
|
455
|
+
* findings stream and feed it into any system — a dashboard, an alerter, a
|
|
456
|
+
* steering layer, or another agent — without shelling out to the CLI.
|
|
457
|
+
*/
|
|
458
|
+
|
|
459
|
+
/** A newly-observed (or grown) stuck loop in a live session. */
|
|
460
|
+
interface ObservedLoop {
|
|
461
|
+
harness: string;
|
|
462
|
+
sessionId: string;
|
|
463
|
+
cwd: string | null;
|
|
464
|
+
toolName: string;
|
|
465
|
+
argHash: string;
|
|
466
|
+
/** Identical repeated calls observed so far. */
|
|
467
|
+
occurrences: number;
|
|
468
|
+
/** Milliseconds between the first and last call in the loop. */
|
|
469
|
+
windowMs: number;
|
|
470
|
+
}
|
|
471
|
+
interface ObserverOptions {
|
|
472
|
+
/** Harness ids/aliases to observe. Omit (or pass `all: true`) → every harness. */
|
|
473
|
+
harnesses?: string[];
|
|
474
|
+
all?: boolean;
|
|
475
|
+
/** Observe these adapters instead of the built-in registry — plug in your
|
|
476
|
+
* own `HarnessTraceAdapter`s. Takes precedence over `harnesses`/`all`. */
|
|
477
|
+
adapters?: HarnessTraceAdapter[];
|
|
478
|
+
/** Only observe sessions whose cwd matches (exact/prefix). */
|
|
479
|
+
cwd?: string;
|
|
480
|
+
/** Consider sessions active within this window (ms). Default 30 min. */
|
|
481
|
+
windowMs?: number;
|
|
482
|
+
/** Poll interval (ms). Default 5 s. */
|
|
483
|
+
intervalMs?: number;
|
|
484
|
+
/** Min identical repeated calls before a loop is reported. Default 3. */
|
|
485
|
+
minLoopOccurrences?: number;
|
|
486
|
+
/** Cancel the observer; `watchSessions` resolves when this aborts. */
|
|
487
|
+
signal?: AbortSignal;
|
|
488
|
+
/** Fired once per stuck loop the first time it reaches threshold, and again
|
|
489
|
+
* each time its occurrence count grows — deduped so you only see new signal. */
|
|
490
|
+
onLoop?: (loop: ObservedLoop) => void | Promise<void>;
|
|
491
|
+
/** Fired for every observed session each tick, with its full pipeline report
|
|
492
|
+
* (stuck loops + tool-use metrics) — react to anything, not just loops. */
|
|
493
|
+
onReport?: (ref: SessionRef, report: PipelineReport) => void | Promise<void>;
|
|
494
|
+
/** Per-session parse/pipeline error (the observer keeps going). */
|
|
495
|
+
onError?: (error: unknown, ref?: SessionRef) => void;
|
|
496
|
+
/** Fired after each poll cycle. */
|
|
497
|
+
onTick?: (stats: {
|
|
498
|
+
sessions: number;
|
|
499
|
+
}) => void;
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Observe active sessions until `opts.signal` aborts. Returns a promise that
|
|
503
|
+
* resolves on abort. Errors in a single session are routed to `onError` and
|
|
504
|
+
* never stop the loop.
|
|
505
|
+
*/
|
|
506
|
+
declare function watchSessions(opts: ObserverOptions): Promise<void>;
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* Privacy redaction for trace spans before upload.
|
|
510
|
+
*
|
|
511
|
+
* Reuses agent-eval's `redactValue` + `DEFAULT_REDACTION_RULES` (email, IPs,
|
|
512
|
+
* generic secret keys, …) and layers on coding-session-specific rules that the
|
|
513
|
+
* defaults miss — GitHub tokens, cloud keys, JWTs, bearer headers, private-key
|
|
514
|
+
* blocks, and `key=secret` assignments common in shell/tool output. Redaction
|
|
515
|
+
* runs over every span attribute + status message, so what leaves the machine
|
|
516
|
+
* is already scrubbed.
|
|
517
|
+
*/
|
|
518
|
+
|
|
519
|
+
/** Secrets common in coding-agent traces that the substrate defaults don't cover. */
|
|
520
|
+
declare const CODING_REDACTION_RULES: RedactionRule[];
|
|
521
|
+
/** Defaults + coding-session rules — the rule set used for upload. */
|
|
522
|
+
declare const TRACES_REDACTION_RULES: RedactionRule[];
|
|
523
|
+
interface SpanRedaction {
|
|
524
|
+
spans: OtlpSpan[];
|
|
525
|
+
report: RedactionReport;
|
|
526
|
+
}
|
|
527
|
+
/** Redact every span's attributes + status message. Returns scrubbed spans and
|
|
528
|
+
* an aggregate report (count + per-rule breakdown) over the whole set. */
|
|
529
|
+
declare function redactSpans(spans: readonly OtlpSpan[], rules?: RedactionRule[]): SpanRedaction;
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Batch collection seam — `collectSessions` returns redacted spans per session
|
|
533
|
+
* for a time window / harness selection, so a third party can feed batches or
|
|
534
|
+
* clusters of traces into their own system (their own analysts, a vector store,
|
|
535
|
+
* a fine-tune corpus, …) without the upload/dedup machinery.
|
|
536
|
+
*
|
|
537
|
+
* Redaction is ON by default — the safe posture when traces leave the box.
|
|
538
|
+
*/
|
|
539
|
+
|
|
540
|
+
interface CollectOptions {
|
|
541
|
+
/** Harness ids/aliases. Omit (or `all: true`) → every harness. */
|
|
542
|
+
harnesses?: string[];
|
|
543
|
+
all?: boolean;
|
|
544
|
+
/** Collect from these adapters instead of the built-in registry. */
|
|
545
|
+
adapters?: HarnessTraceAdapter[];
|
|
546
|
+
/** Filter by working directory (exact/prefix). */
|
|
547
|
+
cwd?: string;
|
|
548
|
+
/** Only sessions modified at/after this epoch ms. */
|
|
549
|
+
sinceMs?: number;
|
|
550
|
+
/** Cap to the most-recent N sessions per harness. */
|
|
551
|
+
last?: number;
|
|
552
|
+
/** Redact PII/secrets (default true). Set false to get raw spans. */
|
|
553
|
+
redact?: boolean;
|
|
554
|
+
/** Called on a per-adapter locate/parse failure (collection continues). */
|
|
555
|
+
onError?: (error: unknown, ref?: SessionRef) => void;
|
|
556
|
+
}
|
|
557
|
+
interface SessionBatch {
|
|
558
|
+
ref: SessionRef;
|
|
559
|
+
spans: OtlpSpan[];
|
|
560
|
+
/** Present when redaction ran (the default). */
|
|
561
|
+
redaction?: RedactionReport;
|
|
562
|
+
}
|
|
563
|
+
/** Select + parse (+ redact) sessions into per-session span batches. */
|
|
564
|
+
declare function collectSessions(opts?: CollectOptions): Promise<SessionBatch[]>;
|
|
565
|
+
|
|
566
|
+
/**
|
|
567
|
+
* Local upload-dedup state.
|
|
568
|
+
*
|
|
569
|
+
* Persists which sessions have been uploaded (keyed by harness:sessionId →
|
|
570
|
+
* content hash) so re-running `upload` over an overlapping window doesn't
|
|
571
|
+
* re-send unchanged sessions. The hash is over the redacted span structure, so
|
|
572
|
+
* a session that has GROWN since last upload re-uploads (new spans), while an
|
|
573
|
+
* unchanged one is skipped. Pairs with the server-side `idempotency-key` for
|
|
574
|
+
* belt-and-suspenders dedup.
|
|
575
|
+
*/
|
|
576
|
+
|
|
577
|
+
interface UploadRecord {
|
|
578
|
+
hash: string;
|
|
579
|
+
uploadedAt: string;
|
|
580
|
+
harness: string;
|
|
581
|
+
spanCount: number;
|
|
582
|
+
}
|
|
583
|
+
/** Keyed by `${harness}:${sessionId}`. */
|
|
584
|
+
type UploadState = Record<string, UploadRecord>;
|
|
585
|
+
declare function uploadKey(harness: string, sessionId: string): string;
|
|
586
|
+
/** Stable content hash of a session's (redacted) spans — changes iff the
|
|
587
|
+
* session's span set changes, so a grown session re-uploads. */
|
|
588
|
+
declare function sessionHash(spans: readonly OtlpSpan[]): string;
|
|
589
|
+
declare function loadState(): Promise<UploadState>;
|
|
590
|
+
declare function saveState(state: UploadState): Promise<void>;
|
|
591
|
+
/** True when this session+hash has already been uploaded. */
|
|
592
|
+
declare function alreadyUploaded(state: UploadState, harness: string, sessionId: string, hash: string): boolean;
|
|
593
|
+
|
|
594
|
+
/**
|
|
595
|
+
* Upload local coding-session traces to the Tangle Intelligence Platform.
|
|
596
|
+
*
|
|
597
|
+
* Pipeline per session: locate (time window) → parse to OTLP spans → REDACT
|
|
598
|
+
* (PII/secrets) → augment with metadata → dedup (skip unchanged, already-sent
|
|
599
|
+
* sessions) → POST via the hosted `ingestTraces` client. The redaction happens
|
|
600
|
+
* before anything leaves the machine; the dedup is local-state + server
|
|
601
|
+
* idempotency-key.
|
|
602
|
+
*
|
|
603
|
+
* `planUpload` is read-only (select + redact + dedup); `executeUpload` does the
|
|
604
|
+
* actual send (or, with `dryRun`, writes the redacted OTLP it *would* send).
|
|
605
|
+
*/
|
|
606
|
+
|
|
607
|
+
interface UploadItem {
|
|
608
|
+
ref: SessionRef;
|
|
609
|
+
/** Redacted spans (safe to send). */
|
|
610
|
+
spans: OtlpSpan[];
|
|
611
|
+
redaction: RedactionReport;
|
|
612
|
+
hash: string;
|
|
613
|
+
isNew: boolean;
|
|
614
|
+
}
|
|
615
|
+
interface UploadPlan {
|
|
616
|
+
items: UploadItem[];
|
|
617
|
+
state: UploadState;
|
|
618
|
+
sinceMs?: number;
|
|
619
|
+
}
|
|
620
|
+
interface PlanOptions {
|
|
621
|
+
harness?: string;
|
|
622
|
+
all?: boolean;
|
|
623
|
+
cwd?: string;
|
|
624
|
+
sinceMs?: number;
|
|
625
|
+
/** Called on a per-adapter locate/parse failure (planning continues). */
|
|
626
|
+
onError?: (error: unknown, ref?: SessionRef) => void;
|
|
627
|
+
}
|
|
628
|
+
/** Read-only: select sessions in the window, redact, and mark which are new. */
|
|
629
|
+
declare function planUpload(opts: PlanOptions): Promise<UploadPlan>;
|
|
630
|
+
/** Map redacted OtlpSpan[] → the hosted TraceSpanEvent[] wire shape, attaching
|
|
631
|
+
* session metadata to the root span(s). */
|
|
632
|
+
declare function toTraceSpanEvents(spans: readonly OtlpSpan[], rootMeta: Record<string, string | number | boolean>): TraceSpanEvent[];
|
|
633
|
+
interface UploadResult {
|
|
634
|
+
uploadedSessions: number;
|
|
635
|
+
skippedSessions: number;
|
|
636
|
+
acceptedSpans: number;
|
|
637
|
+
redactionCount: number;
|
|
638
|
+
dryRun: boolean;
|
|
639
|
+
otlpPath?: string;
|
|
640
|
+
}
|
|
641
|
+
/** A trace sink. The hosted client satisfies this; pass your own to route
|
|
642
|
+
* redacted+deduped traces into a different system. */
|
|
643
|
+
interface UploadBackend {
|
|
644
|
+
ingestTraces(spans: TraceSpanEvent[], idempotencyKey?: string): Promise<{
|
|
645
|
+
accepted: number;
|
|
646
|
+
}>;
|
|
647
|
+
}
|
|
648
|
+
interface ExecuteOptions {
|
|
649
|
+
dryRun?: boolean;
|
|
650
|
+
/** Where to write the redacted OTLP-JSONL on a dry run. */
|
|
651
|
+
otlpOut?: string;
|
|
652
|
+
/** Custom sink. Defaults to the hosted Tangle Intelligence client from env. */
|
|
653
|
+
backend?: UploadBackend;
|
|
654
|
+
log?: (msg: string) => void;
|
|
655
|
+
}
|
|
656
|
+
/** Send the NEW items (or, on dryRun, write the redacted OTLP that would send). */
|
|
657
|
+
declare function executeUpload(plan: UploadPlan, opts?: ExecuteOptions): Promise<UploadResult>;
|
|
658
|
+
|
|
659
|
+
export { AmpAdapter, type AnalyzeOptions, type AnalyzeResult, CODING_REDACTION_RULES, ClaudeAdapter, CodexAdapter, type CollectOptions, CopilotAdapter, type ExecuteOptions, FactoryAdapter, ForgeAdapter, GeminiAdapter, type HarnessTraceAdapter, type LocateOptions, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpSpan, type OtlpSpanKind, type OtlpStatusCode, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, QwenAdapter, type RuntimeTrace, type SessionBatch, type SessionRef, type SpanInput, type SpanRedaction, TRACES_REDACTION_RULES, type UploadBackend, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, alreadyUploaded, analyzeSpans, collectSessions, executeUpload, knownHarnesses, listAdapters, loadState, planUpload, redactSpans, resolveAdapter, runPipelines, saveState, serializeSpans, sessionHash, span, toRuntimeStore, toTraceSpanEvents, uploadKey, watchSessions, writeOtlpFile };
|