@tangle-network/traces 0.11.0 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -13
- package/dist/{chunk-JCI5YXNT.js → chunk-UY3M7XUF.js} +2255 -709
- package/dist/cli.js +200 -21
- package/dist/index.d.ts +709 -9
- package/dist/index.js +53 -1
- package/docs/replay-verify.md +55 -0
- package/package.json +5 -3
- package/skills/inspect-agent-traces/SKILL.md +31 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TraceValidation, SpanKind, Outcome } from '@tangle-network/agent-trace-contract';
|
|
2
|
+
export { Capability, ConformanceFinding, ContractSpan, SpanLink, TraceValidation } from '@tangle-network/agent-trace-contract';
|
|
3
|
+
import { ToolUseMetrics, InMemoryTraceStore, CounterfactualRunner, CounterfactualContext, TraceEmitter, TraceStore } from '@tangle-network/agent-eval';
|
|
2
4
|
import { StuckLoopReport, FailureClusterReport } from '@tangle-network/agent-eval/pipelines';
|
|
3
5
|
import { TraceAnalystDefinition, TraceAnalysisEngine, AnalystRegistry, AnalystFinding, buildDefaultAnalystRegistry, AnalystRunSummary, AnalystRunResult, Analyst } from '@tangle-network/agent-eval/analyst';
|
|
4
6
|
export { Analyst, AnalystContext, AnalystFinding, AnalystRegistry, buildDefaultAnalystRegistry, makeFinding } from '@tangle-network/agent-eval/analyst';
|
|
@@ -10,6 +12,194 @@ import { SupervisorRunNodeRole, SupervisorRunReader, SupervisorRunSources } from
|
|
|
10
12
|
import { TraceSpanEvent } from '@tangle-network/agent-eval/hosted';
|
|
11
13
|
export { HostedClient, createHostedClient, hostedClientFromEnv } from '@tangle-network/agent-eval/hosted';
|
|
12
14
|
|
|
15
|
+
/**
|
|
16
|
+
* The INBOUND path: read OTLP-JSONL that some other system emitted and hand it
|
|
17
|
+
* to the same analysis pipeline the adapters feed.
|
|
18
|
+
*
|
|
19
|
+
* The adapters (`src/adapters/`) exist because Claude Code, Codex, OpenCode and
|
|
20
|
+
* friends write proprietary session logs we do not control, so somebody has to
|
|
21
|
+
* translate. A system you DO control should not need one: emit
|
|
22
|
+
* `@tangle-network/agent-trace-contract` spans and point `--otlp` at the file.
|
|
23
|
+
*
|
|
24
|
+
* Two rules govern everything here:
|
|
25
|
+
*
|
|
26
|
+
* 1. **Never throw on foreign input.** A tool that crashes on somebody else's
|
|
27
|
+
* trace cannot read "any AI system". A malformed line, a missing timestamp, a
|
|
28
|
+
* link pointing at nothing — each becomes a counted, named issue and the run
|
|
29
|
+
* continues on what is readable.
|
|
30
|
+
* 2. **Validate what the PRODUCER wrote.** {@link readOtlpInput} runs
|
|
31
|
+
* `validateTraceSpans` over the raw rows, before this module normalizes
|
|
32
|
+
* anything, so the report tells the producer what to fix rather than
|
|
33
|
+
* describing our own repairs back to them.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
/** Why one row of an OTLP file did not become an analyzable span AT ALL. */
|
|
37
|
+
type OtlpIngestIssueKind = 'unparseable-line' | 'not-an-object' | 'missing-span-id' | 'unreadable-timestamp' | 'conflicting-span-id';
|
|
38
|
+
interface OtlpIngestIssue {
|
|
39
|
+
readonly kind: OtlpIngestIssueKind;
|
|
40
|
+
readonly file: string;
|
|
41
|
+
/** 1-indexed line within `file`. */
|
|
42
|
+
readonly line: number;
|
|
43
|
+
readonly detail: string;
|
|
44
|
+
}
|
|
45
|
+
/** Which unusable field cost a span one measurement, without costing it its existence. */
|
|
46
|
+
type OtlpWithheldFieldKind = 'negative-duration';
|
|
47
|
+
/**
|
|
48
|
+
* A row that DID become a span, with one unusable field withheld.
|
|
49
|
+
*
|
|
50
|
+
* This is a different event from an {@link OtlpIngestIssue} and the two must
|
|
51
|
+
* never be merged. A span whose `end_time` precedes its `start_time` has an
|
|
52
|
+
* unusable INTERVAL — it does not have an unusable identity, name, tool name or
|
|
53
|
+
* status. Dropping the whole span for it silently deleted a real tool call from
|
|
54
|
+
* every count in the report, which is a worse failure than the wrong duration
|
|
55
|
+
* it was avoiding: a wrong number is visible and a missing row is not.
|
|
56
|
+
*
|
|
57
|
+
* So the span survives, its interval is withheld rather than repaired (no
|
|
58
|
+
* synthesized end time enters any total), and the withholding is reported
|
|
59
|
+
* here. The contract's own `negative-duration` finding independently marks
|
|
60
|
+
* `latency-analysis` unavailable, so every table that prints a duration is
|
|
61
|
+
* already gated.
|
|
62
|
+
*/
|
|
63
|
+
interface OtlpFieldWithheld {
|
|
64
|
+
readonly kind: OtlpWithheldFieldKind;
|
|
65
|
+
/** The span field that could not be used. */
|
|
66
|
+
readonly field: string;
|
|
67
|
+
readonly file: string;
|
|
68
|
+
/** 1-indexed line within `file`. */
|
|
69
|
+
readonly line: number;
|
|
70
|
+
readonly detail: string;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* A span field this package ANALYSES as something other than what the producer
|
|
74
|
+
* wrote.
|
|
75
|
+
*
|
|
76
|
+
* Every one of these is a place the reader has to substitute a value to analyse
|
|
77
|
+
* the row at all — a trace id to group by, an end time to subtract, a status to
|
|
78
|
+
* count, a kind to bucket, a link to follow. Substituting is correct; writing
|
|
79
|
+
* the substitute into the EXPORT is not, because the export is re-validated and
|
|
80
|
+
* the substitute is by construction more conforming than the source. Every
|
|
81
|
+
* `missing-trace-id`, `invalid-timestamp`, `invalid-status`, `unknown-span-kind`
|
|
82
|
+
* and `dangling-link` this package's own re-export ever laundered away was one
|
|
83
|
+
* of these fields, so they are enumerated in ONE place and the export reads that
|
|
84
|
+
* list rather than knowing them one at a time — a sixth substitution cannot be
|
|
85
|
+
* added without the export learning about it.
|
|
86
|
+
*/
|
|
87
|
+
type SubstitutedField = 'trace_id' | 'end_time' | 'status.code' | 'kind' | 'links';
|
|
88
|
+
interface OtlpInputFile {
|
|
89
|
+
readonly path: string;
|
|
90
|
+
readonly rows: number;
|
|
91
|
+
readonly spans: number;
|
|
92
|
+
/** Rows that re-declared a span already read, byte-identically. */
|
|
93
|
+
readonly repeats: number;
|
|
94
|
+
/** Distinct trace ids this file contributed, so a caller can attribute a
|
|
95
|
+
* trace back to the file it came from when a directory was ingested. */
|
|
96
|
+
readonly traceIds: readonly string[];
|
|
97
|
+
}
|
|
98
|
+
/** A `*.jsonl` in the directory that does not hold OTLP spans, and how we know. */
|
|
99
|
+
interface SkippedInputFile {
|
|
100
|
+
readonly path: string;
|
|
101
|
+
/** What the first readable line was instead of a span. */
|
|
102
|
+
readonly reason: string;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* How much of the original source could not be represented as spans, summed
|
|
106
|
+
* across every export hop that led to this file.
|
|
107
|
+
*
|
|
108
|
+
* `rows` is the count and `kinds` names the ingest issues behind it. On a file a
|
|
109
|
+
* producer wrote, this is just what THIS ingest dropped. On a file
|
|
110
|
+
* `traces` re-exported, it also carries what earlier hops dropped, read back off
|
|
111
|
+
* the spans — so a re-export can never present itself as a complete copy of a
|
|
112
|
+
* source it is missing rows from.
|
|
113
|
+
*/
|
|
114
|
+
interface UnreadableSourceRows {
|
|
115
|
+
readonly rows: number;
|
|
116
|
+
readonly kinds: readonly OtlpIngestIssueKind[];
|
|
117
|
+
/** Rows dropped by EARLIER hops, already absent from the file just read. */
|
|
118
|
+
readonly inherited: number;
|
|
119
|
+
}
|
|
120
|
+
interface OtlpInput {
|
|
121
|
+
/** Every file read, in ingest order. */
|
|
122
|
+
readonly files: readonly OtlpInputFile[];
|
|
123
|
+
/** JSONL files in the directory that hold something other than OTLP spans. */
|
|
124
|
+
readonly skipped: readonly SkippedInputFile[];
|
|
125
|
+
/** Rows exactly as the producer wrote them — the input to `validateTraceSpans`. */
|
|
126
|
+
readonly rows: readonly unknown[];
|
|
127
|
+
/** Rows normalized into this package's span model, ready for the pipeline. */
|
|
128
|
+
readonly spans: readonly OtlpSpan[];
|
|
129
|
+
/** Conformance of the RAW rows: what this trace can and cannot answer. */
|
|
130
|
+
readonly validation: TraceValidation;
|
|
131
|
+
/** Rows that could not become spans, each with its reason. */
|
|
132
|
+
readonly issues: readonly OtlpIngestIssue[];
|
|
133
|
+
/** Rows that DID become spans, with one unusable field withheld from them. */
|
|
134
|
+
readonly withheld: readonly OtlpFieldWithheld[];
|
|
135
|
+
/** Source rows absent from `spans`, this hop's and every earlier hop's. */
|
|
136
|
+
readonly unreadable: UnreadableSourceRows;
|
|
137
|
+
}
|
|
138
|
+
/** A file or directory resolved to the OTLP files in it, plus the JSONL that is not OTLP. */
|
|
139
|
+
interface ResolvedOtlpInput {
|
|
140
|
+
readonly files: readonly string[];
|
|
141
|
+
readonly skipped: readonly SkippedInputFile[];
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Resolve a file or directory to the OTLP files it names.
|
|
145
|
+
*
|
|
146
|
+
* A named FILE is read as OTLP whatever it contains — the caller pointed at it,
|
|
147
|
+
* and reporting its rows as unreadable is more useful than refusing it. A
|
|
148
|
+
* DIRECTORY is a guess about which files were meant, so its non-OTLP JSONL is
|
|
149
|
+
* skipped and named rather than ingested.
|
|
150
|
+
*/
|
|
151
|
+
declare function resolveOtlpInputFiles(path: string): Promise<ResolvedOtlpInput>;
|
|
152
|
+
/**
|
|
153
|
+
* Classify a span from its EVIDENCE — name and attributes — ignoring whatever
|
|
154
|
+
* kind it declared. Callers decide whether a declaration wins; this answers the
|
|
155
|
+
* separate question "what does this span look like".
|
|
156
|
+
*
|
|
157
|
+
* The contract's `resolveSpanKind` does the classifying, so a foreign trace
|
|
158
|
+
* cannot land in one bucket here and another there. What sits on top of it are
|
|
159
|
+
* only signals the contract deliberately does not read, and only where it
|
|
160
|
+
* returned `UNKNOWN`:
|
|
161
|
+
*
|
|
162
|
+
* - `span.type`, a Tangle-internal marker. Its tool-telemetry values name
|
|
163
|
+
* children that DESCRIBE a tool call; counting them as calls doubles every
|
|
164
|
+
* tool total, which is why that check runs first.
|
|
165
|
+
* - the local tool spellings above, and a name that mentions a tool.
|
|
166
|
+
* - an interaction/agent name, which is the run container this model calls
|
|
167
|
+
* AGENT.
|
|
168
|
+
*
|
|
169
|
+
* Anything still unrecognised stays `UNKNOWN`. It is not `CHAIN`: CHAIN claims
|
|
170
|
+
* the span CONTAINS other work, and inventing that relationship is how an
|
|
171
|
+
* unclassifiable span ends up misreported as a container.
|
|
172
|
+
*/
|
|
173
|
+
declare function inferSpanKind(name: string, attributes: Record<string, unknown>): OtlpSpanKind;
|
|
174
|
+
/**
|
|
175
|
+
* Exactly one of `span` or `issue`: a usable span, or the named reason there
|
|
176
|
+
* isn't one. `withheld` accompanies a SPAN — the row was analysable, one field
|
|
177
|
+
* on it was not.
|
|
178
|
+
*/
|
|
179
|
+
interface OtlpRowConversion {
|
|
180
|
+
span?: OtlpSpan;
|
|
181
|
+
issue?: Omit<OtlpIngestIssue, 'file' | 'line'>;
|
|
182
|
+
withheld?: Omit<OtlpFieldWithheld, 'file' | 'line'>;
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* One raw row to one analyzable span, or one named reason it is not analyzable.
|
|
186
|
+
*
|
|
187
|
+
* `fallbackTraceId` covers a row with no trace_id: `validateTraceSpans` reports
|
|
188
|
+
* that as `missing-trace-id`, and grouping still needs SOME key, so the source
|
|
189
|
+
* file names the anonymous trace instead of each span becoming its own run.
|
|
190
|
+
*/
|
|
191
|
+
declare function otlpRowToSpan(row: unknown, fallbackTraceId: string): OtlpRowConversion;
|
|
192
|
+
/**
|
|
193
|
+
* Read OTLP-JSONL from a file or a directory of them.
|
|
194
|
+
*
|
|
195
|
+
* A directory ingests the OTLP files under it, recursively — every `*.jsonl` /
|
|
196
|
+
* `*.ndjson` that holds spans, or only the contents of an `otlp/` subdirectory
|
|
197
|
+
* when the producer made one — so a results directory of per-shot exports
|
|
198
|
+
* analyses as one body of work while its raw event and stream logs are named
|
|
199
|
+
* and skipped rather than counted as broken spans.
|
|
200
|
+
*/
|
|
201
|
+
declare function readOtlpInput(path: string): Promise<OtlpInput>;
|
|
202
|
+
|
|
13
203
|
/**
|
|
14
204
|
* OTLP-flat-line emitter — the canonical normalized trace shape.
|
|
15
205
|
*
|
|
@@ -21,7 +211,7 @@ export { HostedClient, createHostedClient, hostedClientFromEnv } from '@tangle-n
|
|
|
21
211
|
* (which reads the attribute vocabulary), HALO, and any OpenInference tool.
|
|
22
212
|
*
|
|
23
213
|
* Attribute vocabulary the downstream analysts key off:
|
|
24
|
-
* - `openinference.span.kind` → span kind (
|
|
214
|
+
* - `openinference.span.kind` → span kind ({@link OtlpSpanKind})
|
|
25
215
|
* - `service.name` → harness id (claude-code, codex, …)
|
|
26
216
|
* - `agent.name` → agent / subagent name
|
|
27
217
|
* - `llm.model_name` → model
|
|
@@ -29,8 +219,34 @@ export { HostedClient, createHostedClient, hostedClientFromEnv } from '@tangle-n
|
|
|
29
219
|
* - `tool.name` → tool histogram (monoculture / no-verify)
|
|
30
220
|
* - `step` → run order (falls back to start_time)
|
|
31
221
|
*/
|
|
32
|
-
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* The span-kind vocabulary, which is exactly
|
|
225
|
+
* `@tangle-network/agent-trace-contract`'s — AGENT, CHAIN, LLM, TOOL,
|
|
226
|
+
* EVALUATOR, RETRIEVER, UNKNOWN.
|
|
227
|
+
*
|
|
228
|
+
* It is an alias, not a copy, because the two vocabularies MUST NOT drift: a
|
|
229
|
+
* narrower local union silently re-typed every kind it did not list, so a
|
|
230
|
+
* producer's EVALUATOR verdict span came back out as CHAIN and the re-exported
|
|
231
|
+
* file no longer said what the original did.
|
|
232
|
+
*/
|
|
233
|
+
type OtlpSpanKind = SpanKind;
|
|
33
234
|
type OtlpStatusCode = 'OK' | 'ERROR' | 'UNSET';
|
|
235
|
+
/**
|
|
236
|
+
* A causal edge that is NOT containment. `parent_span_id` says "this span
|
|
237
|
+
* happened INSIDE that one"; a link says "that one CAUSED this one" — round N's
|
|
238
|
+
* verdict steering round N+1, a retry pointing at the attempt it replaces.
|
|
239
|
+
* Encoding those as a parent would claim a nesting that never existed, so OTLP
|
|
240
|
+
* gives them their own edge and this model keeps it.
|
|
241
|
+
*
|
|
242
|
+
* Matches `SpanLink` in `@tangle-network/agent-trace-contract`.
|
|
243
|
+
*/
|
|
244
|
+
interface OtlpSpanLink {
|
|
245
|
+
trace_id: string;
|
|
246
|
+
span_id: string;
|
|
247
|
+
/** e.g. `{ 'agent.link.kind': 'steered_by' }`. */
|
|
248
|
+
attributes?: Record<string, unknown>;
|
|
249
|
+
}
|
|
34
250
|
interface OtlpSpan {
|
|
35
251
|
trace_id: string;
|
|
36
252
|
span_id: string;
|
|
@@ -43,6 +259,8 @@ interface OtlpSpan {
|
|
|
43
259
|
message?: string;
|
|
44
260
|
};
|
|
45
261
|
attributes: Record<string, unknown>;
|
|
262
|
+
/** Causal edges to other spans. Absent when the producer recorded none. */
|
|
263
|
+
links?: OtlpSpanLink[];
|
|
46
264
|
}
|
|
47
265
|
interface SpanInput {
|
|
48
266
|
traceId: string;
|
|
@@ -70,6 +288,8 @@ interface SpanInput {
|
|
|
70
288
|
step?: number;
|
|
71
289
|
/** Verbatim content the agentic analysts read via regex search. */
|
|
72
290
|
content?: string | null;
|
|
291
|
+
/** Causal edges (steered-by / graded-by / retry-of), not containment. */
|
|
292
|
+
links?: readonly OtlpSpanLink[];
|
|
73
293
|
/** Extra attributes merged last (escape hatch for harness-specific signal). */
|
|
74
294
|
extra?: Record<string, unknown>;
|
|
75
295
|
}
|
|
@@ -79,11 +299,43 @@ interface SpanInput {
|
|
|
79
299
|
* so the behavioral analyst's "no LLM data" case stays honest.
|
|
80
300
|
*/
|
|
81
301
|
declare function span(input: SpanInput): OtlpSpan;
|
|
82
|
-
/**
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
*
|
|
86
|
-
*
|
|
302
|
+
/** The three things an export can write for a substituted field, and nothing else. */
|
|
303
|
+
type ExportRule = 'faithful' | 'declared-or-analysed' | 'analysed';
|
|
304
|
+
/**
|
|
305
|
+
* What the export writes for a field this package substituted. One rule per
|
|
306
|
+
* field, in one table, because the difference between them is not a style choice
|
|
307
|
+
* — each `analysed` entry is a consumer that HARD-REQUIRES the field, named.
|
|
308
|
+
*
|
|
309
|
+
* The table is EXPORTED because it is a guarantee, not documentation:
|
|
310
|
+
* `tests/otlp-export-rule.test.ts` reads it and derives, per field, the checks
|
|
311
|
+
* the declared word promises — the exact value written in each of the two
|
|
312
|
+
* producer states, that a `faithful` field erases no finding its source earned,
|
|
313
|
+
* and that the artifact stays readable by the consumer whose hard requirement is
|
|
314
|
+
* the only reason `analysed` exists. A field could be declared `faithful` here
|
|
315
|
+
* while nothing implemented it (`status.code` was, and the finding vanished from
|
|
316
|
+
* every re-export); a table that reads as a guarantee and is not one is worse
|
|
317
|
+
* than no table.
|
|
318
|
+
*
|
|
319
|
+
* - `faithful` — the producer's value, or NO field at all when they wrote none.
|
|
320
|
+
* This is the default and the only rule that reproduces the source's finding
|
|
321
|
+
* exactly.
|
|
322
|
+
* - `declared-or-analysed` — the producer's word when they wrote one, else this
|
|
323
|
+
* package's. A kind is what every consumer buckets by, and a span with none is
|
|
324
|
+
* re-derived differently by each of them (the tool-execution demotion that
|
|
325
|
+
* stops a tool call being counted twice is one such re-derivation), so the
|
|
326
|
+
* export always declares one. Writing the producer's word when there is one is
|
|
327
|
+
* what keeps `unknown-span-kind` alive across the trip.
|
|
328
|
+
* - `analysed` — this package's value regardless. `trace_id` is the one field
|
|
329
|
+
* here, and it is not a choice: `analyze --otlp-out` hands the artifact
|
|
330
|
+
* straight to `agent-eval`'s `OtlpFileTraceStore`, whose `projectOtlpFlatLine`
|
|
331
|
+
* drops any row with a falsy `trace_id` and then throws `TraceFileMalformed`
|
|
332
|
+
* on the drop. A faithful (absent) trace id makes the artifact unreadable by
|
|
333
|
+
* the tool that reads it, so `missing-trace-id` CANNOT survive the trip. The
|
|
334
|
+
* substitution is disclosed instead — the span carries the pairing, and
|
|
335
|
+
* `renderConformance` states it above the findings so a reader of the artifact
|
|
336
|
+
* is told the id was minted here rather than left to assume the source had one.
|
|
337
|
+
*/
|
|
338
|
+
declare const EXPORT_RULE: Readonly<Record<SubstitutedField, ExportRule>>;
|
|
87
339
|
declare function toOpenInferenceSpan(s: OtlpSpan): Record<string, unknown>;
|
|
88
340
|
/** Serialize spans to OpenInference JSONL (one complete span per line). This is a
|
|
89
341
|
* standard OpenInference representation: it feeds our analysts, HALO, and other
|
|
@@ -209,6 +461,226 @@ interface HarnessTraceAdapter {
|
|
|
209
461
|
resolveParentTask?(ref: SessionRef, childSessionId: string, options?: Pick<ParseOptions, 'corruptionMode' | 'signal'>): Promise<ParentTaskResolution>;
|
|
210
462
|
}
|
|
211
463
|
|
|
464
|
+
/**
|
|
465
|
+
* What a trace can and cannot answer, said out loud.
|
|
466
|
+
*
|
|
467
|
+
* `@tangle-network/agent-trace-contract` decides conformance: it reports
|
|
468
|
+
* findings (what is wrong, and the consequence) and capabilities (what a
|
|
469
|
+
* consumer can compute, and why not when it cannot). That package is
|
|
470
|
+
* deliberately generic. This module adds the one thing it cannot know — which
|
|
471
|
+
* section of THIS report each missing capability empties — so a thin report
|
|
472
|
+
* explains its own thinness instead of reading as "the agent did nothing".
|
|
473
|
+
*/
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Capabilities the trace cannot support, mapped to the trace's own reason.
|
|
477
|
+
*
|
|
478
|
+
* This is what makes the conformance section GATE rather than narrate: the
|
|
479
|
+
* renderer takes it and marks every section whose inputs are incomplete, at the
|
|
480
|
+
* table, so a number and the reason it is wrong cannot end up forty lines
|
|
481
|
+
* apart.
|
|
482
|
+
*/
|
|
483
|
+
type UnavailableCapabilities = ReadonlyMap<string, string>;
|
|
484
|
+
/** The trace's unavailable capabilities, each mapped to the validator's own reason. */
|
|
485
|
+
declare function unavailableCapabilities(validation: TraceValidation): UnavailableCapabilities;
|
|
486
|
+
/**
|
|
487
|
+
* The inline marker a section carries when its inputs are incomplete. Empty
|
|
488
|
+
* when the capability is available — the caller concatenates unconditionally,
|
|
489
|
+
* so a section can never be marked in one place and silently not in another.
|
|
490
|
+
*/
|
|
491
|
+
declare function incompleteInputsNote(unavailable: UnavailableCapabilities | undefined, capability: string): string;
|
|
492
|
+
interface SpanStructure {
|
|
493
|
+
readonly spans: number;
|
|
494
|
+
readonly traces: number;
|
|
495
|
+
/** Spans carrying no `parent_span_id` at all. */
|
|
496
|
+
readonly rootless: number;
|
|
497
|
+
/**
|
|
498
|
+
* Rootless spans BEYOND the one root each trace is supposed to have. This,
|
|
499
|
+
* not `rootless`, is the number of spans that attribute to nothing: a
|
|
500
|
+
* textbook-correct trace has exactly one root, so `rootless === traces` with
|
|
501
|
+
* one root each is a conforming shape and must read as one.
|
|
502
|
+
*/
|
|
503
|
+
readonly extraRoots: number;
|
|
504
|
+
/** Traces where every span names a parent, so the trace has no top at all. */
|
|
505
|
+
readonly tracesWithoutRoot: number;
|
|
506
|
+
/** Spans naming a parent that is absent from the export. */
|
|
507
|
+
readonly orphans: number;
|
|
508
|
+
readonly links: number;
|
|
509
|
+
}
|
|
510
|
+
/**
|
|
511
|
+
* How much of the span tree actually attaches.
|
|
512
|
+
*
|
|
513
|
+
* The contract's `flat-hierarchy` finding is per-trace and all-or-nothing: one
|
|
514
|
+
* parented span anywhere in a trace suppresses it for every other span. A trace
|
|
515
|
+
* where the overwhelming majority of spans have no parent is not flat by that
|
|
516
|
+
* definition and is entirely flat in practice, so this reports the DEGREE the
|
|
517
|
+
* binary finding cannot express. It measures, it does not re-implement — the
|
|
518
|
+
* finding still comes from the validator.
|
|
519
|
+
*
|
|
520
|
+
* Roots are counted PER TRACE, because "has no parent" is the definition of a
|
|
521
|
+
* root, not of a defect. Counting every rootless span as unattributed made a
|
|
522
|
+
* correct one-root trace narrate itself as broken.
|
|
523
|
+
*/
|
|
524
|
+
declare function summarizeSpanStructure(spans: readonly OtlpSpan[]): SpanStructure;
|
|
525
|
+
interface SkippedAnalysis {
|
|
526
|
+
readonly capability: string;
|
|
527
|
+
/** The analysis in this report that the missing capability empties. */
|
|
528
|
+
readonly analysis: string;
|
|
529
|
+
/** The trace's own reason, from the contract validator. */
|
|
530
|
+
readonly reason: string;
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Conformance of spans this package produced (adapters, importers), measured on
|
|
534
|
+
* the same OpenInference projection that gets written to disk — so the report
|
|
535
|
+
* grades the artifact a consumer would actually receive, not an in-memory shape
|
|
536
|
+
* nobody sees. For spans read from a foreign OTLP file, validate the RAW rows
|
|
537
|
+
* instead (`readOtlpInput` does): that tells the producer what to fix rather
|
|
538
|
+
* than describing our own normalization back to them.
|
|
539
|
+
*/
|
|
540
|
+
declare function conformanceOfSpans(spans: readonly OtlpSpan[]): TraceValidation;
|
|
541
|
+
/** Unavailable capabilities, paired with the analysis each one empties. */
|
|
542
|
+
declare function skippedAnalyses(validation: TraceValidation): SkippedAnalysis[];
|
|
543
|
+
interface ConformanceRenderOptions {
|
|
544
|
+
/** What was validated, e.g. a file path or "spans emitted by the claude-code adapter". */
|
|
545
|
+
readonly subject: string;
|
|
546
|
+
/** Files read, when the source was an OTLP file or directory. */
|
|
547
|
+
readonly files?: readonly OtlpInputFile[];
|
|
548
|
+
/** JSONL files in the directory that hold something other than OTLP spans. */
|
|
549
|
+
readonly skipped?: readonly SkippedInputFile[];
|
|
550
|
+
/** Rows that never became spans, when the source was an OTLP file or directory. */
|
|
551
|
+
readonly issues?: readonly OtlpIngestIssue[];
|
|
552
|
+
/** Rows that DID become spans, with one field withheld from them. */
|
|
553
|
+
readonly withheld?: readonly OtlpFieldWithheld[];
|
|
554
|
+
/** Source rows absent from this file — this ingest's, and any earlier export's. */
|
|
555
|
+
readonly unreadable?: UnreadableSourceRows;
|
|
556
|
+
/** The spans that were analysed, for the hierarchy summary. */
|
|
557
|
+
readonly spans?: readonly OtlpSpan[];
|
|
558
|
+
/** Markdown heading level for the section title (default 2). */
|
|
559
|
+
readonly headingLevel?: number;
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* The conformance section of an analysis report: findings, the capability
|
|
563
|
+
* table, and — the part that makes a thin report honest — the named analyses
|
|
564
|
+
* that produced nothing and why.
|
|
565
|
+
*/
|
|
566
|
+
declare function renderConformance(validation: TraceValidation, opts: ConformanceRenderOptions): string;
|
|
567
|
+
/**
|
|
568
|
+
* The `traces validate` report. Same body, plus a verdict line a reader (or a
|
|
569
|
+
* CI job reading the exit code) can act on.
|
|
570
|
+
*
|
|
571
|
+
* The three verdicts are the three answers `@tangle-network/agent-trace-contract`
|
|
572
|
+
* can give, in its own terms — `ok` says whether the input is a trace AT ALL, and
|
|
573
|
+
* `capabilities` says what a trace can answer. Nothing here re-derives either.
|
|
574
|
+
*/
|
|
575
|
+
declare function renderValidation(validation: TraceValidation, opts: ConformanceRenderOptions): string;
|
|
576
|
+
/**
|
|
577
|
+
* Exit code for `traces validate`, which is read by CI.
|
|
578
|
+
*
|
|
579
|
+
* Non-zero on exactly one condition: the input is not a trace. That is what
|
|
580
|
+
* `TraceValidation.ok` means and the contract's severities are defined to match
|
|
581
|
+
* it — `error` is "nothing readable", `warn` is "readable but degraded", `info`
|
|
582
|
+
* is "nothing lost" — so a severity scan here would only be a second, driftable
|
|
583
|
+
* copy of the same rule.
|
|
584
|
+
*
|
|
585
|
+
* A degraded trace exits 0 deliberately. Failing on degradation is failing on
|
|
586
|
+
* thinness, and every real-world trace is thin somewhere: a gate that red-lights
|
|
587
|
+
* `flat-hierarchy` or `no-usage` is one a producer learns to ignore. What a
|
|
588
|
+
* missing field costs is stated per capability, in the table, where it can be
|
|
589
|
+
* acted on.
|
|
590
|
+
*/
|
|
591
|
+
declare function validationExitCode(validation: TraceValidation): number;
|
|
592
|
+
|
|
593
|
+
/**
|
|
594
|
+
* The two questions a LOOP trace exists to answer, and that a flat event list
|
|
595
|
+
* cannot: **did round N+1 improve on round N**, and **which verdict caused
|
|
596
|
+
* which retry**.
|
|
597
|
+
*
|
|
598
|
+
* Both are read from the contract's own vocabulary and nothing else:
|
|
599
|
+
*
|
|
600
|
+
* - `agent.loop.id` / `agent.loop.iteration` group and order the rounds.
|
|
601
|
+
* - `agent.outcome` / `agent.outcome.score` carry the verdict. A round rarely
|
|
602
|
+
* grades itself — the grade lives on an EVALUATOR span underneath it — so the
|
|
603
|
+
* verdict is taken from the shallowest descendant that has one, which is the
|
|
604
|
+
* round's own verdict rather than one of its per-layer sub-verdicts.
|
|
605
|
+
* - `links` carry causality. `parent_span_id` says round N+1 happened INSIDE
|
|
606
|
+
* round N, which is false; the link says round N's verdict CAUSED it, which
|
|
607
|
+
* is the edge a steering chain is made of.
|
|
608
|
+
*
|
|
609
|
+
* Nothing here throws on malformed input, and nothing is inferred where the
|
|
610
|
+
* trace is silent: a loop with no scores reports `unscored`, not a flat trend.
|
|
611
|
+
*/
|
|
612
|
+
|
|
613
|
+
/** Whether the rounds got better, stayed put, or got worse — and on what basis. */
|
|
614
|
+
type LoopTrend = 'improved' | 'plateaued' | 'regressed' | 'mixed' | 'unscored';
|
|
615
|
+
/** What the trend was computed from, so a reader can tell a score series from
|
|
616
|
+
* a pass/fail series from nothing at all. */
|
|
617
|
+
type LoopTrendBasis = 'score' | 'outcome' | 'none';
|
|
618
|
+
interface LoopIteration {
|
|
619
|
+
readonly iteration: number;
|
|
620
|
+
readonly spanId: string;
|
|
621
|
+
readonly name: string;
|
|
622
|
+
/** Terminal verdict for the round, from the round or its shallowest graded descendant. */
|
|
623
|
+
readonly outcome: Outcome | null;
|
|
624
|
+
readonly score: number | null;
|
|
625
|
+
/** Span the verdict was read from; equals `spanId` when the round graded itself. */
|
|
626
|
+
readonly verdictSpanId: string | null;
|
|
627
|
+
readonly startTime: string;
|
|
628
|
+
readonly durationMs: number | null;
|
|
629
|
+
}
|
|
630
|
+
interface LoopConvergence {
|
|
631
|
+
readonly loopId: string;
|
|
632
|
+
readonly traceId: string;
|
|
633
|
+
readonly iterations: readonly LoopIteration[];
|
|
634
|
+
readonly trend: LoopTrend;
|
|
635
|
+
readonly basis: LoopTrendBasis;
|
|
636
|
+
/** The trend in the trace's own numbers, e.g. "score 0.4 → 0.9 over 4 rounds". */
|
|
637
|
+
readonly detail: string;
|
|
638
|
+
}
|
|
639
|
+
interface LoopConvergenceReport {
|
|
640
|
+
readonly loops: readonly LoopConvergence[];
|
|
641
|
+
/** Spans carrying `agent.loop.iteration` — 0 means the trace records no rounds. */
|
|
642
|
+
readonly iterationSpans: number;
|
|
643
|
+
}
|
|
644
|
+
/** One causal edge: a graded span, and the work it caused. */
|
|
645
|
+
interface SteeringEdge {
|
|
646
|
+
/** The span that CAUSED the work — a verdict, an earlier attempt. */
|
|
647
|
+
readonly causeSpanId: string;
|
|
648
|
+
readonly causeName: string | null;
|
|
649
|
+
readonly causeOutcome: Outcome | null;
|
|
650
|
+
readonly causeScore: number | null;
|
|
651
|
+
/** The span the link was recorded on: the work that was caused. */
|
|
652
|
+
readonly effectSpanId: string;
|
|
653
|
+
readonly effectName: string;
|
|
654
|
+
readonly effectIteration: number | null;
|
|
655
|
+
/** `agent.link.kind`, or `unspecified` when the producer recorded none. */
|
|
656
|
+
readonly linkKind: string;
|
|
657
|
+
/** False when the cause is absent from this export, so the chain breaks here. */
|
|
658
|
+
readonly resolved: boolean;
|
|
659
|
+
}
|
|
660
|
+
interface SteeringChainReport {
|
|
661
|
+
readonly edges: readonly SteeringEdge[];
|
|
662
|
+
readonly links: number;
|
|
663
|
+
readonly dangling: number;
|
|
664
|
+
readonly byKind: Readonly<Record<string, number>>;
|
|
665
|
+
}
|
|
666
|
+
/**
|
|
667
|
+
* Round-over-round convergence, per loop.
|
|
668
|
+
*
|
|
669
|
+
* Rounds are ordered by `agent.loop.iteration`, not by time: a resumed or
|
|
670
|
+
* re-run round is written later than the round it replaces, and time order
|
|
671
|
+
* would report the replacement as a new round.
|
|
672
|
+
*/
|
|
673
|
+
declare function analyzeLoopConvergence(spans: readonly OtlpSpan[]): LoopConvergenceReport;
|
|
674
|
+
/**
|
|
675
|
+
* The steering chain: which graded span caused which subsequent unit of work.
|
|
676
|
+
*
|
|
677
|
+
* A link is recorded on the EFFECT and points at the CAUSE, so the edge is read
|
|
678
|
+
* backwards from the span that carries it. A link whose target is absent from
|
|
679
|
+
* the export is kept and marked unresolved — dropping it would report a shorter
|
|
680
|
+
* chain than the producer recorded, which is the opposite of the truth.
|
|
681
|
+
*/
|
|
682
|
+
declare function analyzeSteeringChain(spans: readonly OtlpSpan[]): SteeringChainReport;
|
|
683
|
+
|
|
212
684
|
interface CodeTraceBenchFileRef {
|
|
213
685
|
readonly path: string;
|
|
214
686
|
readonly line_start: number;
|
|
@@ -1849,6 +2321,24 @@ interface TraceInvestigationOptions {
|
|
|
1849
2321
|
readonly analyzerPrompt?: string;
|
|
1850
2322
|
readonly otlpOutPath?: string;
|
|
1851
2323
|
readonly generatedAt?: string;
|
|
2324
|
+
/**
|
|
2325
|
+
* Conformance of the SOURCE the spans were read from. Supply it when the
|
|
2326
|
+
* source was a foreign OTLP file, so the report grades what the producer
|
|
2327
|
+
* wrote; omitted, it is computed from the spans themselves.
|
|
2328
|
+
*/
|
|
2329
|
+
readonly conformance?: TraceValidation;
|
|
2330
|
+
/** What was validated, named in the conformance section (default: the harness). */
|
|
2331
|
+
readonly conformanceSubject?: string;
|
|
2332
|
+
/** OTLP files read, when the spans came from `--otlp`. */
|
|
2333
|
+
readonly conformanceFiles?: readonly OtlpInputFile[];
|
|
2334
|
+
/** JSONL files that were not OTLP, when the spans came from `--otlp` on a directory. */
|
|
2335
|
+
readonly conformanceSkipped?: readonly SkippedInputFile[];
|
|
2336
|
+
/** Rows that never became spans, when the spans came from `--otlp`. */
|
|
2337
|
+
readonly conformanceIssues?: readonly OtlpIngestIssue[];
|
|
2338
|
+
/** Rows analysed with one field withheld, when the spans came from `--otlp`. */
|
|
2339
|
+
readonly conformanceWithheld?: readonly OtlpFieldWithheld[];
|
|
2340
|
+
/** Source rows absent from the file read, including any earlier export's. */
|
|
2341
|
+
readonly conformanceUnreadable?: UnreadableSourceRows;
|
|
1852
2342
|
readonly signal?: AbortSignal;
|
|
1853
2343
|
readonly log?: (msg: string, fields?: Record<string, unknown>) => void;
|
|
1854
2344
|
}
|
|
@@ -1881,12 +2371,22 @@ interface TraceInvestigationResult {
|
|
|
1881
2371
|
readonly workflow?: SessionWorkflowSummary;
|
|
1882
2372
|
readonly spanCount: number;
|
|
1883
2373
|
readonly otlpPath: string;
|
|
2374
|
+
/**
|
|
2375
|
+
* What these spans can and cannot answer. Every report carries it: a thin
|
|
2376
|
+
* report that does not explain its own thinness reads as "the agent did
|
|
2377
|
+
* nothing" when the truth is "the trace never recorded it".
|
|
2378
|
+
*/
|
|
2379
|
+
readonly conformance: TraceValidation;
|
|
1884
2380
|
readonly execution: ExecutionReport;
|
|
1885
2381
|
readonly analystResult: AnalystRunResult;
|
|
1886
2382
|
readonly findings: readonly AnalystFinding[];
|
|
1887
2383
|
readonly pipelines: PipelineReport;
|
|
1888
2384
|
readonly reactions: ReactionReport;
|
|
1889
2385
|
readonly adoption: AdoptionReport;
|
|
2386
|
+
/** Did round N+1 improve on round N, per loop. */
|
|
2387
|
+
readonly loopConvergence: LoopConvergenceReport;
|
|
2388
|
+
/** Which verdict caused which subsequent round. */
|
|
2389
|
+
readonly steeringChain: SteeringChainReport;
|
|
1890
2390
|
/** Deterministic routing record for the agentic trace-analysis pass. */
|
|
1891
2391
|
readonly agenticRoute?: TraceAgenticRoute;
|
|
1892
2392
|
/**
|
|
@@ -2618,6 +3118,206 @@ interface SessionBatch {
|
|
|
2618
3118
|
/** Select + parse (+ redact) sessions into per-session span batches. */
|
|
2619
3119
|
declare function collectSessions(opts?: CollectOptions): Promise<SessionBatch[]>;
|
|
2620
3120
|
|
|
3121
|
+
/**
|
|
3122
|
+
* Execution-replay verification for CodeTraceBench-style shell trajectories.
|
|
3123
|
+
*
|
|
3124
|
+
* Turns a cited analyst finding ("step k is the error-critical step") into an
|
|
3125
|
+
* executed proof: replay steps 1..k-1 inside a real sandbox on the
|
|
3126
|
+
* trajectory's own docker image, then
|
|
3127
|
+
* arm A — run the recorded step k and check the recorded failure signature
|
|
3128
|
+
* reproduces (returncode + a stable output substring), and
|
|
3129
|
+
* arm B — run a corrected step k and check the failure vanishes.
|
|
3130
|
+
*
|
|
3131
|
+
* Built on the agent-eval counterfactual scaffold: the trajectory is ingested
|
|
3132
|
+
* into a TraceStore, each arm is a `runCounterfactual` meta-run, and the
|
|
3133
|
+
* sandbox-backed `CounterfactualRunner` here supplies the `executeFrom`
|
|
3134
|
+
* callback that scaffold deliberately leaves to consumers.
|
|
3135
|
+
*
|
|
3136
|
+
* Honest limits:
|
|
3137
|
+
* - Only SWE-style trajectories carry `docker_config.base_image`; tasks whose
|
|
3138
|
+
* environment needs external compose peers cannot be replayed this way.
|
|
3139
|
+
* - Copy-on-write forks (`box.branch()`) are firecracker-only; on the docker
|
|
3140
|
+
* driver each arm replays the prefix in its own fresh sandbox instead.
|
|
3141
|
+
* - Prefix divergence (recorded vs replayed returncode) is recorded per step
|
|
3142
|
+
* and surfaced in the verdict, never hidden: a high divergence count is a
|
|
3143
|
+
* finding about replayability, not an error of the harness.
|
|
3144
|
+
*/
|
|
3145
|
+
|
|
3146
|
+
/** Recorded returncode from a mini-SWE observation, or null when absent
|
|
3147
|
+
* (terminal submit steps carry no returncode tag). */
|
|
3148
|
+
declare function parseRecordedReturncode(observation: string | null): number | null;
|
|
3149
|
+
/** Text between the observation's <output> tags, or the raw observation when
|
|
3150
|
+
* the tags are absent. */
|
|
3151
|
+
declare function parseObservationOutput(observation: string | null): string;
|
|
3152
|
+
/**
|
|
3153
|
+
* Stable failure-signature candidate: the first line of the recorded output
|
|
3154
|
+
* that contains the word "error". Null when no such line exists — the verdict
|
|
3155
|
+
* then falls back to returncode-only matching and says so.
|
|
3156
|
+
* Pass an explicit signature to override (compiler quote glyphs vary with
|
|
3157
|
+
* locale, so a hand-picked ASCII substring is often more robust).
|
|
3158
|
+
*/
|
|
3159
|
+
declare function deriveFailureSignature(observation: string | null): string | null;
|
|
3160
|
+
interface IngestedTrajectory {
|
|
3161
|
+
runId: string;
|
|
3162
|
+
stepCount: number;
|
|
3163
|
+
}
|
|
3164
|
+
/**
|
|
3165
|
+
* Emit one tool span per trajectory step, in order, with a monotonic
|
|
3166
|
+
* injected clock so `buildTrajectory` ordering is deterministic even when
|
|
3167
|
+
* two spans would share a Date.now() millisecond.
|
|
3168
|
+
*/
|
|
3169
|
+
declare function ingestCodeTraceBenchSteps(store: TraceStore, steps: readonly CodeTraceBenchStep[], caseId: string): Promise<IngestedTrajectory>;
|
|
3170
|
+
interface ReplayExecResult {
|
|
3171
|
+
exitCode: number;
|
|
3172
|
+
stdout: string;
|
|
3173
|
+
stderr: string;
|
|
3174
|
+
}
|
|
3175
|
+
interface ReplayExecSession {
|
|
3176
|
+
exec(command: string, timeoutMs: number): Promise<ReplayExecResult>;
|
|
3177
|
+
close(): Promise<void>;
|
|
3178
|
+
}
|
|
3179
|
+
interface ReplayExecBackend {
|
|
3180
|
+
open(): Promise<ReplayExecSession>;
|
|
3181
|
+
}
|
|
3182
|
+
interface SandboxReplayBackendOptions {
|
|
3183
|
+
image: string;
|
|
3184
|
+
apiKey: string;
|
|
3185
|
+
baseUrl: string;
|
|
3186
|
+
maxLifetimeSeconds?: number;
|
|
3187
|
+
/**
|
|
3188
|
+
* The local SDK-adapter test harness does not translate `agent: false`
|
|
3189
|
+
* into the platform-control-runtime marker the way the production
|
|
3190
|
+
* sandbox API does, so images without node need the marker passed as
|
|
3191
|
+
* container env. Harmless against the production API path. Default true.
|
|
3192
|
+
*/
|
|
3193
|
+
platformRuntimeEnvMarker?: boolean;
|
|
3194
|
+
}
|
|
3195
|
+
/** One fresh sandbox per open(); close() deletes it. */
|
|
3196
|
+
declare function sandboxReplayBackend(options: SandboxReplayBackendOptions): ReplayExecBackend;
|
|
3197
|
+
/**
|
|
3198
|
+
* mini-SWE runs every action as a fresh /bin/sh subshell from a fixed
|
|
3199
|
+
* workdir. Reproduce that exactly — and stay quote-proof for arbitrary
|
|
3200
|
+
* recorded actions — by piping the base64 of the action into `sh` after
|
|
3201
|
+
* cd-ing to the workdir. Exit code is sh's, i.e. the action's.
|
|
3202
|
+
*/
|
|
3203
|
+
declare function wrapActionForExec(action: string, cwd: string): string;
|
|
3204
|
+
interface PrefixDivergence {
|
|
3205
|
+
step: number;
|
|
3206
|
+
expectedReturncode: number | null;
|
|
3207
|
+
actualExit: number;
|
|
3208
|
+
}
|
|
3209
|
+
interface PrefixReplayResult {
|
|
3210
|
+
prefixExecuted: number;
|
|
3211
|
+
prefixDivergences: PrefixDivergence[];
|
|
3212
|
+
wallMs: number;
|
|
3213
|
+
}
|
|
3214
|
+
interface ArmExecutionResult {
|
|
3215
|
+
command: string;
|
|
3216
|
+
exitCode: number;
|
|
3217
|
+
stdout: string;
|
|
3218
|
+
stderr: string;
|
|
3219
|
+
wallMs: number;
|
|
3220
|
+
}
|
|
3221
|
+
interface SandboxCounterfactualRunnerOptions {
|
|
3222
|
+
cwd: string;
|
|
3223
|
+
stepTimeoutMs: number;
|
|
3224
|
+
/** Execute at most this many prefix steps (from the start). Fast-iteration
|
|
3225
|
+
* knob; a truncated prefix weakens state fidelity and the verdict says so. */
|
|
3226
|
+
prefixLimit?: number;
|
|
3227
|
+
onProgress?: (message: string) => void;
|
|
3228
|
+
}
|
|
3229
|
+
/**
|
|
3230
|
+
* Replays `ctx.prefix` in a fresh sandbox session, then executes the mutated
|
|
3231
|
+
* step. Divergences are recorded and never abort the replay. Results land on
|
|
3232
|
+
* `lastPrefix` / `lastArm` for the caller; spans for every exec land in the
|
|
3233
|
+
* counterfactual meta-run.
|
|
3234
|
+
*/
|
|
3235
|
+
declare class SandboxCounterfactualRunner implements CounterfactualRunner {
|
|
3236
|
+
private readonly backend;
|
|
3237
|
+
private readonly options;
|
|
3238
|
+
lastPrefix: PrefixReplayResult | null;
|
|
3239
|
+
lastArm: ArmExecutionResult | null;
|
|
3240
|
+
constructor(backend: ReplayExecBackend, options: SandboxCounterfactualRunnerOptions);
|
|
3241
|
+
executeFrom(ctx: CounterfactualContext, emitter: TraceEmitter): Promise<void>;
|
|
3242
|
+
}
|
|
3243
|
+
interface ReplayVerifyOptions {
|
|
3244
|
+
stepsPath: string;
|
|
3245
|
+
image: string;
|
|
3246
|
+
/** 1-based step_id of the error-critical step (matches CodeTraceBench). */
|
|
3247
|
+
at: number;
|
|
3248
|
+
/** Corrected command for arm B. Omit to run arm A only. */
|
|
3249
|
+
fixCommand?: string;
|
|
3250
|
+
cwd: string;
|
|
3251
|
+
out: string;
|
|
3252
|
+
caseId?: string;
|
|
3253
|
+
/** Override the auto-derived failure signature substring. */
|
|
3254
|
+
signature?: string;
|
|
3255
|
+
stepTimeoutMs?: number;
|
|
3256
|
+
prefixLimit?: number;
|
|
3257
|
+
/** Label of the orchestrator driver backing the sandbox (reported, not probed). */
|
|
3258
|
+
driverLabel?: string;
|
|
3259
|
+
/** Injectable for tests; defaults to sandboxReplayBackend(). */
|
|
3260
|
+
backend?: ReplayExecBackend;
|
|
3261
|
+
apiKey?: string;
|
|
3262
|
+
baseUrl?: string;
|
|
3263
|
+
maxLifetimeSeconds?: number;
|
|
3264
|
+
onProgress?: (message: string) => void;
|
|
3265
|
+
}
|
|
3266
|
+
interface ReplayArmVerdict {
|
|
3267
|
+
command: string;
|
|
3268
|
+
exitCode: number;
|
|
3269
|
+
wallMs: number;
|
|
3270
|
+
prefix: PrefixReplayResult;
|
|
3271
|
+
}
|
|
3272
|
+
interface ReplayVerdict {
|
|
3273
|
+
case: string;
|
|
3274
|
+
image: string;
|
|
3275
|
+
driver: string;
|
|
3276
|
+
k: number;
|
|
3277
|
+
cwd: string;
|
|
3278
|
+
recordedReturncode: number | null;
|
|
3279
|
+
signature: string | null;
|
|
3280
|
+
signatureBasis: 'returncode+output-substring' | 'returncode-only';
|
|
3281
|
+
prefixExecuted: number;
|
|
3282
|
+
prefixDivergences: PrefixDivergence[];
|
|
3283
|
+
armA: ReplayArmVerdict & {
|
|
3284
|
+
failureSignatureMatch: boolean;
|
|
3285
|
+
};
|
|
3286
|
+
armB: (ReplayArmVerdict & {
|
|
3287
|
+
failureVanished: boolean;
|
|
3288
|
+
}) | null;
|
|
3289
|
+
timings: {
|
|
3290
|
+
armAMs: number;
|
|
3291
|
+
armBMs: number | null;
|
|
3292
|
+
totalMs: number;
|
|
3293
|
+
};
|
|
3294
|
+
runIds: {
|
|
3295
|
+
original: string;
|
|
3296
|
+
armA: string;
|
|
3297
|
+
armB: string | null;
|
|
3298
|
+
};
|
|
3299
|
+
}
|
|
3300
|
+
declare function replayVerify(options: ReplayVerifyOptions): Promise<ReplayVerdict>;
|
|
3301
|
+
interface ReplayVerifyCliArgs {
|
|
3302
|
+
stepsPath: string;
|
|
3303
|
+
image: string;
|
|
3304
|
+
at: number;
|
|
3305
|
+
fixCommand?: string;
|
|
3306
|
+
cwd: string;
|
|
3307
|
+
out: string;
|
|
3308
|
+
caseId?: string;
|
|
3309
|
+
signature?: string;
|
|
3310
|
+
stepTimeoutMs?: number;
|
|
3311
|
+
prefixLimit?: number;
|
|
3312
|
+
driverLabel?: string;
|
|
3313
|
+
baseUrl: string;
|
|
3314
|
+
apiKeyEnv: string;
|
|
3315
|
+
maxLifetimeSeconds?: number;
|
|
3316
|
+
}
|
|
3317
|
+
declare function parseReplayVerifyArgs(argv: readonly string[]): ReplayVerifyCliArgs | 'help';
|
|
3318
|
+
declare function replayVerifyUsage(): string;
|
|
3319
|
+
declare function cmdReplayVerify(argv: readonly string[]): Promise<void>;
|
|
3320
|
+
|
|
2621
3321
|
/**
|
|
2622
3322
|
* Local upload-dedup state.
|
|
2623
3323
|
*
|
|
@@ -2720,4 +3420,4 @@ interface ExecuteOptions {
|
|
|
2720
3420
|
/** Send the final, new outbound items (or write their dry-run OTLP). */
|
|
2721
3421
|
declare function executeUpload(plan: UploadPlan, opts?: ExecuteOptions): Promise<UploadResult>;
|
|
2722
3422
|
|
|
2723
|
-
export { ACTOR_ATTR, ATTR, type Actor, type AdoptionOptions, type AdoptionReport, AmpAdapter, type AnalyzeOptions, type AnalyzeResult, BLOBS_DIR, type BuildPolicyEvidenceOptions, type BuildSessionIndexOptions, type BuildTraceFindingPacketOptions, CODING_REDACTION_RULES, CONTENT_CAP, COORDINATION_LOG_FILE, CORRECTIVE_REACTIONS, type ChatTrajectory, type ChatTrajectoryMessage, type ChatTrajectoryOptions, ClaudeAdapter, type CodeTraceBenchFileRef, type CodeTraceBenchImportOptions, type CodeTraceBenchImportReceipt, type CodeTraceBenchImportResult, type CodeTraceBenchImportedTrace, type CodeTraceBenchStep, CodexAdapter, CodexExecAdapter, CodexExecStreamError, CodexTaskScopeError, type CollectOptions, type CollectPolicyEvidenceOptions, type CollectSessionIndexOptions, type CollectSessionSelectionOptions, type CollectSessionWorkflowOptions, type CoordinationLog, type CoordinationQuestion, type CoordinationSteer, CopilotAdapter, type CorruptionMode, DEFAULT_HARNESS, EMPTY_SPAN_RUN_SNAPSHOT, EmptySessionError, type ExecuteOptions, type ExternalAnalysisKind, type ExternalAnalysisPayload, type ExternalAnalysisResult, type ExternalAnalyzer, type ExternalAnalyzerOptions, type ExternalDiscoveryCandidate, FactoryAdapter, type FileRunContextReaderOptions, ForgeAdapter, GeminiAdapter, type HaloAnalyzerOptions, type HaloReasoningEffort, type HarnessTraceAdapter, type HodoscopeAnalyzerOptions, type HodoscopeInput, type HodoscopeMessage, type HodoscopeSourceMapEntry, type HodoscopeTrajectory, INGEST_SOURCE_CLI, type InspectSessionIndexOptions, JsonSourceError, type JsonlCorruptionReceipt, JsonlParseError, type JsonlReadOptions, type LiveBatchOptions, type LocateOptions, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpSpan, type OtlpSpanKind, type OtlpStatusCode, type ParentTaskResolution, type ParseOptions, PartialUploadError, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, type PolicyEvidenceLoopSummary, type PolicyEvidenceRecord, type PolicyEvidenceToolSummary, QwenAdapter, REACTIONS, RESULT_FILE, RULE, type Reaction, type ReactionReport, type ReactionSessionReport, type Redactor, type RepoAttrs, type RepoResolution, type RepoResolutionSource, type RunContextJournal, type RunContextNode, type RunContextResult, type RunContextSnapshot, type RunResult, type RunWatchSnapshot, type RunWatchTarget, type RuntimeTrace, SPAWN_JOURNAL_FILE, STATUS_MARK, type ScanOptions, type ScannedSession, type SessionBatch, type SessionCorruptionReceipt, type SessionIdentityConflict, type SessionIdentityIndex, type SessionIntegrity, type SessionRef, type SessionRelationship, type SessionRole, type SessionSeedGroup, type SessionSelection, type SessionSelectionRow, type SessionSourceFile, type SessionWorkflow, SessionWorkflowError, type SessionWorkflowErrorCode, type SessionWorkflowIssue, type SessionWorkflowSession, type SessionWorkflowSummary, type SpanInput, type SpanNodeStatus, type SpanRedaction, type SpanRunNode, type SpanRunSnapshot, type SpendRoll, TRACES_REDACTION_RULES, type TraceAgenticRoute, type TraceAgenticRouteReason, type TraceAgenticRouteReasonCode, type TraceContextIndex, type TraceContextRoot, type TraceEvidenceExportOptions, type TraceEvidenceExportResult, type TraceEvidenceFormatOption, type TraceEvidenceInputFormat, type TraceEvidenceRow, type TraceExecutionOptions, type TraceFindingPacket, type TraceImprovementArtifacts, type TraceImprovementOptions, type TraceImprovementResult, type TraceIndexedFile, type TraceInspectionArea, type TraceInspectionFinding, type TraceInspectionReport, type TraceInspectionSeverity, type TraceInvestigationOptions, type TraceInvestigationResult, type TraceLiveAction, type TraceLiveActionKind, type TraceLiveAnalysisContext, type TraceLiveAnalyst, type TraceLiveBatch, type TraceLiveEvidence, type TraceLiveFinding, type TraceLiveLoop, type TraceLiveSeverity, type TraceSessionIndex, type TraceSessionIndexRow, type TraceStoreInvestigationOptions, type TraceStoreInvestigationResult, type TraceStreamEvent, type TraceStreamOptions, type TraceStreamReplayOptions, type TraceStreamSession, type TracesConfig, type TriggerPair, type UploadBackend, type UploadIdentity, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, type UserPromptInput, type WatchRunTargetOptions, ZERO_ROLL, addSpend, alreadyUploaded, analyzeAdoption, analyzeLiveBatch, analyzeReactions, analyzeSpans, applyRedactor, buildPolicyEvidenceRecord, buildSessionIndex, buildSessionIndexFromRows, buildSpanRunTree, buildTraceFindingPacket, capText, chatTrajectoryToSpans, classifyLiveActions, classifyReaction, claudeActor, codexActor, collectPolicyEvidence, collectSessionIndex, collectSessionSelection, collectSessionWorkflow, collectSessions, commandAnalyzer, commandRedactor, connectors, countSkillRunsJsonl, cwdMatchesSelection, defaultTraceLiveAnalysts, describeSessionRelationship, equivalentGitCwds, executeUpload, exportTraceEvidenceFile, exportTraceEvidenceRows, exportTraceEvidenceText, fileRunContextSupervisorRunReader, findSpanSourceFiles, forEachTreeNode, haloAnalyzer, hodoscopeAnalyzer, importCodeTraceBench, indexSessionIdsByTrace, inferRole, inspectSessionIndex, int, isChatTrajectory, isChatTrajectoryMessage, isFileRunContextDir, knownHarnesses, listAdapters, loadState, loadTracesConfig, locateSessions, looksLikeAgentPrompt, mergeRolls, mergeTracesConfig, ms, normalizeRemote, outboundHash, parseIsoToEpochMs, parseSession, parseSince, planTraceAgenticRoute, planUpload, readCoordinationLog, readFileRunContext, readJsonFile, readJsonl, readRunContextJournal, readRunContextSnapshot, readRunWatchSnapshot, readSessionIndexFile, readSpanRunSnapshot, recordSessionCorruption, redactSpans, renderInspectionReport, renderRunContextSnapshot, renderRunWatchSnapshot, renderSpanRunSnapshot, resolveAdapter, resolveRepoAttrs, resolveRunContextDir, resolveRunWatchTarget, resolveSessionRepoAttrs, runCommand, runExternalAnalyzers, runPipelines, runTotals, runTraceImprovement, runTraceInvestigation, runTraceStoreInvestigation, saveReport, saveState, scanSessions, selectAdapters, serializeInspectionReport, serializePolicyEvidence, serializeSessionIndex, serializeSpans, serializeTraceStreamEvent, sessionHash, sessionIdFromAttributes, sessionIntegrityAttributes, span, spanRunTotals, stampRepoAttrs, stampSessionIdentity, stampSessionIntegrity, stampSpanWorkdirRepoAttrs, streamSessions, summarizeOtlpExecution, summarizeSpanExecution, takeJsonl, textIsSynthetic, toOpenInferenceSpan, toRuntimeStore, toTraceSpanEvents, tokens, totalAgenticFailureMessage, traceAgenticKinds, traceStreamEventsFromSpans, uploadKey, usd, userPromptSpan, validateOtlpSpans, watchRunTarget, watchSessions, wrap, writeHodoscopeInput, writeInspectionReportFile, writeOtlpFile, writePolicyEvidenceFile, writeSessionIndexFile, writeTraceEvidenceExportFile, writeTraceImprovementArtifacts, zeroPricedCost };
|
|
3423
|
+
export { ACTOR_ATTR, ATTR, type Actor, type AdoptionOptions, type AdoptionReport, AmpAdapter, type AnalyzeOptions, type AnalyzeResult, type ArmExecutionResult, BLOBS_DIR, type BuildPolicyEvidenceOptions, type BuildSessionIndexOptions, type BuildTraceFindingPacketOptions, CODING_REDACTION_RULES, CONTENT_CAP, COORDINATION_LOG_FILE, CORRECTIVE_REACTIONS, type ChatTrajectory, type ChatTrajectoryMessage, type ChatTrajectoryOptions, ClaudeAdapter, type CodeTraceBenchFileRef, type CodeTraceBenchImportOptions, type CodeTraceBenchImportReceipt, type CodeTraceBenchImportResult, type CodeTraceBenchImportedTrace, type CodeTraceBenchStep, CodexAdapter, CodexExecAdapter, CodexExecStreamError, CodexTaskScopeError, type CollectOptions, type CollectPolicyEvidenceOptions, type CollectSessionIndexOptions, type CollectSessionSelectionOptions, type CollectSessionWorkflowOptions, type ConformanceRenderOptions, type CoordinationLog, type CoordinationQuestion, type CoordinationSteer, CopilotAdapter, type CorruptionMode, DEFAULT_HARNESS, EMPTY_SPAN_RUN_SNAPSHOT, EXPORT_RULE, EmptySessionError, type ExecuteOptions, type ExportRule, type ExternalAnalysisKind, type ExternalAnalysisPayload, type ExternalAnalysisResult, type ExternalAnalyzer, type ExternalAnalyzerOptions, type ExternalDiscoveryCandidate, FactoryAdapter, type FileRunContextReaderOptions, ForgeAdapter, GeminiAdapter, type HaloAnalyzerOptions, type HaloReasoningEffort, type HarnessTraceAdapter, type HodoscopeAnalyzerOptions, type HodoscopeInput, type HodoscopeMessage, type HodoscopeSourceMapEntry, type HodoscopeTrajectory, INGEST_SOURCE_CLI, type IngestedTrajectory, type InspectSessionIndexOptions, JsonSourceError, type JsonlCorruptionReceipt, JsonlParseError, type JsonlReadOptions, type LiveBatchOptions, type LocateOptions, type LoopConvergence, type LoopConvergenceReport, type LoopIteration, type LoopTrend, type LoopTrendBasis, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpIngestIssue, type OtlpIngestIssueKind, type OtlpInput, type OtlpInputFile, type OtlpRowConversion, type OtlpSpan, type OtlpSpanKind, type OtlpSpanLink, type OtlpStatusCode, type ParentTaskResolution, type ParseOptions, PartialUploadError, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, type PolicyEvidenceLoopSummary, type PolicyEvidenceRecord, type PolicyEvidenceToolSummary, type PrefixDivergence, type PrefixReplayResult, QwenAdapter, REACTIONS, RESULT_FILE, RULE, type Reaction, type ReactionReport, type ReactionSessionReport, type Redactor, type ReplayArmVerdict, type ReplayExecBackend, type ReplayExecResult, type ReplayExecSession, type ReplayVerdict, type ReplayVerifyCliArgs, type ReplayVerifyOptions, type RepoAttrs, type RepoResolution, type RepoResolutionSource, type ResolvedOtlpInput, type RunContextJournal, type RunContextNode, type RunContextResult, type RunContextSnapshot, type RunResult, type RunWatchSnapshot, type RunWatchTarget, type RuntimeTrace, SPAWN_JOURNAL_FILE, STATUS_MARK, SandboxCounterfactualRunner, type SandboxCounterfactualRunnerOptions, type SandboxReplayBackendOptions, type ScanOptions, type ScannedSession, type SessionBatch, type SessionCorruptionReceipt, type SessionIdentityConflict, type SessionIdentityIndex, type SessionIntegrity, type SessionRef, type SessionRelationship, type SessionRole, type SessionSeedGroup, type SessionSelection, type SessionSelectionRow, type SessionSourceFile, type SessionWorkflow, SessionWorkflowError, type SessionWorkflowErrorCode, type SessionWorkflowIssue, type SessionWorkflowSession, type SessionWorkflowSummary, type SkippedAnalysis, type SkippedInputFile, type SpanInput, type SpanNodeStatus, type SpanRedaction, type SpanRunNode, type SpanRunSnapshot, type SpanStructure, type SpendRoll, type SteeringChainReport, type SteeringEdge, TRACES_REDACTION_RULES, type TraceAgenticRoute, type TraceAgenticRouteReason, type TraceAgenticRouteReasonCode, type TraceContextIndex, type TraceContextRoot, type TraceEvidenceExportOptions, type TraceEvidenceExportResult, type TraceEvidenceFormatOption, type TraceEvidenceInputFormat, type TraceEvidenceRow, type TraceExecutionOptions, type TraceFindingPacket, type TraceImprovementArtifacts, type TraceImprovementOptions, type TraceImprovementResult, type TraceIndexedFile, type TraceInspectionArea, type TraceInspectionFinding, type TraceInspectionReport, type TraceInspectionSeverity, type TraceInvestigationOptions, type TraceInvestigationResult, type TraceLiveAction, type TraceLiveActionKind, type TraceLiveAnalysisContext, type TraceLiveAnalyst, type TraceLiveBatch, type TraceLiveEvidence, type TraceLiveFinding, type TraceLiveLoop, type TraceLiveSeverity, type TraceSessionIndex, type TraceSessionIndexRow, type TraceStoreInvestigationOptions, type TraceStoreInvestigationResult, type TraceStreamEvent, type TraceStreamOptions, type TraceStreamReplayOptions, type TraceStreamSession, type TracesConfig, type TriggerPair, type UnavailableCapabilities, type UploadBackend, type UploadIdentity, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, type UserPromptInput, type WatchRunTargetOptions, ZERO_ROLL, addSpend, alreadyUploaded, analyzeAdoption, analyzeLiveBatch, analyzeLoopConvergence, analyzeReactions, analyzeSpans, analyzeSteeringChain, applyRedactor, buildPolicyEvidenceRecord, buildSessionIndex, buildSessionIndexFromRows, buildSpanRunTree, buildTraceFindingPacket, capText, chatTrajectoryToSpans, classifyLiveActions, classifyReaction, claudeActor, cmdReplayVerify, codexActor, collectPolicyEvidence, collectSessionIndex, collectSessionSelection, collectSessionWorkflow, collectSessions, commandAnalyzer, commandRedactor, conformanceOfSpans, connectors, countSkillRunsJsonl, cwdMatchesSelection, defaultTraceLiveAnalysts, deriveFailureSignature, describeSessionRelationship, equivalentGitCwds, executeUpload, exportTraceEvidenceFile, exportTraceEvidenceRows, exportTraceEvidenceText, fileRunContextSupervisorRunReader, findSpanSourceFiles, forEachTreeNode, haloAnalyzer, hodoscopeAnalyzer, importCodeTraceBench, incompleteInputsNote, indexSessionIdsByTrace, inferRole, inferSpanKind, ingestCodeTraceBenchSteps, inspectSessionIndex, int, isChatTrajectory, isChatTrajectoryMessage, isFileRunContextDir, knownHarnesses, listAdapters, loadState, loadTracesConfig, locateSessions, looksLikeAgentPrompt, mergeRolls, mergeTracesConfig, ms, normalizeRemote, otlpRowToSpan, outboundHash, parseIsoToEpochMs, parseObservationOutput, parseRecordedReturncode, parseReplayVerifyArgs, parseSession, parseSince, planTraceAgenticRoute, planUpload, readCoordinationLog, readFileRunContext, readJsonFile, readJsonl, readOtlpInput, readRunContextJournal, readRunContextSnapshot, readRunWatchSnapshot, readSessionIndexFile, readSpanRunSnapshot, recordSessionCorruption, redactSpans, renderConformance, renderInspectionReport, renderRunContextSnapshot, renderRunWatchSnapshot, renderSpanRunSnapshot, renderValidation, replayVerify, replayVerifyUsage, resolveAdapter, resolveOtlpInputFiles, resolveRepoAttrs, resolveRunContextDir, resolveRunWatchTarget, resolveSessionRepoAttrs, runCommand, runExternalAnalyzers, runPipelines, runTotals, runTraceImprovement, runTraceInvestigation, runTraceStoreInvestigation, sandboxReplayBackend, saveReport, saveState, scanSessions, selectAdapters, serializeInspectionReport, serializePolicyEvidence, serializeSessionIndex, serializeSpans, serializeTraceStreamEvent, sessionHash, sessionIdFromAttributes, sessionIntegrityAttributes, skippedAnalyses, span, spanRunTotals, stampRepoAttrs, stampSessionIdentity, stampSessionIntegrity, stampSpanWorkdirRepoAttrs, streamSessions, summarizeOtlpExecution, summarizeSpanExecution, summarizeSpanStructure, takeJsonl, textIsSynthetic, toOpenInferenceSpan, toRuntimeStore, toTraceSpanEvents, tokens, totalAgenticFailureMessage, traceAgenticKinds, traceStreamEventsFromSpans, unavailableCapabilities, uploadKey, usd, userPromptSpan, validateOtlpSpans, validationExitCode, watchRunTarget, watchSessions, wrap, wrapActionForExec, writeHodoscopeInput, writeInspectionReportFile, writeOtlpFile, writePolicyEvidenceFile, writeSessionIndexFile, writeTraceEvidenceExportFile, writeTraceImprovementArtifacts, zeroPricedCost };
|