@tangle-network/traces 0.16.2 → 0.16.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -2
- package/dist/{chunk-G5KKIQZ6.js → chunk-SMT635QV.js} +115 -56
- package/dist/cli.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +6 -6
- package/skills/inspect-agent-traces/SKILL.md +33 -31
- package/docs/replay-batch-20260802.md +0 -76
package/README.md
CHANGED
|
@@ -739,6 +739,12 @@ Claude Workflow subagents are selected by the run ID and transcript directory re
|
|
|
739
739
|
Each Workflow child attaches to the latest matching call that started before the child.
|
|
740
740
|
Task selection then keeps only children attached to calls in that task.
|
|
741
741
|
Returned directories from another resumed Claude session are included in parsing and source hashes.
|
|
742
|
+
For a copied Claude session, preserve `<session UUID>/subagents/workflows/<run ID>` beside the parent JSONL file.
|
|
743
|
+
Recorded native `.claude/projects/<project>/<session UUID>/subagents/workflows/<run ID>` paths resolve to that copied subtree when outside the selected store.
|
|
744
|
+
Copy every referenced Workflow directory, including runs stored under another resumed session UUID.
|
|
745
|
+
The original transcript stays unchanged, and child attribution uses its recorded Workflow identity.
|
|
746
|
+
Missing copied directories, conflicting source bindings, and symlinks that change the copied session or run identity stop parsing.
|
|
747
|
+
Copied Workflow directories contain regular transcript and metadata files; symlink entries stop parsing.
|
|
742
748
|
Missing or conflicting Workflow identities stop the parse instead of widening the selected history.
|
|
743
749
|
Cross-harness parent/child trees are not inferred.
|
|
744
750
|
For a reproducible report, run `traces list` first and pass its session ID with `--session`.
|
|
@@ -964,7 +970,7 @@ if (input.validation.ok) await analyzeSpans(input.spans)
|
|
|
964
970
|
|
|
965
971
|
// Observe live sessions, feed findings anywhere (read-only, cancellable):
|
|
966
972
|
const c = new AbortController()
|
|
967
|
-
await watchSessions({ all: true, signal: c.signal, onLoop: (l) => alert(l.toolName
|
|
973
|
+
await watchSessions({ all: true, signal: c.signal, onLoop: (l) => alert(`${l.toolName} x${l.occurrences}`) })
|
|
968
974
|
|
|
969
975
|
// Feed a visualizer or dashboard:
|
|
970
976
|
await streamSessions({ all: true, signal: c.signal, includeSpans: false, onEvent: (event) => console.log(event) })
|
|
@@ -977,7 +983,7 @@ registry.register({
|
|
|
977
983
|
return [makeFinding({ analyst_id: 'mine', area: 'custom', claim: '…', severity: 'info', evidence_refs: [], confidence: 0.9 })]
|
|
978
984
|
},
|
|
979
985
|
})
|
|
980
|
-
await analyzeSpans(spans, { registry })
|
|
986
|
+
await analyzeSpans(input.spans, { registry })
|
|
981
987
|
```
|
|
982
988
|
|
|
983
989
|
## Examples
|
|
@@ -1814,7 +1814,7 @@ var CORRECTIVE_REACTIONS = ["correction", "frustration", "jargon", "structure"];
|
|
|
1814
1814
|
import { createHash as createHash6 } from "crypto";
|
|
1815
1815
|
import { readdir as readdir5, stat as stat3 } from "fs/promises";
|
|
1816
1816
|
import { homedir as homedir3 } from "os";
|
|
1817
|
-
import { basename as
|
|
1817
|
+
import { basename as basename4, join as join5, relative as relative3 } from "path";
|
|
1818
1818
|
import {
|
|
1819
1819
|
applyLlmSpanOtlpAttributes as applyLlmSpanOtlpAttributes2,
|
|
1820
1820
|
LLM_CACHED_TOKENS,
|
|
@@ -1827,7 +1827,7 @@ import { ATTR as ATTR3, deriveHexId as deriveHexId2 } from "@tangle-network/agen
|
|
|
1827
1827
|
// src/adapters/claude-subagent-sources.ts
|
|
1828
1828
|
import { readdir as readdir4, realpath } from "fs/promises";
|
|
1829
1829
|
import { homedir as homedir2 } from "os";
|
|
1830
|
-
import {
|
|
1830
|
+
import { dirname as dirname3, isAbsolute as isAbsolute2, join as join4, relative as relative2, resolve as resolve3 } from "path";
|
|
1831
1831
|
|
|
1832
1832
|
// src/adapters/claude-workflow.ts
|
|
1833
1833
|
import { basename as basename3, dirname as dirname2, isAbsolute, relative, resolve as resolve2 } from "path";
|
|
@@ -1928,7 +1928,7 @@ function workflowRunIdForSubagent(subDir, file) {
|
|
|
1928
1928
|
}
|
|
1929
1929
|
return parts[1];
|
|
1930
1930
|
}
|
|
1931
|
-
function bindWorkflowSubagent(file, runId, transcriptDir, bindings) {
|
|
1931
|
+
function bindWorkflowSubagent(file, runId, transcriptDir, bindings, sourceTranscriptDir = transcriptDir) {
|
|
1932
1932
|
const runBindings = bindings.get(runId);
|
|
1933
1933
|
if (!runBindings || runBindings.length === 0) {
|
|
1934
1934
|
throw new ClaudeTaskScopeError(
|
|
@@ -1942,7 +1942,7 @@ function bindWorkflowSubagent(file, runId, transcriptDir, bindings) {
|
|
|
1942
1942
|
);
|
|
1943
1943
|
}
|
|
1944
1944
|
const localBindings = runBindings.filter(
|
|
1945
|
-
(binding) => resolve2(binding.transcriptDir) ===
|
|
1945
|
+
(binding) => resolve2(binding.transcriptDir) === resolve2(sourceTranscriptDir)
|
|
1946
1946
|
);
|
|
1947
1947
|
if (localBindings.length === 0) {
|
|
1948
1948
|
throw new ClaudeTaskScopeError(
|
|
@@ -1995,7 +1995,7 @@ function selectWorkflowBinding(file, bindings, childStartedAt, explicitToolUseId
|
|
|
1995
1995
|
}
|
|
1996
1996
|
|
|
1997
1997
|
// src/adapters/claude-subagent-sources.ts
|
|
1998
|
-
async function listSubagentFiles(root, signal) {
|
|
1998
|
+
async function listSubagentFiles(root, signal, copiedWorkflow = false) {
|
|
1999
1999
|
const pending = [root];
|
|
2000
2000
|
const files = [];
|
|
2001
2001
|
while (pending.length > 0) {
|
|
@@ -2012,6 +2012,11 @@ async function listSubagentFiles(root, signal) {
|
|
|
2012
2012
|
signal?.throwIfAborted();
|
|
2013
2013
|
for (const entry of entries) {
|
|
2014
2014
|
const path = join4(dir, entry.name);
|
|
2015
|
+
if (copiedWorkflow && entry.isSymbolicLink()) {
|
|
2016
|
+
throw new ClaudeTaskScopeError(
|
|
2017
|
+
`Claude Workflow subagent does not match copied source: ${path}`
|
|
2018
|
+
);
|
|
2019
|
+
}
|
|
2015
2020
|
if (entry.isDirectory()) {
|
|
2016
2021
|
pending.push(path);
|
|
2017
2022
|
} else if (entry.isFile() && /^agent-.*\.jsonl$/.test(entry.name)) {
|
|
@@ -2029,23 +2034,47 @@ function workflowStorageRoot(ref) {
|
|
|
2029
2034
|
const nativeRoot = resolve3(homedir2(), ".claude", "projects");
|
|
2030
2035
|
return pathIsWithin(nativeRoot, ref.path) ? nativeRoot : dirname3(resolve3(ref.path));
|
|
2031
2036
|
}
|
|
2037
|
+
function relocatedWorkflowDirectory(ref, binding) {
|
|
2038
|
+
const source = binding.transcriptDir;
|
|
2039
|
+
if (resolve3(source) !== source || source.includes("\\")) return void 0;
|
|
2040
|
+
const parts = source.split("/").slice(-7);
|
|
2041
|
+
const sessionId = parts[3];
|
|
2042
|
+
if (parts[0] !== ".claude" || parts[1] !== "projects" || !parts[2] || !sessionId || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(sessionId) || parts[4] !== "subagents" || parts[5] !== "workflows" || parts[6] !== binding.runId) return void 0;
|
|
2043
|
+
const sessionRoot2 = join4(dirname3(resolve3(ref.path)), sessionId);
|
|
2044
|
+
return {
|
|
2045
|
+
sessionRoot: sessionRoot2,
|
|
2046
|
+
transcriptDir: join4(sessionRoot2, "subagents", "workflows", binding.runId)
|
|
2047
|
+
};
|
|
2048
|
+
}
|
|
2032
2049
|
async function workflowTranscriptDirectories(ref, bindings) {
|
|
2033
2050
|
const allowedRoot = workflowStorageRoot(ref);
|
|
2034
2051
|
const allowedRealRoot = await realpath(allowedRoot);
|
|
2035
2052
|
const directories = /* @__PURE__ */ new Map();
|
|
2036
2053
|
for (const runBindings of bindings.values()) {
|
|
2037
2054
|
for (const binding of runBindings) {
|
|
2038
|
-
|
|
2055
|
+
let directory = resolve3(binding.transcriptDir);
|
|
2056
|
+
let relocation;
|
|
2039
2057
|
if (!pathIsWithin(allowedRoot, directory)) {
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2058
|
+
relocation = relocatedWorkflowDirectory(ref, binding);
|
|
2059
|
+
if (!relocation || !pathIsWithin(allowedRoot, relocation.transcriptDir)) {
|
|
2060
|
+
throw new ClaudeTaskScopeError(
|
|
2061
|
+
`Claude Workflow transcript directory escapes ${allowedRoot}: ${directory}`
|
|
2062
|
+
);
|
|
2063
|
+
}
|
|
2064
|
+
directory = relocation.transcriptDir;
|
|
2043
2065
|
}
|
|
2044
2066
|
let realDirectory;
|
|
2045
2067
|
try {
|
|
2046
2068
|
realDirectory = await realpath(directory);
|
|
2047
2069
|
} catch (error) {
|
|
2048
|
-
if (isMissingPathError(error))
|
|
2070
|
+
if (isMissingPathError(error)) {
|
|
2071
|
+
if (relocation) {
|
|
2072
|
+
throw new ClaudeTaskScopeError(
|
|
2073
|
+
`Cannot relocate Claude Workflow run ${binding.runId}: copied transcript directory is missing: ${directory}`
|
|
2074
|
+
);
|
|
2075
|
+
}
|
|
2076
|
+
continue;
|
|
2077
|
+
}
|
|
2049
2078
|
throw error;
|
|
2050
2079
|
}
|
|
2051
2080
|
if (!pathIsWithin(allowedRealRoot, realDirectory)) {
|
|
@@ -2053,10 +2082,38 @@ async function workflowTranscriptDirectories(ref, bindings) {
|
|
|
2053
2082
|
`Claude Workflow transcript directory escapes ${allowedRoot}: ${directory}`
|
|
2054
2083
|
);
|
|
2055
2084
|
}
|
|
2056
|
-
|
|
2085
|
+
if (relocation) {
|
|
2086
|
+
const copiedRoot = dirname3(resolve3(ref.path));
|
|
2087
|
+
const copiedRealRoot = await realpath(copiedRoot);
|
|
2088
|
+
const sessionRealRoot = await realpath(relocation.sessionRoot);
|
|
2089
|
+
const expectedSessionRealRoot = resolve3(
|
|
2090
|
+
copiedRealRoot,
|
|
2091
|
+
relative2(copiedRoot, relocation.sessionRoot)
|
|
2092
|
+
);
|
|
2093
|
+
const expectedRealDirectory = resolve3(copiedRealRoot, relative2(copiedRoot, directory));
|
|
2094
|
+
if (sessionRealRoot !== expectedSessionRealRoot || realDirectory !== expectedRealDirectory) {
|
|
2095
|
+
throw new ClaudeTaskScopeError(
|
|
2096
|
+
`Claude Workflow transcript directory does not match copied session subtree ${relocation.sessionRoot}: ${directory}`
|
|
2097
|
+
);
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
const previous = directories.get(realDirectory);
|
|
2101
|
+
if (previous && (previous.runId !== binding.runId || resolve3(previous.sourceTranscriptDir ?? previous.transcriptDir) !== resolve3(binding.transcriptDir))) {
|
|
2102
|
+
throw new ClaudeTaskScopeError(
|
|
2103
|
+
`Claude Workflow transcript directory ${directory} has conflicting source bindings`
|
|
2104
|
+
);
|
|
2105
|
+
}
|
|
2106
|
+
directories.set(realDirectory, {
|
|
2107
|
+
runId: binding.runId,
|
|
2108
|
+
transcriptDir: directory,
|
|
2109
|
+
sourceTranscriptDir: binding.transcriptDir,
|
|
2110
|
+
...relocation ? { copiedWorkflow: true } : {}
|
|
2111
|
+
});
|
|
2057
2112
|
}
|
|
2058
2113
|
}
|
|
2059
|
-
return [...directories.values()].sort(
|
|
2114
|
+
return [...directories.values()].sort(
|
|
2115
|
+
(left, right) => left.transcriptDir.localeCompare(right.transcriptDir)
|
|
2116
|
+
);
|
|
2060
2117
|
}
|
|
2061
2118
|
async function collectClaudeSubagentSources(ref, bindings, signal) {
|
|
2062
2119
|
const subDir = join4(ref.path.replace(/\.jsonl$/, ""), "subagents");
|
|
@@ -2073,10 +2130,11 @@ async function collectClaudeSubagentSources(ref, bindings, signal) {
|
|
|
2073
2130
|
});
|
|
2074
2131
|
}
|
|
2075
2132
|
}
|
|
2076
|
-
for (const
|
|
2133
|
+
for (const location of await workflowTranscriptDirectories(ref, bindings)) {
|
|
2077
2134
|
signal?.throwIfAborted();
|
|
2078
|
-
const runId =
|
|
2079
|
-
|
|
2135
|
+
const { runId, transcriptDir } = location;
|
|
2136
|
+
const files2 = await listSubagentFiles(transcriptDir, signal, location.copiedWorkflow);
|
|
2137
|
+
for (const file of files2) {
|
|
2080
2138
|
const path = resolve3(file);
|
|
2081
2139
|
filesByPath.set(path, file);
|
|
2082
2140
|
const previous = workflowByPath.get(path);
|
|
@@ -2085,7 +2143,7 @@ async function collectClaudeSubagentSources(ref, bindings, signal) {
|
|
|
2085
2143
|
`Claude Workflow subagent ${file} belongs to multiple transcript directories`
|
|
2086
2144
|
);
|
|
2087
2145
|
}
|
|
2088
|
-
workflowByPath.set(path,
|
|
2146
|
+
workflowByPath.set(path, location);
|
|
2089
2147
|
}
|
|
2090
2148
|
}
|
|
2091
2149
|
const files = [...filesByPath.values()].sort();
|
|
@@ -2794,7 +2852,7 @@ var ClaudeAdapter = class {
|
|
|
2794
2852
|
if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
|
|
2795
2853
|
const cwd = `/${dir.replace(/^-/, "").replace(/-/g, "/")}`;
|
|
2796
2854
|
if (opts.cwd && !cwd.startsWith(opts.cwd)) continue;
|
|
2797
|
-
refs.push({ harness: this.harness, sessionId:
|
|
2855
|
+
refs.push({ harness: this.harness, sessionId: basename4(f, ".jsonl"), path, cwd, mtimeMs: st.mtimeMs });
|
|
2798
2856
|
}
|
|
2799
2857
|
}
|
|
2800
2858
|
return refs.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
@@ -2945,7 +3003,7 @@ var ClaudeAdapter = class {
|
|
|
2945
3003
|
const agent = {
|
|
2946
3004
|
file,
|
|
2947
3005
|
sourceKey,
|
|
2948
|
-
agentId:
|
|
3006
|
+
agentId: basename4(file, ".jsonl").replace(/^agent-/, ""),
|
|
2949
3007
|
meta,
|
|
2950
3008
|
...meta.toolUseId ? { parentToolUseId: meta.toolUseId } : {}
|
|
2951
3009
|
};
|
|
@@ -2954,11 +3012,12 @@ var ClaudeAdapter = class {
|
|
|
2954
3012
|
file,
|
|
2955
3013
|
workflowRunId,
|
|
2956
3014
|
workflowLocation.transcriptDir,
|
|
2957
|
-
workflowBindings
|
|
3015
|
+
workflowBindings,
|
|
3016
|
+
workflowLocation.sourceTranscriptDir
|
|
2958
3017
|
);
|
|
2959
3018
|
const parsed = await parseClaudeSubagent(ref, traceId, agent, options);
|
|
2960
3019
|
const { binding, confidence } = selectWorkflowBinding(
|
|
2961
|
-
|
|
3020
|
+
basename4(file),
|
|
2962
3021
|
runBindings,
|
|
2963
3022
|
firstSubagentTimestamp(agent, parsed),
|
|
2964
3023
|
meta.toolUseId
|
|
@@ -3527,7 +3586,7 @@ async function resolveCodexParentTask(ref, childSessionId, options = {}) {
|
|
|
3527
3586
|
// src/adapters/codex.ts
|
|
3528
3587
|
import { readdir as readdir6, stat as stat4 } from "fs/promises";
|
|
3529
3588
|
import { homedir as homedir4 } from "os";
|
|
3530
|
-
import { basename as
|
|
3589
|
+
import { basename as basename5, join as join6 } from "path";
|
|
3531
3590
|
import { deriveHexId as deriveHexId3 } from "@tangle-network/agent-trace-contract";
|
|
3532
3591
|
var SERVICE3 = "codex";
|
|
3533
3592
|
var SESSION_HEAD_LINES = 40;
|
|
@@ -3999,7 +4058,7 @@ var CodexAdapter = class {
|
|
|
3999
4058
|
}
|
|
4000
4059
|
if (opts.sinceMs && st.mtimeMs < opts.sinceMs) return void 0;
|
|
4001
4060
|
let cwd = null;
|
|
4002
|
-
let id =
|
|
4061
|
+
let id = basename5(path).replace(/^rollout-[\dT-]+-/, "").replace(/\.jsonl$/, "");
|
|
4003
4062
|
const ref = { harness: this.harness, sessionId: id, path, cwd, mtimeMs: st.mtimeMs };
|
|
4004
4063
|
const head = await takeJsonl(path, SESSION_HEAD_LINES, sessionJsonlOptions(ref));
|
|
4005
4064
|
for (const parsed of head) {
|
|
@@ -4027,7 +4086,7 @@ var CodexAdapter = class {
|
|
|
4027
4086
|
async locateBySessionId(sessionId, opts = {}) {
|
|
4028
4087
|
const refs = [];
|
|
4029
4088
|
for await (const path of walkRollouts(this.root())) {
|
|
4030
|
-
if (!
|
|
4089
|
+
if (!basename5(path).includes(sessionId)) continue;
|
|
4031
4090
|
const ref = await this.refFromPath(path, opts);
|
|
4032
4091
|
if (ref?.sessionId === sessionId) refs.push(ref);
|
|
4033
4092
|
}
|
|
@@ -5276,7 +5335,7 @@ var CopilotAdapter = class {
|
|
|
5276
5335
|
// src/adapters/factory.ts
|
|
5277
5336
|
import { readdir as readdir8, stat as stat6 } from "fs/promises";
|
|
5278
5337
|
import { homedir as homedir6 } from "os";
|
|
5279
|
-
import { basename as
|
|
5338
|
+
import { basename as basename6, join as join8 } from "path";
|
|
5280
5339
|
var SERVICE6 = "factory";
|
|
5281
5340
|
function textOf4(content) {
|
|
5282
5341
|
return capText(
|
|
@@ -5321,7 +5380,7 @@ var FactoryAdapter = class {
|
|
|
5321
5380
|
throw error;
|
|
5322
5381
|
}
|
|
5323
5382
|
if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
|
|
5324
|
-
refs.push({ harness: this.harness, sessionId:
|
|
5383
|
+
refs.push({ harness: this.harness, sessionId: basename6(f, ".jsonl"), path, cwd, mtimeMs: st.mtimeMs });
|
|
5325
5384
|
}
|
|
5326
5385
|
}
|
|
5327
5386
|
return refs.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
@@ -5460,7 +5519,7 @@ var FactoryAdapter = class {
|
|
|
5460
5519
|
// src/adapters/forge.ts
|
|
5461
5520
|
import { readdir as readdir9, stat as stat7 } from "fs/promises";
|
|
5462
5521
|
import { homedir as homedir7 } from "os";
|
|
5463
|
-
import { basename as
|
|
5522
|
+
import { basename as basename7, join as join9 } from "path";
|
|
5464
5523
|
var SERVICE7 = "forge";
|
|
5465
5524
|
function tokens(t) {
|
|
5466
5525
|
if (!t) return null;
|
|
@@ -5498,7 +5557,7 @@ var ForgeAdapter = class {
|
|
|
5498
5557
|
throw error;
|
|
5499
5558
|
}
|
|
5500
5559
|
if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
|
|
5501
|
-
refs.push({ harness: this.harness, sessionId:
|
|
5560
|
+
refs.push({ harness: this.harness, sessionId: basename7(f, ".json"), path, cwd: dir, mtimeMs: st.mtimeMs });
|
|
5502
5561
|
}
|
|
5503
5562
|
}
|
|
5504
5563
|
return refs.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
@@ -5960,7 +6019,7 @@ var OpencodeAdapter = class {
|
|
|
5960
6019
|
// src/adapters/pi.ts
|
|
5961
6020
|
import { readdir as readdir12, stat as stat10 } from "fs/promises";
|
|
5962
6021
|
import { homedir as homedir10 } from "os";
|
|
5963
|
-
import { basename as
|
|
6022
|
+
import { basename as basename8, join as join12, resolve as resolve4 } from "path";
|
|
5964
6023
|
var SERVICE9 = "pi";
|
|
5965
6024
|
function terminalStatus(message) {
|
|
5966
6025
|
const stopReason = message?.stopReason ?? "missing";
|
|
@@ -6042,7 +6101,7 @@ var PiAdapter = class {
|
|
|
6042
6101
|
}
|
|
6043
6102
|
const refs = [];
|
|
6044
6103
|
for (const path of paths) {
|
|
6045
|
-
const f =
|
|
6104
|
+
const f = basename8(path);
|
|
6046
6105
|
let st;
|
|
6047
6106
|
try {
|
|
6048
6107
|
st = await stat10(path);
|
|
@@ -6051,7 +6110,7 @@ var PiAdapter = class {
|
|
|
6051
6110
|
throw error;
|
|
6052
6111
|
}
|
|
6053
6112
|
if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
|
|
6054
|
-
let id =
|
|
6113
|
+
let id = basename8(f, ".jsonl").replace(/^[\dTZ.-]+_/, "");
|
|
6055
6114
|
let cwd = null;
|
|
6056
6115
|
const ref = {
|
|
6057
6116
|
harness: this.harness,
|
|
@@ -6263,7 +6322,7 @@ var PiAdapter = class {
|
|
|
6263
6322
|
// src/adapters/qwen.ts
|
|
6264
6323
|
import { readdir as readdir13, stat as stat11 } from "fs/promises";
|
|
6265
6324
|
import { homedir as homedir11 } from "os";
|
|
6266
|
-
import { basename as
|
|
6325
|
+
import { basename as basename9, join as join13 } from "path";
|
|
6267
6326
|
var SERVICE10 = "qwen";
|
|
6268
6327
|
function textOf9(parts) {
|
|
6269
6328
|
if (!parts) return "";
|
|
@@ -6306,7 +6365,7 @@ var QwenAdapter = class {
|
|
|
6306
6365
|
throw error;
|
|
6307
6366
|
}
|
|
6308
6367
|
if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
|
|
6309
|
-
refs.push({ harness: this.harness, sessionId:
|
|
6368
|
+
refs.push({ harness: this.harness, sessionId: basename9(f, ".jsonl"), path, cwd: null, mtimeMs: st.mtimeMs });
|
|
6310
6369
|
}
|
|
6311
6370
|
}
|
|
6312
6371
|
return refs.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
@@ -6719,7 +6778,7 @@ function extractAbsolutePaths(spans) {
|
|
|
6719
6778
|
async function resolveRepoAttrs(cwd) {
|
|
6720
6779
|
if (!cwd) return {};
|
|
6721
6780
|
const attrs = { [ATTR.CWD]: cwd };
|
|
6722
|
-
const
|
|
6781
|
+
const basename18 = (() => {
|
|
6723
6782
|
const trimmed = cwd.replace(/\/+$/g, "");
|
|
6724
6783
|
const idx = trimmed.lastIndexOf("/");
|
|
6725
6784
|
const name = idx === -1 ? trimmed : trimmed.slice(idx + 1);
|
|
@@ -6741,7 +6800,7 @@ async function resolveRepoAttrs(cwd) {
|
|
|
6741
6800
|
if (commit) attrs[ATTR.GIT_COMMIT] = commit;
|
|
6742
6801
|
} catch {
|
|
6743
6802
|
}
|
|
6744
|
-
attrs[ATTR.SUBJECT_KEY] =
|
|
6803
|
+
attrs[ATTR.SUBJECT_KEY] = basename18;
|
|
6745
6804
|
return attrs;
|
|
6746
6805
|
}
|
|
6747
6806
|
async function repoCandidateForPath(path) {
|
|
@@ -7933,7 +7992,7 @@ var COMMAND_PREFIXES = /* @__PURE__ */ new Set([
|
|
|
7933
7992
|
var ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=/;
|
|
7934
7993
|
var MAX_SHELL_DEPTH = 4;
|
|
7935
7994
|
var MAX_WORDS = 2e4;
|
|
7936
|
-
function
|
|
7995
|
+
function basename10(word) {
|
|
7937
7996
|
const cut = word.lastIndexOf("/");
|
|
7938
7997
|
return cut === -1 ? word : word.slice(cut + 1);
|
|
7939
7998
|
}
|
|
@@ -8032,7 +8091,7 @@ var ShellScanner = class {
|
|
|
8032
8091
|
let start = 0;
|
|
8033
8092
|
while (start < words.length) {
|
|
8034
8093
|
const word = words[start];
|
|
8035
|
-
if (ASSIGNMENT.test(word) || COMMAND_PREFIXES.has(
|
|
8094
|
+
if (ASSIGNMENT.test(word) || COMMAND_PREFIXES.has(basename10(word))) start += 1;
|
|
8036
8095
|
else break;
|
|
8037
8096
|
}
|
|
8038
8097
|
return start === 0 || start >= words.length ? words : words.slice(start);
|
|
@@ -8173,7 +8232,7 @@ function shellCommands(script, depth = 0) {
|
|
|
8173
8232
|
const out = [];
|
|
8174
8233
|
for (const words of commands) {
|
|
8175
8234
|
out.push(words);
|
|
8176
|
-
const name =
|
|
8235
|
+
const name = basename10(words[0] ?? "");
|
|
8177
8236
|
if (!SHELLS.has(name) || depth >= MAX_SHELL_DEPTH) continue;
|
|
8178
8237
|
for (let index = 1; index < words.length; index += 1) {
|
|
8179
8238
|
const word = words[index];
|
|
@@ -8201,7 +8260,7 @@ function commandTextFromInput(input) {
|
|
|
8201
8260
|
if (!Array.isArray(value) || value.length === 0) return void 0;
|
|
8202
8261
|
const words = value.filter((entry) => typeof entry === "string");
|
|
8203
8262
|
if (words.length !== value.length) return void 0;
|
|
8204
|
-
const head =
|
|
8263
|
+
const head = basename10(words[0] ?? "");
|
|
8205
8264
|
if (SHELLS.has(head)) {
|
|
8206
8265
|
const flag = words.findIndex((word, position) => position > 0 && /^-[a-z]*c$/.test(word));
|
|
8207
8266
|
if (flag !== -1 && words[flag + 1] !== void 0) return words[flag + 1];
|
|
@@ -8263,7 +8322,7 @@ function stringAttribute2(span2, key) {
|
|
|
8263
8322
|
const value = span2.attributes[key];
|
|
8264
8323
|
return typeof value === "string" ? value : null;
|
|
8265
8324
|
}
|
|
8266
|
-
function
|
|
8325
|
+
function basename11(word) {
|
|
8267
8326
|
const cut = word.lastIndexOf("/");
|
|
8268
8327
|
return cut === -1 ? word : word.slice(cut + 1);
|
|
8269
8328
|
}
|
|
@@ -8346,7 +8405,7 @@ function numbersInLine(line) {
|
|
|
8346
8405
|
function inferHeadBranch(commands, upTo, output) {
|
|
8347
8406
|
for (let index = upTo - 1; index >= 0; index -= 1) {
|
|
8348
8407
|
const words = commands[index];
|
|
8349
|
-
if (
|
|
8408
|
+
if (basename11(words[0] ?? "") !== "git" || words.length < 2) continue;
|
|
8350
8409
|
const sub = words[1];
|
|
8351
8410
|
const rest = words.slice(2);
|
|
8352
8411
|
if (sub === "push") {
|
|
@@ -8421,7 +8480,7 @@ function eventsFromRecord(record2, recordIndex) {
|
|
|
8421
8480
|
let createIndex = 0;
|
|
8422
8481
|
for (let index = 0; index < commands.length; index += 1) {
|
|
8423
8482
|
const words = commands[index];
|
|
8424
|
-
if (words.length < 3 || !GH_NAMES.has(
|
|
8483
|
+
if (words.length < 3 || !GH_NAMES.has(basename11(words[0])) || words[1] !== "pr") continue;
|
|
8425
8484
|
const action = words[2];
|
|
8426
8485
|
if (action === "create") {
|
|
8427
8486
|
const printedNumber = printed[createIndex] ?? null;
|
|
@@ -13178,7 +13237,7 @@ import {
|
|
|
13178
13237
|
readFile as readFile10,
|
|
13179
13238
|
rm as rm2
|
|
13180
13239
|
} from "fs/promises";
|
|
13181
|
-
import { basename as
|
|
13240
|
+
import { basename as basename13, dirname as dirname10, join as join23, resolve as resolve9 } from "path";
|
|
13182
13241
|
|
|
13183
13242
|
// src/codetracebench-files.ts
|
|
13184
13243
|
import { createHash as createHash11, randomUUID } from "crypto";
|
|
@@ -13196,13 +13255,13 @@ import {
|
|
|
13196
13255
|
writeFile as writeFile6
|
|
13197
13256
|
} from "fs/promises";
|
|
13198
13257
|
import { hostname } from "os";
|
|
13199
|
-
import { basename as
|
|
13258
|
+
import { basename as basename12, dirname as dirname9, isAbsolute as isAbsolute5, join as join21, relative as relative6, resolve as resolve8, sep as sep3 } from "path";
|
|
13200
13259
|
async function claimImportLock(input) {
|
|
13201
13260
|
const claimId = randomUUID();
|
|
13202
13261
|
const path = `${input.outputDirectory}.lock`;
|
|
13203
13262
|
const temporary = join21(
|
|
13204
13263
|
dirname9(path),
|
|
13205
|
-
`.${
|
|
13264
|
+
`.${basename12(path)}.${claimId}.tmp`
|
|
13206
13265
|
);
|
|
13207
13266
|
const metadata = {
|
|
13208
13267
|
kind: "traces.codetracebench-import-lock",
|
|
@@ -13364,7 +13423,7 @@ async function atomicWriteFile(path, content, signal) {
|
|
|
13364
13423
|
signal.throwIfAborted();
|
|
13365
13424
|
const temporary = join21(
|
|
13366
13425
|
dirname9(path),
|
|
13367
|
-
`.${
|
|
13426
|
+
`.${basename12(path)}.${randomUUID()}.tmp`
|
|
13368
13427
|
);
|
|
13369
13428
|
try {
|
|
13370
13429
|
await writeFile6(temporary, content, {
|
|
@@ -13697,7 +13756,7 @@ async function importCodeTraceBench(options) {
|
|
|
13697
13756
|
const rowsPath = resolve9(options.rowsPath);
|
|
13698
13757
|
const trajectoryPath = resolve9(options.trajectoryDir);
|
|
13699
13758
|
const outputDirectory = resolve9(options.outDir);
|
|
13700
|
-
const outputBase =
|
|
13759
|
+
const outputBase = basename13(outputDirectory);
|
|
13701
13760
|
if (!outputBase || outputDirectory === dirname10(outputDirectory)) {
|
|
13702
13761
|
throw new Error("CodeTraceBench output directory must not be a filesystem root");
|
|
13703
13762
|
}
|
|
@@ -13801,10 +13860,10 @@ async function runCodeTraceBenchImport(input) {
|
|
|
13801
13860
|
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13802
13861
|
input: {
|
|
13803
13862
|
revision,
|
|
13804
|
-
rowsFile:
|
|
13863
|
+
rowsFile: basename13(rowsPath),
|
|
13805
13864
|
rowsSha256,
|
|
13806
13865
|
rowsBytes: rowsBytes.byteLength,
|
|
13807
|
-
trajectoryDirectory:
|
|
13866
|
+
trajectoryDirectory: basename13(trajectoryDirectory),
|
|
13808
13867
|
sha256: inputSha256
|
|
13809
13868
|
},
|
|
13810
13869
|
settings: {
|
|
@@ -16148,7 +16207,7 @@ async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
|
|
|
16148
16207
|
// src/inspect.ts
|
|
16149
16208
|
import { mkdtemp as mkdtemp7, readFile as readFile13, writeFile as writeFile9 } from "fs/promises";
|
|
16150
16209
|
import { tmpdir as tmpdir7 } from "os";
|
|
16151
|
-
import { basename as
|
|
16210
|
+
import { basename as basename14, join as join26 } from "path";
|
|
16152
16211
|
var severityRank2 = {
|
|
16153
16212
|
high: 0,
|
|
16154
16213
|
medium: 1,
|
|
@@ -16331,7 +16390,7 @@ function addContextFindings(index, findings) {
|
|
|
16331
16390
|
refs: invalidJsonlFiles.slice(0, 5).map((file) => file.path)
|
|
16332
16391
|
});
|
|
16333
16392
|
}
|
|
16334
|
-
const skillRunFiles = files.filter((file) =>
|
|
16393
|
+
const skillRunFiles = files.filter((file) => basename14(file.path) === "skill-runs.jsonl" && (file.jsonl?.rows ?? 0) > 0).map((file) => {
|
|
16335
16394
|
const rows = file.jsonl?.rows ?? 0;
|
|
16336
16395
|
const keys = file.jsonl?.keys ?? {};
|
|
16337
16396
|
const linkKeyCounts = {
|
|
@@ -16871,7 +16930,7 @@ async function streamSessions(opts) {
|
|
|
16871
16930
|
|
|
16872
16931
|
// src/supervisor-run-context.ts
|
|
16873
16932
|
import { readFile as readFile14, stat as stat16 } from "fs/promises";
|
|
16874
|
-
import { basename as
|
|
16933
|
+
import { basename as basename15, join as join27 } from "path";
|
|
16875
16934
|
import { materializeTreeView } from "@tangle-network/agent-runtime/kernel";
|
|
16876
16935
|
import {
|
|
16877
16936
|
readRuntimeSupervisorRun
|
|
@@ -17037,7 +17096,7 @@ async function readFileRunContext(runDir, opts = {}) {
|
|
|
17037
17096
|
return {
|
|
17038
17097
|
...sources,
|
|
17039
17098
|
instanceId: opts.instanceId !== void 0 ? opts.instanceId : sources.instanceId,
|
|
17040
|
-
arm: opts.arm !== void 0 ? opts.arm :
|
|
17099
|
+
arm: opts.arm !== void 0 ? opts.arm : basename15(runDir),
|
|
17041
17100
|
workers: sources.workers !== null && coordination.present ? workerSources(sources.workers, coordination) : sources.workers,
|
|
17042
17101
|
...sources.journal !== null ? {
|
|
17043
17102
|
rootTranscriptRef: join27(runDir, SPAWN_JOURNAL_FILE),
|
|
@@ -19169,7 +19228,7 @@ function alreadyUploaded(state, harness, sessionId, hash2) {
|
|
|
19169
19228
|
import { readFile as readFile18 } from "fs/promises";
|
|
19170
19229
|
import { createRequire as createRequire2 } from "module";
|
|
19171
19230
|
import { hostname as hostname2 } from "os";
|
|
19172
|
-
import { basename as
|
|
19231
|
+
import { basename as basename16, join as join33 } from "path";
|
|
19173
19232
|
import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
|
|
19174
19233
|
import { assessShareSafety as assessShareSafety2, REDACTION_VERSION, redact as redact2, shareAllowed } from "@tangle-network/agent-eval/traces";
|
|
19175
19234
|
var require2 = createRequire2(import.meta.url);
|
|
@@ -19203,7 +19262,7 @@ async function gitBranch(cwd) {
|
|
|
19203
19262
|
async function sessionMeta(item) {
|
|
19204
19263
|
const meta = {
|
|
19205
19264
|
[ATTR.HARNESS]: item.ref.harness,
|
|
19206
|
-
[ATTR.SESSION_FILE]:
|
|
19265
|
+
[ATTR.SESSION_FILE]: basename16(item.ref.path),
|
|
19207
19266
|
[ATTR.HOST]: hostname2(),
|
|
19208
19267
|
[ATTR.UPLOADER]: `tangle-traces@${TRACES_VERSION}`,
|
|
19209
19268
|
[ATTR.REDACTION_VERSION]: REDACTION_VERSION,
|
|
@@ -19369,7 +19428,7 @@ async function executeUpload(plan, opts = {}) {
|
|
|
19369
19428
|
|
|
19370
19429
|
// src/verify-safe.ts
|
|
19371
19430
|
import { lstat as lstat4, readdir as readdir21, readFile as readFile19 } from "fs/promises";
|
|
19372
|
-
import { basename as
|
|
19431
|
+
import { basename as basename17, extname, join as join34, relative as relative7 } from "path";
|
|
19373
19432
|
import {
|
|
19374
19433
|
assessShareSafety as assessShareSafety3,
|
|
19375
19434
|
combineVerdicts as combineVerdicts2
|
|
@@ -19382,7 +19441,7 @@ async function verifySafe(target, options = {}) {
|
|
|
19382
19441
|
const unreadable = [];
|
|
19383
19442
|
const files = await listFiles(target, unreadable);
|
|
19384
19443
|
for (const file of files) {
|
|
19385
|
-
const label2 = relative7(target, file) ||
|
|
19444
|
+
const label2 = relative7(target, file) || basename17(file);
|
|
19386
19445
|
let bytes;
|
|
19387
19446
|
try {
|
|
19388
19447
|
bytes = await readFile19(file);
|
package/dist/cli.js
CHANGED
package/dist/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/traces",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.4",
|
|
4
4
|
"description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -61,11 +61,11 @@
|
|
|
61
61
|
"prepublishOnly": "pnpm check:source && pnpm build && pnpm check:package"
|
|
62
62
|
},
|
|
63
63
|
"dependencies": {
|
|
64
|
-
"@tangle-network/agent-eval": "0.
|
|
65
|
-
"@tangle-network/agent-interface": "2.
|
|
66
|
-
"@tangle-network/agent-runtime": "0.
|
|
67
|
-
"@tangle-network/agent-trace-contract": "
|
|
68
|
-
"@tangle-network/sandbox": "0.
|
|
64
|
+
"@tangle-network/agent-eval": "0.205.1",
|
|
65
|
+
"@tangle-network/agent-interface": "2.17.0",
|
|
66
|
+
"@tangle-network/agent-runtime": "0.293.0",
|
|
67
|
+
"@tangle-network/agent-trace-contract": "2.0.0",
|
|
68
|
+
"@tangle-network/sandbox": "0.60.10"
|
|
69
69
|
},
|
|
70
70
|
"devDependencies": {
|
|
71
71
|
"@types/node": "^22.20.4",
|
|
@@ -9,37 +9,30 @@ Use the deterministic CLI first; keep inspection local and read-only.
|
|
|
9
9
|
|
|
10
10
|
## Choose the way in
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
conforming exporter), read it directly. No adapter is involved.
|
|
12
|
+
Read conforming OTLP directly, including `@tangle-network/agent-trace-contract` exports.
|
|
14
13
|
|
|
15
14
|
```bash
|
|
16
|
-
traces validate spans.otlp.jsonl #
|
|
15
|
+
traces validate spans.otlp.jsonl # exit 1 on error findings
|
|
17
16
|
traces analyze --otlp spans.otlp.jsonl --out .traces/current.md
|
|
18
17
|
traces analyze --otlp results/sessions --out .traces/all.md # a directory of exports
|
|
19
18
|
```
|
|
20
19
|
|
|
21
|
-
Use `--harness`
|
|
22
|
-
Those adapters are the legacy edge, not the way to integrate a system you own.
|
|
20
|
+
Use `--harness` for native coding-agent formats; integrate systems you own through OTLP.
|
|
23
21
|
|
|
24
|
-
|
|
25
|
-
what they hold. An `otlp/` subdirectory, when present, is read on its own.
|
|
22
|
+
Directories read only OTLP files, preferring an `otlp/` subdirectory; other JSONL logs are listed separately.
|
|
26
23
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
as uncaptured, never as zero spend. The `trace conformance` section names each such
|
|
30
|
-
capability once; the markers repeat it at the number.
|
|
24
|
+
Report `Inputs incomplete` values as uncaptured, never zero spend.
|
|
25
|
+
The `trace conformance` section lists missing capabilities.
|
|
31
26
|
|
|
32
|
-
For
|
|
33
|
-
`steering chain` (which verdict caused which retry) before drawing any conclusion about
|
|
34
|
-
whether the agent was making progress.
|
|
27
|
+
For loops, inspect `round-over-round convergence` and `steering chain` before judging progress.
|
|
35
28
|
|
|
36
29
|
## Select the workflow
|
|
37
30
|
|
|
38
31
|
```bash
|
|
39
32
|
traces --version
|
|
40
33
|
traces list --harness codex --cwd "$PWD"
|
|
41
|
-
traces
|
|
42
|
-
--out .traces/current.
|
|
34
|
+
traces facts --harness codex --current --latest-turn --workflow \
|
|
35
|
+
--format text --out .traces/current-facts.txt
|
|
43
36
|
```
|
|
44
37
|
|
|
45
38
|
- Use `--current` for the active Codex session.
|
|
@@ -50,6 +43,19 @@ traces analyze --harness codex --current --latest-turn --workflow \
|
|
|
50
43
|
- For Claude Code, use `--harness claude-code --session <path> --latest-turn`; nested subagents are included.
|
|
51
44
|
|
|
52
45
|
Never join agents by display name or timestamp when Traces reports missing or conflicting IDs.
|
|
46
|
+
Read facts before full transcripts or analysis.
|
|
47
|
+
Check parent and worker counts against the task being investigated.
|
|
48
|
+
Run `analyze` only for questions the facts sheet cannot answer:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
traces analyze --harness codex --current --latest-turn --workflow \
|
|
52
|
+
--out .traces/current.md
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
For moved evidence, keep byte-identical transcripts and record their source paths and hashes.
|
|
56
|
+
Preserve Claude's `<session UUID>/subagents/workflows/<run ID>` subtree beside the copied parent JSONL file.
|
|
57
|
+
Restore missing child files from the source; rewriting recorded paths loses the original evidence.
|
|
58
|
+
Check current branches, edits, commits, and PRs before reporting unfinished work; transcripts describe past actions.
|
|
53
59
|
|
|
54
60
|
## Export
|
|
55
61
|
|
|
@@ -70,41 +76,37 @@ traces improve --harness codex --current --latest-turn --workflow \
|
|
|
70
76
|
`improve` writes findings, evidence, a report, and spans.
|
|
71
77
|
It edits no agent, repository, memory store, or knowledge base.
|
|
72
78
|
|
|
73
|
-
|
|
79
|
+
Preserve a session for later inspection:
|
|
74
80
|
|
|
75
81
|
```bash
|
|
76
82
|
traces bundle --harness claude-code --session <id-or-path> --out .traces/bundle
|
|
77
83
|
```
|
|
78
84
|
|
|
79
|
-
`bundle` copies
|
|
80
|
-
It
|
|
81
|
-
|
|
85
|
+
`bundle` copies transcripts, report, spans, and session-window `.evolve` rows with per-file SHA-256 hashes.
|
|
86
|
+
It uses no model calls.
|
|
87
|
+
Missing transcripts stop assembly; optional missing inputs appear in `manifest.absent`.
|
|
82
88
|
|
|
83
89
|
## Pick the view for the reader
|
|
84
90
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
Project the writer's copy instead:
|
|
91
|
+
The full bundle includes transcripts containing prior reports, even when report files are removed.
|
|
92
|
+
For a writer who must not see earlier conclusions, project an evidence-only copy:
|
|
89
93
|
|
|
90
94
|
```bash
|
|
91
95
|
traces bundle-view .traces/bundle --view evidence-only --out .traces/writer
|
|
92
96
|
```
|
|
93
97
|
|
|
94
|
-
|
|
95
|
-
It excludes
|
|
96
|
-
|
|
97
|
-
`manifest.view` names which copy you hold.
|
|
98
|
+
This retains the session index, evidence JSONL, and structured ledger records.
|
|
99
|
+
It excludes transcripts, reports, spans, prose ledgers, and files repeating an eight-word passage from excluded prose.
|
|
100
|
+
Check `manifest.view` and the rules and hashes in `manifest.excluded`.
|
|
98
101
|
|
|
99
102
|
## Ask your own question
|
|
100
103
|
|
|
101
|
-
`traces ask --last 1 --question "<text>" --budget 2 --dir .traces/ask`
|
|
102
|
-
prose answer, resolves every `trace://` citation in it, and spends model calls to do it.
|
|
104
|
+
`traces ask --last 1 --question "<text>" --budget 2 --dir .traces/ask` spends model calls and resolves answer citations.
|
|
103
105
|
|
|
104
106
|
## Report
|
|
105
107
|
|
|
106
108
|
- State the source (`--otlp <path>` or the harness), selected task boundary, session and span counts, and integrity warnings.
|
|
107
|
-
-
|
|
109
|
+
- Name missing capabilities and analyses affected by them.
|
|
108
110
|
- Cite each finding with its exact `trace://` reference.
|
|
109
111
|
- Mark missing outcome, cost, token, skill, or relationship data unknown.
|
|
110
112
|
- Do not infer task success from a completion message.
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
# Replay-verify batch — first measured replayability and fix-flip rates (2026-08-02)
|
|
2
|
-
|
|
3
|
-
Provenance: labels shas in `~/bench-cache/ctb-20260801/replay-batch/label-shas.txt`; docker 29.6.1; node v24.16.0; orchestrator :4095 + SDK adapter :4097; seed 17; fix model glm-5.2 via z.ai direct (16k cap, one doubled retry on length-empty); command recorded in `run2-20260802.log` header.
|
|
4
|
-
Corpora: dev-32, holdout-1, holdout-2, split3 (all spent mini-SWE splits; submit-only golds excluded per the split3 positional-degeneracy finding).
|
|
5
|
-
|
|
6
|
-
## Headline (n=22 replayable of 133 label entries)
|
|
7
|
-
|
|
8
|
-
| Rate | Value |
|
|
9
|
-
| --- | --- |
|
|
10
|
-
| Replayability (prefix ≤10% divergence AND arm A reproduces recorded returncode at the gold step) | 16/22 = 72.7% |
|
|
11
|
-
| Strict signature match (returncode + output substring) | 13/22 = 59.1% |
|
|
12
|
-
| Fix-flip (one-shot glm-5.2 corrected command makes the failure vanish) | 9/11 = 81.8% |
|
|
13
|
-
| Fix-flip, nonzero-returncode subset | 7/9 = 77.8% |
|
|
14
|
-
|
|
15
|
-
Exclusions (101 of 133, all named): 65 no-swe-raw-trajectory, 21 no-docker-image, 21 gold-only-submit-step, 4 no-gold-incorrect-step.
|
|
16
|
-
LLM: 16 calls, 5 residual failures (long-reasoning cases exceeding the doubled cap), 41,878 prompt + 51,312 completion tokens (~$0.14).
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
Generated 2026-08-02T00:55:37.309Z.
|
|
20
|
-
|
|
21
|
-
## Headline
|
|
22
|
-
|
|
23
|
-
- **Replayability rate: 72.7%** (16/22 replayable cases where the prefix replayed with ≤10% returncode divergence AND arm A reproduced the recorded returncode at the gold step k).
|
|
24
|
-
- Signature-strict rate: 59.1% (13/22; additionally requires the recorded error substring in arm A output).
|
|
25
|
-
- **Fix-flip rate: 81.8%** (9/11 arm-B-executed cases where the generated fix made the failure vanish).
|
|
26
|
-
- Fix-flip rate on recorded-rc≠0 cases: 77.8% (7/9; real recorded failures — a gold step recorded with rc 0 flips vacuously).
|
|
27
|
-
|
|
28
|
-
## Enumeration
|
|
29
|
-
|
|
30
|
-
133 label entries across 4 corpora → 22 replayable (SWE-style docker image + ≥1 gold incorrect step), 22 executed.
|
|
31
|
-
|
|
32
|
-
| exclusion reason | count |
|
|
33
|
-
| --- | --- |
|
|
34
|
-
| no-swe-raw-trajectory | 65 |
|
|
35
|
-
| no-docker-image | 21 |
|
|
36
|
-
| gold-only-submit-step | 21 |
|
|
37
|
-
| no-gold-incorrect-step | 4 |
|
|
38
|
-
|
|
39
|
-
Submit-command golds are never counterfactual targets (a gold on the submit step marks a bad submit decision, not a failed command):
|
|
40
|
-
|
|
41
|
-
| corpus | cases excluded (all golds = submit) | golds skipped within replayable cases |
|
|
42
|
-
| --- | --- | --- |
|
|
43
|
-
| dev-32 | 1 | 0 |
|
|
44
|
-
| holdout-1 | 2 | 0 |
|
|
45
|
-
| split3 | 18 | 0 |
|
|
46
|
-
|
|
47
|
-
## Per-case results
|
|
48
|
-
|
|
49
|
-
| corpus | trajectory | k | rc@k | prefix | div | div% | armA exit | rc match | sig match | replayed | fix | armB exit | vanished | wall s |
|
|
50
|
-
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
51
|
-
holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3179-107b… | 33 | 0 | 32 | 4 | 12.5 | 0 | yes | yes | no | — | — | — | 10.6
|
|
52
|
-
holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3421-8c92… | 23 | 2 | 22 | 1 | 4.5 | 2 | yes | no | **yes** | llm-failed | — | — | 7.2
|
|
53
|
-
holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-3975-5f1a… | 33 | 101 | 32 | 1 | 3.1 | 127 | no | no | no | — | — | — | 13.0
|
|
54
|
-
holdout-1 | miniswe-OpenAI__GPT-5-clap-rs__clap-4248-85bd… | 23 | 2 | 22 | 0 | 0 | 2 | yes | no | **yes** | generated | 0 | **yes** | 35.3
|
|
55
|
-
holdout-2 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 7 | 127 | 6 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 20.0
|
|
56
|
-
holdout-2 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 13 | 0 | 12 | 0 | 0 | 0 | yes | yes | **yes** | llm-failed | — | — | 13.5
|
|
57
|
-
holdout-2 | miniswe-OpenAI__GPT-5-instance_element-hq__el… | 17 | 2 | 16 | 0 | 0 | 2 | yes | no | **yes** | llm-failed | — | — | 25.9
|
|
58
|
-
holdout-2 | miniswe-OpenAI__GPT-5-instance_internetarchiv… | 10 | 0 | 9 | 0 | 0 | 0 | yes | yes | **yes** | generated | 0 | **yes** | 4.8
|
|
59
|
-
holdout-2 | miniswe-OpenAI__GPT-5-instance_qutebrowser__q… | 15 | 127 | 14 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.1
|
|
60
|
-
holdout-2 | miniswe-OpenAI__GPT-5-instance_qutebrowser__q… | 18 | 127 | 17 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.1
|
|
61
|
-
holdout-2 | miniswe-OpenAI__GPT-5-ponylang__ponyc-1057-56… | 35 | 127 | 34 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.4
|
|
62
|
-
holdout-2 | miniswe-OpenAI__GPT-5-ponylang__ponyc-2532-98… | 12 | 1 | 11 | 0 | 0 | 1 | yes | yes | **yes** | generated | 0 | **yes** | 4.4
|
|
63
|
-
holdout-2 | miniswe-OpenAI__GPT-5-sveltejs__svelte-10608-… | 23 | 127 | 22 | 0 | 0 | 127 | yes | yes | **yes** | generated | 0 | **yes** | 4.7
|
|
64
|
-
split3 | miniswe-OpenAI__GPT-5-fasterxml__jackson-data… | 18 | 127 | 17 | 0 | 0 | 127 | yes | yes | **yes** | generated | 127 | no | 5.7
|
|
65
|
-
split3 | miniswe-OpenAI__GPT-5-instance_ansible__ansib… | 21 | 0 | 20 | 1 | 5 | 2 | no | no | no | — | — | — | 4.8
|
|
66
|
-
split3 | miniswe-OpenAI__GPT-5-instance_element-hq__el… | 21 | 0 | 20 | 0 | 0 | 2 | no | no | no | — | — | — | 6.3
|
|
67
|
-
split3 | miniswe-OpenAI__GPT-5-nushell__nushell-13357-… | 28 | 127 | 27 | 0 | 0 | 127 | yes | yes | **yes** | generated | 127 | no | 5.9
|
|
68
|
-
split3 | miniswe-OpenAI__GPT-5-ponylang__ponyc-2205-2a… | 27 | 127 | 26 | 0 | 0 | 0 | no | no | no | — | — | — | 5.4
|
|
69
|
-
split3 | miniswe-OpenAI__GPT-5-ponylang__ponyc-3973-60… | 20 | 0 | 19 | 0 | 0 | 127 | no | no | no | — | — | — | 5.8
|
|
70
|
-
split3 | miniswe-OpenAI__GPT-5-simdjson__simdjson-2016… | 24 | 0 | 23 | 0 | 0 | 0 | yes | yes | **yes** | llm-failed | — | — | 5.2
|
|
71
|
-
split3 | miniswe-OpenAI__GPT-5-sveltejs__svelte-11913-… | 19 | 0 | 18 | 0 | 0 | 0 | yes | yes | **yes** | generated | 0 | **yes** | 41.2
|
|
72
|
-
split3 | miniswe-OpenAI__GPT-5-sveltejs__svelte-9962-5… | 21 | 127 | 20 | 0 | 0 | 127 | yes | yes | **yes** | llm-failed | — | — | 4.3
|
|
73
|
-
|
|
74
|
-
## LLM fix generation
|
|
75
|
-
|
|
76
|
-
Model glm-5.2: 16 calls (5 failed), 41878 prompt + 51312 completion tokens.
|