@forwardimpact/libeval 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/bin/fit-trace.js +53 -2
- package/package.json +1 -1
- package/src/agent-runner.js +3 -2
- package/src/commands/trace.js +46 -14
- package/src/index.js +2 -0
- package/src/signature-filter.js +27 -0
- package/src/tee-writer.js +21 -0
- package/src/trace-collector.js +52 -3
- package/src/trace-github.js +46 -11
- package/src/trace-query.js +141 -28
package/README.md
ADDED
package/bin/fit-trace.js
CHANGED
|
@@ -20,6 +20,9 @@ import {
|
|
|
20
20
|
runReasoningCommand,
|
|
21
21
|
runTimelineCommand,
|
|
22
22
|
runStatsCommand,
|
|
23
|
+
runInitCommand,
|
|
24
|
+
runTurnCommand,
|
|
25
|
+
runFilterCommand,
|
|
23
26
|
} from "../src/commands/trace.js";
|
|
24
27
|
|
|
25
28
|
const { version: VERSION } = JSON.parse(
|
|
@@ -42,7 +45,8 @@ const definition = {
|
|
|
42
45
|
},
|
|
43
46
|
repo: {
|
|
44
47
|
type: "string",
|
|
45
|
-
description:
|
|
48
|
+
description:
|
|
49
|
+
"GitHub repo override (default: $GITHUB_REPOSITORY or 'origin' git remote)",
|
|
46
50
|
},
|
|
47
51
|
},
|
|
48
52
|
},
|
|
@@ -55,7 +59,8 @@ const definition = {
|
|
|
55
59
|
artifact: { type: "string", description: "Artifact name override" },
|
|
56
60
|
repo: {
|
|
57
61
|
type: "string",
|
|
58
|
-
description:
|
|
62
|
+
description:
|
|
63
|
+
"GitHub repo override (default: $GITHUB_REPOSITORY or 'origin' git remote)",
|
|
59
64
|
},
|
|
60
65
|
},
|
|
61
66
|
},
|
|
@@ -97,6 +102,10 @@ const definition = {
|
|
|
97
102
|
type: "string",
|
|
98
103
|
description: "Surrounding turns per hit (default: 0)",
|
|
99
104
|
},
|
|
105
|
+
full: {
|
|
106
|
+
type: "boolean",
|
|
107
|
+
description: "Full content block in match descriptions",
|
|
108
|
+
},
|
|
100
109
|
},
|
|
101
110
|
},
|
|
102
111
|
{
|
|
@@ -133,11 +142,45 @@ const definition = {
|
|
|
133
142
|
args: "<file>",
|
|
134
143
|
description: "Token usage and cost breakdown",
|
|
135
144
|
},
|
|
145
|
+
{
|
|
146
|
+
name: "init",
|
|
147
|
+
args: "<file>",
|
|
148
|
+
description: "Full system/init event",
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
name: "turn",
|
|
152
|
+
args: "<file> <index>",
|
|
153
|
+
description: "Single turn by index",
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
name: "filter",
|
|
157
|
+
args: "<file>",
|
|
158
|
+
description: "Filter turns by structural properties",
|
|
159
|
+
options: {
|
|
160
|
+
role: {
|
|
161
|
+
type: "string",
|
|
162
|
+
description: "Turn role (system, user, assistant, tool_result)",
|
|
163
|
+
},
|
|
164
|
+
tool: {
|
|
165
|
+
type: "string",
|
|
166
|
+
description: "Tool name (matches assistant turns)",
|
|
167
|
+
},
|
|
168
|
+
error: {
|
|
169
|
+
type: "boolean",
|
|
170
|
+
description:
|
|
171
|
+
"Error tool_result turns only (flag-only; for non-errors use the API)",
|
|
172
|
+
},
|
|
173
|
+
},
|
|
174
|
+
},
|
|
136
175
|
],
|
|
137
176
|
globalOptions: {
|
|
138
177
|
help: { type: "boolean", short: "h", description: "Show this help" },
|
|
139
178
|
version: { type: "boolean", description: "Show version" },
|
|
140
179
|
json: { type: "boolean", description: "Output help as JSON" },
|
|
180
|
+
signatures: {
|
|
181
|
+
type: "boolean",
|
|
182
|
+
description: "Include thinking.signature blobs in output",
|
|
183
|
+
},
|
|
141
184
|
},
|
|
142
185
|
examples: [
|
|
143
186
|
"fit-trace runs --lookback 7d",
|
|
@@ -147,6 +190,11 @@ const definition = {
|
|
|
147
190
|
"fit-trace search structured.json 'error|fail' --context 1",
|
|
148
191
|
"fit-trace tool structured.json Bash",
|
|
149
192
|
"fit-trace batch structured.json 0 20",
|
|
193
|
+
"fit-trace init structured.json",
|
|
194
|
+
"fit-trace turn structured.json 3",
|
|
195
|
+
"fit-trace filter structured.json --role system",
|
|
196
|
+
"fit-trace filter structured.json --tool Bash --role assistant",
|
|
197
|
+
"fit-trace search structured.json 'error' --full",
|
|
150
198
|
],
|
|
151
199
|
};
|
|
152
200
|
|
|
@@ -168,6 +216,9 @@ const COMMANDS = {
|
|
|
168
216
|
reasoning: runReasoningCommand,
|
|
169
217
|
timeline: runTimelineCommand,
|
|
170
218
|
stats: runStatsCommand,
|
|
219
|
+
init: runInitCommand,
|
|
220
|
+
turn: runTurnCommand,
|
|
221
|
+
filter: runFilterCommand,
|
|
171
222
|
};
|
|
172
223
|
|
|
173
224
|
async function main() {
|
package/package.json
CHANGED
package/src/agent-runner.js
CHANGED
|
@@ -38,7 +38,7 @@ export class AgentRunner {
|
|
|
38
38
|
* @param {function} deps.query - SDK query function (injected for testing)
|
|
39
39
|
* @param {import("stream").Writable} deps.output - Stream to emit NDJSON to
|
|
40
40
|
* @param {string} [deps.model] - Claude model identifier
|
|
41
|
-
* @param {number} [deps.maxTurns] - Maximum agentic turns
|
|
41
|
+
* @param {number} [deps.maxTurns] - Maximum agentic turns; 0 means unlimited
|
|
42
42
|
* @param {string[]} [deps.allowedTools] - Tools the agent may use
|
|
43
43
|
* @param {function} [deps.onLine] - Callback invoked with each NDJSON line as it's produced
|
|
44
44
|
* @param {function} [deps.onBatch] - Async callback invoked with a batch of NDJSON lines at flush boundaries: every `batchSize` assistant text blocks, the terminal `result` message, and — on iterator crash/abort — once more in a final flush carrying any lines that never reached a boundary. Receives `(lines, { abort })` where calling `abort()` stops the in-flight SDK session via the AbortController. Optional; assignable at runtime so the Supervisor can swap it per turn.
|
|
@@ -73,7 +73,8 @@ export class AgentRunner {
|
|
|
73
73
|
options: {
|
|
74
74
|
cwd: this.cwd,
|
|
75
75
|
allowedTools: this.allowedTools,
|
|
76
|
-
|
|
76
|
+
maxTurns:
|
|
77
|
+
this.maxTurns === 0 ? Number.MAX_SAFE_INTEGER : this.maxTurns,
|
|
77
78
|
model: this.model,
|
|
78
79
|
permissionMode: PERMISSION_MODE,
|
|
79
80
|
allowDangerouslySkipPermissions: true,
|
package/src/commands/trace.js
CHANGED
|
@@ -3,6 +3,7 @@ import { join } from "node:path";
|
|
|
3
3
|
import { createTraceCollector } from "@forwardimpact/libeval";
|
|
4
4
|
import { createTraceQuery } from "../trace-query.js";
|
|
5
5
|
import { createTraceGitHub } from "../trace-github.js";
|
|
6
|
+
import { stripSignatures } from "../signature-filter.js";
|
|
6
7
|
|
|
7
8
|
// --- GitHub commands ---
|
|
8
9
|
|
|
@@ -20,7 +21,7 @@ export async function runRunsCommand(values, args, ctx) {
|
|
|
20
21
|
const pattern = args[0] ?? "agent";
|
|
21
22
|
const lookback = values.lookback ?? "7d";
|
|
22
23
|
const runs = await gh.listRuns({ pattern, lookback });
|
|
23
|
-
writeJSON(runs);
|
|
24
|
+
writeJSON(runs, values);
|
|
24
25
|
}
|
|
25
26
|
|
|
26
27
|
/**
|
|
@@ -51,14 +52,14 @@ export async function runDownloadCommand(values, args, ctx) {
|
|
|
51
52
|
result.files.push("structured.json");
|
|
52
53
|
}
|
|
53
54
|
|
|
54
|
-
writeJSON(result);
|
|
55
|
+
writeJSON(result, values);
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
// --- Query commands ---
|
|
58
59
|
|
|
59
60
|
/** @param {object} values @param {string[]} args - [file] */
|
|
60
61
|
export async function runOverviewCommand(values, args) {
|
|
61
|
-
writeJSON(loadTrace(args[0]).overview());
|
|
62
|
+
writeJSON(loadTrace(args[0]).overview(), values);
|
|
62
63
|
}
|
|
63
64
|
|
|
64
65
|
/** @param {object} values @param {string[]} args - [file] */
|
|
@@ -70,48 +71,53 @@ export async function runCountCommand(values, args) {
|
|
|
70
71
|
export async function runBatchCommand(values, args) {
|
|
71
72
|
writeJSON(
|
|
72
73
|
loadTrace(args[0]).batch(parseInt(args[1], 10), parseInt(args[2], 10)),
|
|
74
|
+
values,
|
|
73
75
|
);
|
|
74
76
|
}
|
|
75
77
|
|
|
76
78
|
/** @param {object} values @param {string[]} args - [file, N?] */
|
|
77
79
|
export async function runHeadCommand(values, args) {
|
|
78
80
|
const n = args[1] ? parseInt(args[1], 10) : 10;
|
|
79
|
-
writeJSON(loadTrace(args[0]).head(n));
|
|
81
|
+
writeJSON(loadTrace(args[0]).head(n), values);
|
|
80
82
|
}
|
|
81
83
|
|
|
82
84
|
/** @param {object} values @param {string[]} args - [file, N?] */
|
|
83
85
|
export async function runTailCommand(values, args) {
|
|
84
86
|
const n = args[1] ? parseInt(args[1], 10) : 10;
|
|
85
|
-
writeJSON(loadTrace(args[0]).tail(n));
|
|
87
|
+
writeJSON(loadTrace(args[0]).tail(n), values);
|
|
86
88
|
}
|
|
87
89
|
|
|
88
90
|
/** @param {object} values @param {string[]} args - [file, pattern] */
|
|
89
91
|
export async function runSearchCommand(values, args) {
|
|
90
92
|
const limit = values.limit ? parseInt(values.limit, 10) : 50;
|
|
91
93
|
const context = values.context ? parseInt(values.context, 10) : 0;
|
|
92
|
-
|
|
94
|
+
const full = values.full ?? false;
|
|
95
|
+
writeJSON(
|
|
96
|
+
loadTrace(args[0]).search(args[1], { limit, context, full }),
|
|
97
|
+
values,
|
|
98
|
+
);
|
|
93
99
|
}
|
|
94
100
|
|
|
95
101
|
/** @param {object} values @param {string[]} args - [file] */
|
|
96
102
|
export async function runToolsCommand(values, args) {
|
|
97
|
-
writeJSON(loadTrace(args[0]).toolFrequency());
|
|
103
|
+
writeJSON(loadTrace(args[0]).toolFrequency(), values);
|
|
98
104
|
}
|
|
99
105
|
|
|
100
106
|
/** @param {object} values @param {string[]} args - [file, name] */
|
|
101
107
|
export async function runToolCommand(values, args) {
|
|
102
|
-
writeJSON(loadTrace(args[0]).tool(args[1]));
|
|
108
|
+
writeJSON(loadTrace(args[0]).tool(args[1]), values);
|
|
103
109
|
}
|
|
104
110
|
|
|
105
111
|
/** @param {object} values @param {string[]} args - [file] */
|
|
106
112
|
export async function runErrorsCommand(values, args) {
|
|
107
|
-
writeJSON(loadTrace(args[0]).errors());
|
|
113
|
+
writeJSON(loadTrace(args[0]).errors(), values);
|
|
108
114
|
}
|
|
109
115
|
|
|
110
116
|
/** @param {object} values @param {string[]} args - [file] */
|
|
111
117
|
export async function runReasoningCommand(values, args) {
|
|
112
118
|
const from = values.from ? parseInt(values.from, 10) : undefined;
|
|
113
119
|
const to = values.to ? parseInt(values.to, 10) : undefined;
|
|
114
|
-
writeJSON(loadTrace(args[0]).reasoning({ from, to }));
|
|
120
|
+
writeJSON(loadTrace(args[0]).reasoning({ from, to }), values);
|
|
115
121
|
}
|
|
116
122
|
|
|
117
123
|
/** @param {object} values @param {string[]} args - [file] */
|
|
@@ -122,7 +128,26 @@ export async function runTimelineCommand(values, args) {
|
|
|
122
128
|
|
|
123
129
|
/** @param {object} values @param {string[]} args - [file] */
|
|
124
130
|
export async function runStatsCommand(values, args) {
|
|
125
|
-
writeJSON(loadTrace(args[0]).stats());
|
|
131
|
+
writeJSON(loadTrace(args[0]).stats(), values);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** @param {object} values @param {string[]} args - [file] */
|
|
135
|
+
export async function runInitCommand(values, args) {
|
|
136
|
+
writeJSON(loadTrace(args[0]).init(), values);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** @param {object} values @param {string[]} args - [file, index] */
|
|
140
|
+
export async function runTurnCommand(values, args) {
|
|
141
|
+
writeJSON(loadTrace(args[0]).turn(parseInt(args[1], 10)), values);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** @param {object} values @param {string[]} args - [file] */
|
|
145
|
+
export async function runFilterCommand(values, args) {
|
|
146
|
+
const opts = {};
|
|
147
|
+
if (values.role) opts.role = values.role;
|
|
148
|
+
if (values.tool) opts.toolName = values.tool;
|
|
149
|
+
if (values.error) opts.isError = true;
|
|
150
|
+
writeJSON(loadTrace(args[0]).filter(opts), values);
|
|
126
151
|
}
|
|
127
152
|
|
|
128
153
|
// --- Shared helpers ---
|
|
@@ -151,7 +176,14 @@ function loadTrace(file) {
|
|
|
151
176
|
return createTraceQuery(collector.toJSON());
|
|
152
177
|
}
|
|
153
178
|
|
|
154
|
-
/**
|
|
155
|
-
|
|
156
|
-
|
|
179
|
+
/**
|
|
180
|
+
* Write JSON output to stdout. By default strips `thinking.signature`
|
|
181
|
+
* base64 blobs from the payload so they don't dominate terminal output;
|
|
182
|
+
* pass `--signatures` (surfaced as `values.signatures`) to keep them.
|
|
183
|
+
* @param {*} data
|
|
184
|
+
* @param {object} [values]
|
|
185
|
+
*/
|
|
186
|
+
function writeJSON(data, values = {}) {
|
|
187
|
+
const output = values.signatures ? data : stripSignatures(data);
|
|
188
|
+
process.stdout.write(JSON.stringify(output, null, 2) + "\n");
|
|
157
189
|
}
|
package/src/index.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
export { TraceCollector, createTraceCollector } from "./trace-collector.js";
|
|
2
2
|
export { TraceQuery, createTraceQuery } from "./trace-query.js";
|
|
3
|
+
export { stripSignatures } from "./signature-filter.js";
|
|
3
4
|
export {
|
|
4
5
|
TraceGitHub,
|
|
5
6
|
createTraceGitHub,
|
|
7
|
+
detectRepoSlug,
|
|
6
8
|
parseGitRemote,
|
|
7
9
|
} from "./trace-github.js";
|
|
8
10
|
export { AgentRunner, createAgentRunner } from "./agent-runner.js";
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strip `thinking.signature` base64 blobs from a JSON-serializable value.
|
|
3
|
+
*
|
|
4
|
+
* Applied at the CLI output boundary — the stored structured trace keeps
|
|
5
|
+
* signatures intact (lossless storage), and the display filter drops them
|
|
6
|
+
* by default because they dominate output without helping analysis.
|
|
7
|
+
*
|
|
8
|
+
* Recursively walks the input. For any object whose `type === "thinking"`,
|
|
9
|
+
* the `signature` field is removed after copying. Signatures on objects of
|
|
10
|
+
* any other type are preserved.
|
|
11
|
+
*
|
|
12
|
+
* @param {*} value - Any JSON-serializable value
|
|
13
|
+
* @returns {*} A deep-copy with thinking signatures removed
|
|
14
|
+
*/
|
|
15
|
+
export function stripSignatures(value) {
|
|
16
|
+
if (value === null || typeof value !== "object") return value;
|
|
17
|
+
if (Array.isArray(value)) return value.map(stripSignatures);
|
|
18
|
+
|
|
19
|
+
const result = {};
|
|
20
|
+
for (const [key, val] of Object.entries(value)) {
|
|
21
|
+
result[key] = stripSignatures(val);
|
|
22
|
+
}
|
|
23
|
+
if (result.type === "thinking") {
|
|
24
|
+
delete result.signature;
|
|
25
|
+
}
|
|
26
|
+
return result;
|
|
27
|
+
}
|
package/src/tee-writer.js
CHANGED
|
@@ -163,6 +163,27 @@ export class TeeWriter extends Writable {
|
|
|
163
163
|
withPrefix,
|
|
164
164
|
}),
|
|
165
165
|
);
|
|
166
|
+
} else if (turn.role === "system") {
|
|
167
|
+
const label = turn.subtype ?? "system";
|
|
168
|
+
this.textStream.write(
|
|
169
|
+
renderTextLine({
|
|
170
|
+
source: turn.source,
|
|
171
|
+
text: `[${label}]`,
|
|
172
|
+
withPrefix,
|
|
173
|
+
}),
|
|
174
|
+
);
|
|
175
|
+
} else if (turn.role === "user") {
|
|
176
|
+
for (const block of turn.content) {
|
|
177
|
+
if (block.type === "text") {
|
|
178
|
+
this.textStream.write(
|
|
179
|
+
renderTextLine({
|
|
180
|
+
source: turn.source,
|
|
181
|
+
text: `[user] ${block.text}`,
|
|
182
|
+
withPrefix,
|
|
183
|
+
}),
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
166
187
|
}
|
|
167
188
|
}
|
|
168
189
|
}
|
package/src/trace-collector.js
CHANGED
|
@@ -37,6 +37,8 @@ export class TraceCollector {
|
|
|
37
37
|
this.result = null;
|
|
38
38
|
/** @type {number} */
|
|
39
39
|
this.turnIndex = 0;
|
|
40
|
+
/** @type {object|null} */
|
|
41
|
+
this.initEvent = null;
|
|
40
42
|
}
|
|
41
43
|
|
|
42
44
|
/**
|
|
@@ -73,7 +75,7 @@ export class TraceCollector {
|
|
|
73
75
|
|
|
74
76
|
switch (event.type) {
|
|
75
77
|
case "system":
|
|
76
|
-
this.handleSystem(event);
|
|
78
|
+
this.handleSystem(event, source);
|
|
77
79
|
break;
|
|
78
80
|
case "assistant":
|
|
79
81
|
this.handleAssistant(event, source);
|
|
@@ -91,8 +93,11 @@ export class TraceCollector {
|
|
|
91
93
|
|
|
92
94
|
/**
|
|
93
95
|
* @param {object} event
|
|
96
|
+
* @param {string|null} source
|
|
94
97
|
*/
|
|
95
|
-
handleSystem(event) {
|
|
98
|
+
handleSystem(event, source) {
|
|
99
|
+
const { type: _type, ...payload } = event;
|
|
100
|
+
|
|
96
101
|
if (event.subtype === "init") {
|
|
97
102
|
this.metadata = {
|
|
98
103
|
timestamp: event.timestamp ?? this.now(),
|
|
@@ -102,7 +107,16 @@ export class TraceCollector {
|
|
|
102
107
|
tools: event.tools ?? [],
|
|
103
108
|
permissionMode: event.permissionMode ?? null,
|
|
104
109
|
};
|
|
110
|
+
this.initEvent = payload;
|
|
105
111
|
}
|
|
112
|
+
|
|
113
|
+
this.turns.push({
|
|
114
|
+
index: this.turnIndex++,
|
|
115
|
+
role: "system",
|
|
116
|
+
source,
|
|
117
|
+
subtype: event.subtype ?? null,
|
|
118
|
+
data: payload,
|
|
119
|
+
});
|
|
106
120
|
}
|
|
107
121
|
|
|
108
122
|
/**
|
|
@@ -158,6 +172,19 @@ export class TraceCollector {
|
|
|
158
172
|
const contentItems = message.content;
|
|
159
173
|
if (!Array.isArray(contentItems)) return;
|
|
160
174
|
|
|
175
|
+
const textBlocks = contentItems
|
|
176
|
+
.filter((item) => item.type === "text")
|
|
177
|
+
.map((item) => ({ type: "text", text: item.text }));
|
|
178
|
+
|
|
179
|
+
if (textBlocks.length > 0) {
|
|
180
|
+
this.turns.push({
|
|
181
|
+
index: this.turnIndex++,
|
|
182
|
+
role: "user",
|
|
183
|
+
source,
|
|
184
|
+
content: textBlocks,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
|
|
161
188
|
for (const item of contentItems) {
|
|
162
189
|
if (item.type === "tool_result") {
|
|
163
190
|
this.turns.push({
|
|
@@ -204,7 +231,7 @@ export class TraceCollector {
|
|
|
204
231
|
*/
|
|
205
232
|
toJSON() {
|
|
206
233
|
return {
|
|
207
|
-
version: "1.
|
|
234
|
+
version: "1.1.0",
|
|
208
235
|
metadata: this.metadata ?? {
|
|
209
236
|
timestamp: this.now(),
|
|
210
237
|
sessionId: null,
|
|
@@ -213,6 +240,7 @@ export class TraceCollector {
|
|
|
213
240
|
tools: [],
|
|
214
241
|
permissionMode: null,
|
|
215
242
|
},
|
|
243
|
+
initEvent: this.initEvent ?? null,
|
|
216
244
|
turns: this.turns,
|
|
217
245
|
summary: this.result ?? {
|
|
218
246
|
result: "unknown",
|
|
@@ -271,6 +299,27 @@ export class TraceCollector {
|
|
|
271
299
|
withPrefix,
|
|
272
300
|
}),
|
|
273
301
|
);
|
|
302
|
+
} else if (turn.role === "system") {
|
|
303
|
+
const label = turn.subtype ?? "system";
|
|
304
|
+
out.push(
|
|
305
|
+
renderTextLine({
|
|
306
|
+
source: turn.source,
|
|
307
|
+
text: `[${label}]`,
|
|
308
|
+
withPrefix,
|
|
309
|
+
}),
|
|
310
|
+
);
|
|
311
|
+
} else if (turn.role === "user") {
|
|
312
|
+
for (const block of turn.content) {
|
|
313
|
+
if (block.type === "text") {
|
|
314
|
+
out.push(
|
|
315
|
+
renderTextLine({
|
|
316
|
+
source: turn.source,
|
|
317
|
+
text: `[user] ${block.text}`,
|
|
318
|
+
withPrefix,
|
|
319
|
+
}),
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
274
323
|
}
|
|
275
324
|
}
|
|
276
325
|
|
package/src/trace-github.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { execSync } from "node:child_process";
|
|
1
2
|
import { createWriteStream } from "node:fs";
|
|
2
3
|
import { mkdir } from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
@@ -116,7 +117,6 @@ export class TraceGitHub {
|
|
|
116
117
|
// Stream to disk then extract.
|
|
117
118
|
await pipeline(Readable.fromWeb(response.body), createWriteStream(zipPath));
|
|
118
119
|
|
|
119
|
-
const { execSync } = await import("node:child_process");
|
|
120
120
|
execSync(
|
|
121
121
|
`unzip -o -q ${JSON.stringify(zipPath)} -d ${JSON.stringify(dir)}`,
|
|
122
122
|
);
|
|
@@ -182,9 +182,51 @@ export function parseGitRemote(remote) {
|
|
|
182
182
|
const simple = remote.match(/^([^/:@]+)\/([^/]+)$/);
|
|
183
183
|
if (simple) return { owner: simple[1], repo: simple[2] };
|
|
184
184
|
|
|
185
|
+
// Generic URL fallback: any remote whose path ends in /owner/repo(.git)?
|
|
186
|
+
// Covers GitHub Enterprise, proxied git URLs, and mirrors.
|
|
187
|
+
const generic = remote.match(/[/:]([^/:@?#]+)\/([^/:@?#]+?)(?:\.git)?\/?$/);
|
|
188
|
+
if (generic) return { owner: generic[1], repo: generic[2] };
|
|
189
|
+
|
|
185
190
|
throw new Error(`Cannot parse GitHub remote: ${remote}`);
|
|
186
191
|
}
|
|
187
192
|
|
|
193
|
+
/**
|
|
194
|
+
* Detect the current GitHub repository slug as `{owner, repo}`.
|
|
195
|
+
*
|
|
196
|
+
* Resolution order:
|
|
197
|
+
* 1. `GITHUB_REPOSITORY` env var (set automatically by GitHub Actions).
|
|
198
|
+
* 2. `git remote get-url origin` in the current working directory.
|
|
199
|
+
*
|
|
200
|
+
* @returns {{owner: string, repo: string}}
|
|
201
|
+
* @throws {Error} with a clear message if neither source yields a parseable slug.
|
|
202
|
+
*/
|
|
203
|
+
export function detectRepoSlug() {
|
|
204
|
+
const env = process.env.GITHUB_REPOSITORY;
|
|
205
|
+
if (env && env.trim()) {
|
|
206
|
+
return parseGitRemote(env.trim());
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
let remote;
|
|
210
|
+
try {
|
|
211
|
+
remote = execSync("git remote get-url origin", {
|
|
212
|
+
encoding: "utf8",
|
|
213
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
214
|
+
}).trim();
|
|
215
|
+
} catch {
|
|
216
|
+
throw new Error(
|
|
217
|
+
"Cannot detect repository: set --repo <owner/repo>, export GITHUB_REPOSITORY, or run inside a git checkout with an 'origin' remote.",
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (!remote) {
|
|
222
|
+
throw new Error(
|
|
223
|
+
"Cannot detect repository: 'git remote get-url origin' returned an empty value. Pass --repo <owner/repo> or set GITHUB_REPOSITORY.",
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return parseGitRemote(remote);
|
|
228
|
+
}
|
|
229
|
+
|
|
188
230
|
/**
|
|
189
231
|
* Create a TraceGitHub instance. The caller is responsible for resolving
|
|
190
232
|
* the GitHub token — typically via `Config.ghToken()` — so credential
|
|
@@ -207,16 +249,9 @@ export async function createTraceGitHub(opts = {}) {
|
|
|
207
249
|
);
|
|
208
250
|
}
|
|
209
251
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
} else {
|
|
214
|
-
const { execSync } = await import("node:child_process");
|
|
215
|
-
const remote = execSync("git remote get-url origin", {
|
|
216
|
-
encoding: "utf8",
|
|
217
|
-
}).trim();
|
|
218
|
-
({ owner, repo } = parseGitRemote(remote));
|
|
219
|
-
}
|
|
252
|
+
const { owner, repo } = repoOverride
|
|
253
|
+
? parseGitRemote(repoOverride)
|
|
254
|
+
: detectRepoSlug();
|
|
220
255
|
|
|
221
256
|
return new TraceGitHub({ token, owner, repo });
|
|
222
257
|
}
|
package/src/trace-query.js
CHANGED
|
@@ -17,18 +17,90 @@ export class TraceQuery {
|
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
|
-
* High-level overview: metadata, summary, turn count,
|
|
20
|
+
* High-level overview: metadata, summary, turn count, tool frequency,
|
|
21
|
+
* and the first user message text (taskPrompt) when present.
|
|
21
22
|
* @returns {object}
|
|
22
23
|
*/
|
|
23
24
|
overview() {
|
|
25
|
+
const firstUser = this.turns.find((t) => t.role === "user");
|
|
26
|
+
const taskPrompt = firstUser
|
|
27
|
+
? firstUser.content
|
|
28
|
+
.filter((b) => b.type === "text")
|
|
29
|
+
.map((b) => b.text)
|
|
30
|
+
.join("\n")
|
|
31
|
+
: null;
|
|
24
32
|
return {
|
|
25
33
|
metadata: this.metadata,
|
|
26
34
|
summary: this.summary,
|
|
27
35
|
turnCount: this.turns.length,
|
|
28
36
|
tools: this.toolFrequency(),
|
|
37
|
+
taskPrompt,
|
|
29
38
|
};
|
|
30
39
|
}
|
|
31
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Full system/init event — the single most diagnostic message for
|
|
43
|
+
* root-cause analysis. Returns null for traces collected before this
|
|
44
|
+
* field existed.
|
|
45
|
+
* @returns {object|null}
|
|
46
|
+
*/
|
|
47
|
+
init() {
|
|
48
|
+
return this.trace.initEvent ?? null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Retrieve a single turn by its index.
|
|
53
|
+
* @param {number} index
|
|
54
|
+
* @returns {object|null}
|
|
55
|
+
*/
|
|
56
|
+
turn(index) {
|
|
57
|
+
return this.turns.find((t) => t.index === index) ?? null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Filter turns by composable structural criteria. All criteria are
|
|
62
|
+
* combined as AND. `tool()` and `errors()` remain as convenience
|
|
63
|
+
* shortcuts for pre-existing workflows.
|
|
64
|
+
*
|
|
65
|
+
* `toolName` matches assistant turns only. Applying `toolName` without
|
|
66
|
+
* `role: "assistant"` still drops every non-assistant turn, because
|
|
67
|
+
* resolving tool_use → tool_result pairs requires the `tool()` method.
|
|
68
|
+
* `isError` matches tool_result turns only. Combining `toolName` with
|
|
69
|
+
* `isError` therefore always returns `[]` (no turn is both assistant
|
|
70
|
+
* and tool_result) — use `tool(name)` for "errors from Bash"–shaped
|
|
71
|
+
* queries.
|
|
72
|
+
*
|
|
73
|
+
* @param {object} [opts]
|
|
74
|
+
* @param {string} [opts.role] - Exact role match (system | user |
|
|
75
|
+
* assistant | tool_result).
|
|
76
|
+
* @param {string} [opts.toolName] - Matches assistant turns with a
|
|
77
|
+
* tool_use block of this name. Drops all non-assistant turns.
|
|
78
|
+
* @param {boolean} [opts.isError] - Matches tool_result turns by
|
|
79
|
+
* `isError` value. Drops all non-tool_result turns.
|
|
80
|
+
* @returns {object[]}
|
|
81
|
+
*/
|
|
82
|
+
filter(opts = {}) {
|
|
83
|
+
const { role, toolName, isError } = opts;
|
|
84
|
+
return this.turns.filter((turn) => {
|
|
85
|
+
if (role !== undefined && turn.role !== role) return false;
|
|
86
|
+
if (isError !== undefined) {
|
|
87
|
+
if (turn.role !== "tool_result") return false;
|
|
88
|
+
if (turn.isError !== isError) return false;
|
|
89
|
+
}
|
|
90
|
+
if (toolName !== undefined) {
|
|
91
|
+
if (turn.role === "assistant") {
|
|
92
|
+
const has = turn.content.some(
|
|
93
|
+
(b) => b.type === "tool_use" && b.name === toolName,
|
|
94
|
+
);
|
|
95
|
+
if (!has) return false;
|
|
96
|
+
} else {
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return true;
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
32
104
|
/** @returns {number} */
|
|
33
105
|
count() {
|
|
34
106
|
return this.turns.length;
|
|
@@ -73,16 +145,18 @@ export class TraceQuery {
|
|
|
73
145
|
* @param {object} [opts]
|
|
74
146
|
* @param {number} [opts.context=0] - Number of surrounding turns to include
|
|
75
147
|
* @param {number} [opts.limit=50] - Max results
|
|
148
|
+
* @param {boolean} [opts.full=false] - Emit full content block text in
|
|
149
|
+
* match descriptions instead of the default narrow excerpt window.
|
|
76
150
|
* @returns {object[]} Array of {turn, matches, context?}
|
|
77
151
|
*/
|
|
78
152
|
search(pattern, opts = {}) {
|
|
79
|
-
const { context = 0, limit = 50 } = opts;
|
|
153
|
+
const { context = 0, limit = 50, full = false } = opts;
|
|
80
154
|
// eslint-disable-next-line security/detect-non-literal-regexp -- pattern is caller-controlled, not untrusted input
|
|
81
155
|
const re = new RegExp(pattern, "gi");
|
|
82
156
|
const hits = [];
|
|
83
157
|
|
|
84
158
|
for (const turn of this.turns) {
|
|
85
|
-
const matches = matchTurn(turn, re);
|
|
159
|
+
const matches = matchTurn(turn, re, full);
|
|
86
160
|
if (matches.length > 0) {
|
|
87
161
|
const entry = { turn, matches };
|
|
88
162
|
if (context > 0) {
|
|
@@ -273,40 +347,79 @@ export class TraceQuery {
|
|
|
273
347
|
* Search a single turn for regex matches. Returns array of match descriptions.
|
|
274
348
|
* @param {object} turn
|
|
275
349
|
* @param {RegExp} re
|
|
350
|
+
* @param {boolean} [full=false] - Emit full block text instead of an excerpt.
|
|
276
351
|
* @returns {string[]}
|
|
277
352
|
*/
|
|
278
|
-
function matchTurn(turn, re) {
|
|
353
|
+
function matchTurn(turn, re, full = false) {
|
|
354
|
+
if (turn.role === "assistant") return matchAssistantTurn(turn, re, full);
|
|
355
|
+
if (turn.role === "tool_result") return matchToolResultTurn(turn, re, full);
|
|
356
|
+
if (turn.role === "user") return matchUserTurn(turn, re, full);
|
|
357
|
+
return [];
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
function matchAssistantTurn(turn, re, full) {
|
|
279
361
|
const matches = [];
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
if (block.type === "tool_use") {
|
|
287
|
-
if (re.test(block.name)) {
|
|
288
|
-
re.lastIndex = 0;
|
|
289
|
-
matches.push(`tool_name: ${block.name}`);
|
|
290
|
-
}
|
|
291
|
-
const inputStr = JSON.stringify(block.input);
|
|
292
|
-
if (re.test(inputStr)) {
|
|
293
|
-
re.lastIndex = 0;
|
|
294
|
-
matches.push(
|
|
295
|
-
`tool_input(${block.name}): ${excerptAround(inputStr, re)}`,
|
|
296
|
-
);
|
|
297
|
-
}
|
|
298
|
-
}
|
|
362
|
+
for (const block of turn.content) {
|
|
363
|
+
if (block.type === "text") {
|
|
364
|
+
const desc = describeText(block.text, re, "text", full);
|
|
365
|
+
if (desc) matches.push(desc);
|
|
366
|
+
} else if (block.type === "tool_use") {
|
|
367
|
+
matches.push(...matchToolUseBlock(block, re, full));
|
|
299
368
|
}
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
369
|
+
}
|
|
370
|
+
return matches;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function matchToolUseBlock(block, re, full) {
|
|
374
|
+
const matches = [];
|
|
375
|
+
if (re.test(block.name)) {
|
|
376
|
+
re.lastIndex = 0;
|
|
377
|
+
matches.push(`tool_name: ${block.name}`);
|
|
378
|
+
}
|
|
379
|
+
const inputStr = JSON.stringify(block.input);
|
|
380
|
+
const inputDesc = describeText(
|
|
381
|
+
inputStr,
|
|
382
|
+
re,
|
|
383
|
+
`tool_input(${block.name})`,
|
|
384
|
+
full,
|
|
385
|
+
);
|
|
386
|
+
if (inputDesc) matches.push(inputDesc);
|
|
387
|
+
return matches;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function matchToolResultTurn(turn, re, full) {
|
|
391
|
+
const content = turn.content ?? "";
|
|
392
|
+
const desc = describeText(content, re, "result", full);
|
|
393
|
+
return desc ? [desc] : [];
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
function matchUserTurn(turn, re, full) {
|
|
397
|
+
const matches = [];
|
|
398
|
+
for (const block of turn.content ?? []) {
|
|
399
|
+
if (block.type === "text") {
|
|
400
|
+
const desc = describeText(block.text, re, "user_text", full);
|
|
401
|
+
if (desc) matches.push(desc);
|
|
305
402
|
}
|
|
306
403
|
}
|
|
307
404
|
return matches;
|
|
308
405
|
}
|
|
309
406
|
|
|
407
|
+
/**
|
|
408
|
+
* Return a `<prefix>: <text-or-excerpt>` description when `text` matches
|
|
409
|
+
* the regex, or null when it does not. Centralises the full-vs-excerpt
|
|
410
|
+
* choice so each call site just supplies its prefix.
|
|
411
|
+
* @param {string} text
|
|
412
|
+
* @param {RegExp} re
|
|
413
|
+
* @param {string} prefix
|
|
414
|
+
* @param {boolean} full
|
|
415
|
+
* @returns {string|null}
|
|
416
|
+
*/
|
|
417
|
+
function describeText(text, re, prefix, full) {
|
|
418
|
+
if (!re.test(text)) return null;
|
|
419
|
+
re.lastIndex = 0;
|
|
420
|
+
return `${prefix}: ${full ? text : excerptAround(text, re)}`;
|
|
421
|
+
}
|
|
422
|
+
|
|
310
423
|
/**
|
|
311
424
|
* Extract a short excerpt around the first regex match in text.
|
|
312
425
|
* @param {string} text
|