@forwardimpact/libharness 0.1.22 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -201
- package/README.md +196 -80
- package/bin/fit-benchmark.js +44 -0
- package/bin/fit-harness.js +358 -0
- package/bin/fit-selfedit.js +165 -0
- package/bin/fit-trace.js +510 -0
- package/package.json +41 -11
- package/src/agent-runner.js +256 -0
- package/src/benchmark/apm-installer.js +207 -0
- package/src/benchmark/env-loader.js +158 -0
- package/src/benchmark/hook-env.js +40 -0
- package/src/benchmark/invariants.js +141 -0
- package/src/benchmark/judge.js +187 -0
- package/src/benchmark/npm-installer.js +87 -0
- package/src/benchmark/report.js +604 -0
- package/src/benchmark/result.js +127 -0
- package/src/benchmark/runner.js +688 -0
- package/src/benchmark/scheduler.js +78 -0
- package/src/benchmark/task-family.js +260 -0
- package/src/benchmark/workdir.js +344 -0
- package/src/commands/assert.js +153 -0
- package/src/commands/benchmark-definition.js +175 -0
- package/src/commands/benchmark-invariants.js +73 -0
- package/src/commands/benchmark-report.js +51 -0
- package/src/commands/benchmark-run.js +175 -0
- package/src/commands/by-discussion.js +94 -0
- package/src/commands/callback.js +119 -0
- package/src/commands/discuss.js +132 -0
- package/src/commands/facilitate.js +123 -0
- package/src/commands/output.js +36 -0
- package/src/commands/run.js +152 -0
- package/src/commands/supervise.js +136 -0
- package/src/commands/task-input.js +54 -0
- package/src/commands/tee.js +53 -0
- package/src/commands/trace.js +630 -0
- package/src/commands/work-tracker.js +35 -0
- package/src/cost.js +79 -0
- package/src/discuss-tools.js +173 -0
- package/src/discusser.js +394 -0
- package/src/events/github.js +161 -0
- package/src/facilitator.js +205 -0
- package/src/inbox-poller.js +81 -0
- package/src/index.js +72 -2
- package/src/judge.js +210 -0
- package/src/message-bus.js +118 -0
- package/src/orchestration-loop.js +330 -0
- package/src/orchestration-toolkit.js +441 -0
- package/src/orchestrator-helpers.js +23 -0
- package/src/profile-prompt.js +266 -0
- package/src/redaction.js +253 -0
- package/src/render/line-renderer.js +54 -0
- package/src/render/orchestrator-filter.js +19 -0
- package/src/render/palette.js +63 -0
- package/src/render/tool-hints.js +154 -0
- package/src/render/turn-renderer.js +96 -0
- package/src/reply-emitter.js +47 -0
- package/src/sequence-counter.js +21 -0
- package/src/signature-filter.js +27 -0
- package/src/supervisor.js +236 -0
- package/src/tee-writer.js +150 -0
- package/src/trace-collector.js +444 -0
- package/src/trace-github.js +473 -0
- package/src/trace-multi.js +101 -0
- package/src/trace-query.js +748 -0
- package/src/trace-render.js +211 -0
- package/src/trace-usage.js +249 -0
- package/src/fixture/assertions.js +0 -42
- package/src/fixture/cache.js +0 -50
- package/src/fixture/eval.js +0 -146
- package/src/fixture/index.js +0 -9
- package/src/fixture/pathway.js +0 -451
- package/src/fixture/services.js +0 -56
- package/src/mock/clients.js +0 -135
- package/src/mock/config.js +0 -45
- package/src/mock/data.js +0 -46
- package/src/mock/fs.js +0 -111
- package/src/mock/grpc.js +0 -94
- package/src/mock/http.js +0 -60
- package/src/mock/index.js +0 -36
- package/src/mock/infra.js +0 -219
- package/src/mock/logger.js +0 -42
- package/src/mock/observer.js +0 -74
- package/src/mock/resource-index.js +0 -95
- package/src/mock/service-callbacks.js +0 -39
- package/src/mock/services.js +0 -79
- package/src/mock/spy.js +0 -44
- package/src/mock/storage.js +0 -118
|
@@ -0,0 +1,630 @@
|
|
|
1
|
+
import { join, dirname, basename } from "node:path";
|
|
2
|
+
import { isoTimestamp } from "@forwardimpact/libutil";
|
|
3
|
+
import { createTraceCollector, sumTraceCost } from "@forwardimpact/libharness";
|
|
4
|
+
import { createTraceQuery } from "../trace-query.js";
|
|
5
|
+
import { createTraceGitHub } from "../trace-github.js";
|
|
6
|
+
import { stripSignatures } from "../signature-filter.js";
|
|
7
|
+
import { runOver, aggregate, compareTwo } from "../trace-multi.js";
|
|
8
|
+
import {
|
|
9
|
+
renderToolCalls,
|
|
10
|
+
renderCommands,
|
|
11
|
+
renderPaths,
|
|
12
|
+
renderCompare,
|
|
13
|
+
renderStatsByTool,
|
|
14
|
+
renderStatsSummary,
|
|
15
|
+
renderSearch,
|
|
16
|
+
renderDefault,
|
|
17
|
+
} from "../trace-render.js";
|
|
18
|
+
|
|
19
|
+
// Every handler receives a libcli `InvocationContext`:
|
|
20
|
+
// ctx.options — parsed flag values (`cli.parse().values`)
|
|
21
|
+
// ctx.args — named positionals declared on the subcommand
|
|
22
|
+
// ctx.deps — host-injected collaborators: `{ runtime, config }`
|
|
23
|
+
// Handlers read/write the filesystem and stdout exclusively through
|
|
24
|
+
// `ctx.deps.runtime` and return `{ ok: true }` on success.
|
|
25
|
+
|
|
26
|
+
/** Characters whose presence in a `--file` value marks it as a glob. */
|
|
27
|
+
const GLOB_CHARS = /[*?[\]{}]/;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Resolve the cross-trace `--file` option (`ctx.options.file`) into a sorted
|
|
31
|
+
* flat list of file paths. A literal path passes through; a value carrying
|
|
32
|
+
* glob metacharacters expands via `runtime.fsSync.globSync`. The literal-path
|
|
33
|
+
* fast path means the common single-file and shell-pre-expanded cases never
|
|
34
|
+
* touch `globSync`.
|
|
35
|
+
* @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
|
|
36
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
37
|
+
* @returns {string[]}
|
|
38
|
+
*/
|
|
39
|
+
function resolveFiles(runtime, ctx) {
|
|
40
|
+
const raw = ctx.options.file;
|
|
41
|
+
const values = raw === undefined ? [] : Array.isArray(raw) ? raw : [raw];
|
|
42
|
+
const out = [];
|
|
43
|
+
for (const value of values) {
|
|
44
|
+
if (GLOB_CHARS.test(value)) {
|
|
45
|
+
out.push(...runtime.fsSync.globSync(value));
|
|
46
|
+
} else {
|
|
47
|
+
out.push(value);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return out.sort();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Emit a query result for a cross-trace verb: under `--format json` write the
|
|
55
|
+
* JSON payload (single-object verbs unwrap when single-file so the envelope
|
|
56
|
+
* deep-equals today's output); otherwise render text to stdout. Source
|
|
57
|
+
* attribution is the renderer's job, gated by `multi`.
|
|
58
|
+
* @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
|
|
59
|
+
* @param {object|object[]} result
|
|
60
|
+
* @param {Function} renderer
|
|
61
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
62
|
+
* @param {boolean} multi
|
|
63
|
+
* @param {boolean} [unwrap=false] - Single-object verb wrapped in a one-element array.
|
|
64
|
+
*/
|
|
65
|
+
function emit(runtime, result, renderer, ctx, multi, unwrap = false) {
|
|
66
|
+
if (ctx.options.format === "json") {
|
|
67
|
+
const payload = unwrap && !multi ? result[0] : result;
|
|
68
|
+
writeJSON(runtime, payload, ctx.options);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const text = renderer(result, {
|
|
72
|
+
multi,
|
|
73
|
+
signatures: !!ctx.options.signatures,
|
|
74
|
+
});
|
|
75
|
+
runtime.proc.stdout.write(text + "\n");
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// --- GitHub commands ---
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* List recent workflow runs matching a pattern.
|
|
82
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
83
|
+
*/
|
|
84
|
+
export async function runRunsCommand(ctx) {
|
|
85
|
+
const { runtime, config } = ctx.deps;
|
|
86
|
+
const gh = await createTraceGitHub({
|
|
87
|
+
token: config.ghToken(),
|
|
88
|
+
repo: ctx.options.repo,
|
|
89
|
+
runtime,
|
|
90
|
+
});
|
|
91
|
+
const lookback = ctx.options.lookback ?? "7d";
|
|
92
|
+
const runs = await gh.listRuns({
|
|
93
|
+
pattern: ctx.args.pattern,
|
|
94
|
+
lookback,
|
|
95
|
+
participant: ctx.options.participant,
|
|
96
|
+
});
|
|
97
|
+
writeJSON(runtime, runs, ctx.options);
|
|
98
|
+
return { ok: true };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Resolve a participant's lane trace for a known run id in one keyed lookup.
|
|
103
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
104
|
+
*/
|
|
105
|
+
export async function runFindCommand(ctx) {
|
|
106
|
+
const { runtime, config } = ctx.deps;
|
|
107
|
+
const gh = await createTraceGitHub({
|
|
108
|
+
token: config.ghToken(),
|
|
109
|
+
repo: ctx.options.repo,
|
|
110
|
+
runtime,
|
|
111
|
+
});
|
|
112
|
+
const result = await gh.findByKey(ctx.args["run-id"], ctx.args.participant, {
|
|
113
|
+
dir: ctx.options.dir,
|
|
114
|
+
});
|
|
115
|
+
writeJSON(runtime, result, ctx.options);
|
|
116
|
+
return { ok: true };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Download a trace artifact and auto-convert to structured JSON.
|
|
121
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
122
|
+
*/
|
|
123
|
+
export async function runDownloadCommand(ctx) {
|
|
124
|
+
const { runtime, config } = ctx.deps;
|
|
125
|
+
const gh = await createTraceGitHub({
|
|
126
|
+
token: config.ghToken(),
|
|
127
|
+
repo: ctx.options.repo,
|
|
128
|
+
runtime,
|
|
129
|
+
});
|
|
130
|
+
const result = await gh.downloadTrace(ctx.args["run-id"], {
|
|
131
|
+
dir: ctx.options.dir,
|
|
132
|
+
name: ctx.options.artifact,
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const ndjsonFile = result.files.find((f) => f.endsWith(".ndjson"));
|
|
136
|
+
if (ndjsonFile) {
|
|
137
|
+
const ndjsonPath = join(result.dir, ndjsonFile);
|
|
138
|
+
const collector = createTraceCollector({
|
|
139
|
+
now: () => isoTimestamp(runtime.clock.now()),
|
|
140
|
+
});
|
|
141
|
+
for (const line of runtime.fsSync
|
|
142
|
+
.readFileSync(ndjsonPath, "utf8")
|
|
143
|
+
.split("\n")) {
|
|
144
|
+
collector.addLine(line);
|
|
145
|
+
}
|
|
146
|
+
const structuredPath = join(result.dir, "structured.json");
|
|
147
|
+
runtime.fsSync.writeFileSync(
|
|
148
|
+
structuredPath,
|
|
149
|
+
JSON.stringify(collector.toJSON()) + "\n",
|
|
150
|
+
);
|
|
151
|
+
result.files.push("structured.json");
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
writeJSON(runtime, result, ctx.options);
|
|
155
|
+
return { ok: true };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// --- Query commands ---
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Build the injected loader the orchestrator uses (wires the runtime IO seam).
|
|
162
|
+
* @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
|
|
163
|
+
* @returns {(file: string) => import("../trace-query.js").TraceQuery}
|
|
164
|
+
*/
|
|
165
|
+
function loader(runtime) {
|
|
166
|
+
return (file) => loadTrace(runtime, file);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** No-files error envelope for a cross-trace verb. */
|
|
170
|
+
function noFiles(verb) {
|
|
171
|
+
return { ok: false, code: 1, error: `${verb}: no files (use --file)` };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
175
|
+
export async function runOverviewCommand(ctx) {
|
|
176
|
+
const { runtime } = ctx.deps;
|
|
177
|
+
const files = resolveFiles(runtime, ctx);
|
|
178
|
+
if (files.length === 0) return noFiles("overview");
|
|
179
|
+
const result = runOver(files, (tq) => [tq.overview()], loader(runtime));
|
|
180
|
+
emit(runtime, result, renderDefault, ctx, files.length > 1, true);
|
|
181
|
+
return { ok: true };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
185
|
+
export async function runCountCommand(ctx) {
|
|
186
|
+
const { runtime } = ctx.deps;
|
|
187
|
+
const files = resolveFiles(runtime, ctx);
|
|
188
|
+
if (files.length === 0) return noFiles("count");
|
|
189
|
+
const multi = files.length > 1;
|
|
190
|
+
const result = runOver(
|
|
191
|
+
files,
|
|
192
|
+
(tq) => [{ count: tq.count() }],
|
|
193
|
+
loader(runtime),
|
|
194
|
+
);
|
|
195
|
+
for (const r of result) {
|
|
196
|
+
const prefix = multi && r.source ? `${r.source}:` : "";
|
|
197
|
+
runtime.proc.stdout.write(`${prefix}${r.count}\n`);
|
|
198
|
+
}
|
|
199
|
+
return { ok: true };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
203
|
+
export async function runBatchCommand(ctx) {
|
|
204
|
+
const { runtime } = ctx.deps;
|
|
205
|
+
const result = loadTrace(runtime, ctx.args.file).batch(
|
|
206
|
+
parseInt(ctx.args.from, 10),
|
|
207
|
+
parseInt(ctx.args.to, 10),
|
|
208
|
+
);
|
|
209
|
+
emit(runtime, result, renderDefault, ctx, false);
|
|
210
|
+
return { ok: true };
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
214
|
+
export async function runHeadCommand(ctx) {
|
|
215
|
+
const { runtime } = ctx.deps;
|
|
216
|
+
const files = resolveFiles(runtime, ctx);
|
|
217
|
+
if (files.length === 0) return noFiles("head");
|
|
218
|
+
const n = ctx.options.lines ? parseInt(ctx.options.lines, 10) : 10;
|
|
219
|
+
const result = runOver(files, (tq) => tq.head(n), loader(runtime));
|
|
220
|
+
emit(runtime, result, renderDefault, ctx, files.length > 1);
|
|
221
|
+
return { ok: true };
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
225
|
+
export async function runTailCommand(ctx) {
|
|
226
|
+
const { runtime } = ctx.deps;
|
|
227
|
+
const files = resolveFiles(runtime, ctx);
|
|
228
|
+
if (files.length === 0) return noFiles("tail");
|
|
229
|
+
const n = ctx.options.lines ? parseInt(ctx.options.lines, 10) : 10;
|
|
230
|
+
const result = runOver(files, (tq) => tq.tail(n), loader(runtime));
|
|
231
|
+
emit(runtime, result, renderDefault, ctx, files.length > 1);
|
|
232
|
+
return { ok: true };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
236
|
+
export async function runSearchCommand(ctx) {
|
|
237
|
+
const { runtime } = ctx.deps;
|
|
238
|
+
const limit = ctx.options.limit ? parseInt(ctx.options.limit, 10) : 50;
|
|
239
|
+
const context = ctx.options.context ? parseInt(ctx.options.context, 10) : 0;
|
|
240
|
+
const full = ctx.options.full ?? false;
|
|
241
|
+
const result = loadTrace(runtime, ctx.args.file).search(ctx.args.pattern, {
|
|
242
|
+
limit,
|
|
243
|
+
context,
|
|
244
|
+
full,
|
|
245
|
+
});
|
|
246
|
+
emit(runtime, result, renderSearch, ctx, false);
|
|
247
|
+
return { ok: true };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
251
|
+
export async function runToolsCommand(ctx) {
|
|
252
|
+
const { runtime } = ctx.deps;
|
|
253
|
+
const files = resolveFiles(runtime, ctx);
|
|
254
|
+
if (files.length === 0) return noFiles("tools");
|
|
255
|
+
const result = aggregate(
|
|
256
|
+
files,
|
|
257
|
+
(tq) => tq.toolFrequency(),
|
|
258
|
+
(r) => r.tool,
|
|
259
|
+
loader(runtime),
|
|
260
|
+
);
|
|
261
|
+
emit(runtime, result, renderDefault, ctx, files.length > 1);
|
|
262
|
+
return { ok: true };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
266
|
+
export async function runToolCommand(ctx) {
|
|
267
|
+
const { runtime } = ctx.deps;
|
|
268
|
+
const result = loadTrace(runtime, ctx.args.file).tool(ctx.args.name);
|
|
269
|
+
emit(runtime, result, renderDefault, ctx, false);
|
|
270
|
+
return { ok: true };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
274
|
+
export async function runErrorsCommand(ctx) {
|
|
275
|
+
const { runtime } = ctx.deps;
|
|
276
|
+
const files = resolveFiles(runtime, ctx);
|
|
277
|
+
if (files.length === 0) return noFiles("errors");
|
|
278
|
+
const result = runOver(files, (tq) => tq.errors(), loader(runtime));
|
|
279
|
+
emit(runtime, result, renderDefault, ctx, files.length > 1);
|
|
280
|
+
return { ok: true };
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
284
|
+
export async function runReasoningCommand(ctx) {
|
|
285
|
+
const { runtime } = ctx.deps;
|
|
286
|
+
const files = resolveFiles(runtime, ctx);
|
|
287
|
+
if (files.length === 0) return noFiles("reasoning");
|
|
288
|
+
const from = ctx.options.from ? parseInt(ctx.options.from, 10) : undefined;
|
|
289
|
+
const to = ctx.options.to ? parseInt(ctx.options.to, 10) : undefined;
|
|
290
|
+
const result = runOver(
|
|
291
|
+
files,
|
|
292
|
+
(tq) => tq.reasoning({ from, to }),
|
|
293
|
+
loader(runtime),
|
|
294
|
+
);
|
|
295
|
+
emit(runtime, result, renderDefault, ctx, files.length > 1);
|
|
296
|
+
return { ok: true };
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
300
|
+
export async function runTimelineCommand(ctx) {
|
|
301
|
+
const { runtime } = ctx.deps;
|
|
302
|
+
const files = resolveFiles(runtime, ctx);
|
|
303
|
+
if (files.length === 0) return noFiles("timeline");
|
|
304
|
+
const multi = files.length > 1;
|
|
305
|
+
for (const file of files) {
|
|
306
|
+
if (multi) runtime.proc.stdout.write(`# ${basename(file)}\n`);
|
|
307
|
+
runtime.proc.stdout.write(
|
|
308
|
+
loadTrace(runtime, file).timeline().join("\n") + "\n",
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
return { ok: true };
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/** Select the per-file `stats` query for the active flag combination. */
|
|
315
|
+
function statsQuery(ctx) {
|
|
316
|
+
if (ctx.options.summary) return (tq) => tq.statsSummary();
|
|
317
|
+
if (ctx.options["by-tool"]) return (tq) => tq.statsByTool();
|
|
318
|
+
return (tq) => tq.stats();
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** Select the `stats` text renderer for the active flag combination. */
|
|
322
|
+
function statsRenderer(ctx) {
|
|
323
|
+
if (ctx.options.summary) return renderStatsSummary;
|
|
324
|
+
if (ctx.options["by-tool"]) return renderStatsByTool;
|
|
325
|
+
return (result) => renderDefault(result);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
329
|
+
export async function runStatsCommand(ctx) {
|
|
330
|
+
const { runtime } = ctx.deps;
|
|
331
|
+
const files = resolveFiles(runtime, ctx);
|
|
332
|
+
if (files.length === 0) return noFiles("stats");
|
|
333
|
+
const multi = files.length > 1;
|
|
334
|
+
const query = statsQuery(ctx);
|
|
335
|
+
// stats results are per-file objects; one block per file (no cross-file sum),
|
|
336
|
+
// tagged with source only when multi-file.
|
|
337
|
+
const results = files.map((file) => ({
|
|
338
|
+
result: query(loadTrace(runtime, file)),
|
|
339
|
+
source: multi ? basename(file) : undefined,
|
|
340
|
+
}));
|
|
341
|
+
|
|
342
|
+
if (ctx.options.format === "json") {
|
|
343
|
+
const payloads = results.map((r) =>
|
|
344
|
+
multi ? { ...r.result, source: r.source } : r.result,
|
|
345
|
+
);
|
|
346
|
+
writeJSON(runtime, multi ? payloads : payloads[0], ctx.options);
|
|
347
|
+
return { ok: true };
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
const render = statsRenderer(ctx);
|
|
351
|
+
const blocks = results.map((r) =>
|
|
352
|
+
multi ? `# ${r.source}\n${render(r.result)}` : render(r.result),
|
|
353
|
+
);
|
|
354
|
+
runtime.proc.stdout.write(blocks.join("\n") + "\n");
|
|
355
|
+
return { ok: true };
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Total run cost across every participant (agent, supervisor, judge, and any
|
|
360
|
+
* named profile), summed from each `result` event in the trace and attributed
|
|
361
|
+
* per source. The combined trace from a supervised, facilitated, or discuss
|
|
362
|
+
* session already interleaves all participants, so one file yields the whole
|
|
363
|
+
* run's spend. Default output is `{totalCostUsd, bySource}` JSON; `--markdown`
|
|
364
|
+
* emits a GitHub-flavored block to redirect into `$GITHUB_STEP_SUMMARY`.
|
|
365
|
+
*
|
|
366
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
367
|
+
*/
|
|
368
|
+
export async function runCostCommand(ctx) {
|
|
369
|
+
const { runtime } = ctx.deps;
|
|
370
|
+
const cost = computeTraceCost(
|
|
371
|
+
runtime.fsSync.readFileSync(ctx.args.file, "utf8"),
|
|
372
|
+
);
|
|
373
|
+
if (ctx.options.markdown) {
|
|
374
|
+
runtime.proc.stdout.write(renderCostMarkdown(cost));
|
|
375
|
+
} else {
|
|
376
|
+
writeJSON(runtime, cost, ctx.options);
|
|
377
|
+
}
|
|
378
|
+
return { ok: true };
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Render a cost summary as a GitHub-flavored markdown block for a CI step
|
|
383
|
+
* summary: a headline total plus a per-participant table (descending).
|
|
384
|
+
* @param {{totalCostUsd: number, bySource: Record<string, number>}} cost
|
|
385
|
+
* @returns {string}
|
|
386
|
+
*/
|
|
387
|
+
function renderCostMarkdown(cost) {
|
|
388
|
+
const lines = [
|
|
389
|
+
`### 💰 Run cost: $${cost.totalCostUsd.toFixed(4)}`,
|
|
390
|
+
"",
|
|
391
|
+
"Summed across every participant (agent, supervisor, judge, named profiles).",
|
|
392
|
+
];
|
|
393
|
+
const sources = Object.entries(cost.bySource).sort((a, b) => b[1] - a[1]);
|
|
394
|
+
if (sources.length > 0) {
|
|
395
|
+
lines.push("", "| Participant | Cost (USD) |", "| --- | --- |");
|
|
396
|
+
for (const [source, usd] of sources) {
|
|
397
|
+
lines.push(`| ${source} | ${usd.toFixed(4)} |`);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
return lines.join("\n") + "\n";
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
404
|
+
export async function runInitCommand(ctx) {
|
|
405
|
+
const { runtime } = ctx.deps;
|
|
406
|
+
const files = resolveFiles(runtime, ctx);
|
|
407
|
+
if (files.length === 0) return noFiles("init");
|
|
408
|
+
const result = runOver(files, (tq) => [tq.init()], loader(runtime));
|
|
409
|
+
emit(runtime, result, renderDefault, ctx, files.length > 1, true);
|
|
410
|
+
return { ok: true };
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
414
|
+
export async function runTurnCommand(ctx) {
|
|
415
|
+
const { runtime } = ctx.deps;
|
|
416
|
+
const result = loadTrace(runtime, ctx.args.file).turn(
|
|
417
|
+
parseInt(ctx.args.index, 10),
|
|
418
|
+
);
|
|
419
|
+
emit(runtime, result, renderDefault, ctx, false);
|
|
420
|
+
return { ok: true };
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
424
|
+
export async function runFilterCommand(ctx) {
|
|
425
|
+
const { runtime } = ctx.deps;
|
|
426
|
+
const files = resolveFiles(runtime, ctx);
|
|
427
|
+
if (files.length === 0) return noFiles("filter");
|
|
428
|
+
const opts = {};
|
|
429
|
+
if (ctx.options.role) opts.role = ctx.options.role;
|
|
430
|
+
if (ctx.options.tool) opts.toolName = ctx.options.tool;
|
|
431
|
+
if (ctx.options.error) opts.isError = true;
|
|
432
|
+
const result = runOver(files, (tq) => tq.filter(opts), loader(runtime));
|
|
433
|
+
emit(runtime, result, renderDefault, ctx, files.length > 1);
|
|
434
|
+
return { ok: true };
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// --- Aggregator verbs (tool-calls, commands, paths, compare) ---
|
|
438
|
+
|
|
439
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
440
|
+
export async function runToolCallsCommand(ctx) {
|
|
441
|
+
const { runtime } = ctx.deps;
|
|
442
|
+
const files = resolveFiles(runtime, ctx);
|
|
443
|
+
if (files.length === 0) return noFiles("tool-calls");
|
|
444
|
+
const result = runOver(files, (tq) => tq.toolCalls(), loader(runtime));
|
|
445
|
+
emit(runtime, result, renderToolCalls, ctx, files.length > 1);
|
|
446
|
+
return { ok: true };
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
450
|
+
export async function runCommandsCommand(ctx) {
|
|
451
|
+
const { runtime } = ctx.deps;
|
|
452
|
+
const files = resolveFiles(runtime, ctx);
|
|
453
|
+
if (files.length === 0) return noFiles("commands");
|
|
454
|
+
const result = runOver(
|
|
455
|
+
files,
|
|
456
|
+
(tq) => tq.commands(ctx.options.match),
|
|
457
|
+
loader(runtime),
|
|
458
|
+
);
|
|
459
|
+
emit(runtime, result, renderCommands, ctx, files.length > 1);
|
|
460
|
+
return { ok: true };
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
464
|
+
export async function runPathsCommand(ctx) {
|
|
465
|
+
const { runtime } = ctx.deps;
|
|
466
|
+
const files = resolveFiles(runtime, ctx);
|
|
467
|
+
if (files.length === 0) return noFiles("paths");
|
|
468
|
+
const result = aggregate(
|
|
469
|
+
files,
|
|
470
|
+
(tq) => tq.paths(ctx.options.prefix),
|
|
471
|
+
(r) => r.path,
|
|
472
|
+
loader(runtime),
|
|
473
|
+
);
|
|
474
|
+
emit(runtime, result, renderPaths, ctx, files.length > 1);
|
|
475
|
+
return { ok: true };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/** @param {import("@forwardimpact/libcli").InvocationContext} ctx */
|
|
479
|
+
export async function runCompareCommand(ctx) {
|
|
480
|
+
const { runtime } = ctx.deps;
|
|
481
|
+
const result = compareTwo(
|
|
482
|
+
ctx.args["file-a"],
|
|
483
|
+
ctx.args["file-b"],
|
|
484
|
+
loader(runtime),
|
|
485
|
+
);
|
|
486
|
+
emit(runtime, result, renderCompare, ctx, false);
|
|
487
|
+
return { ok: true };
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
// --- Split command ---
|
|
491
|
+
|
|
492
|
+
/** Valid source name pattern: lowercase letter, then lowercase alphanumeric or hyphen. */
|
|
493
|
+
const VALID_SOURCE_NAME = /^[a-z][a-z0-9-]*$/;
|
|
494
|
+
|
|
495
|
+
/** Sources whose name is itself a structural role; classified into the role they represent. */
|
|
496
|
+
const STRUCTURAL_ROLES = new Set(["agent", "supervisor", "facilitator"]);
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Split a combined NDJSON trace into per-source files using the
|
|
500
|
+
* `trace--<case>--<participant>.<role>.ndjson` convention.
|
|
501
|
+
*
|
|
502
|
+
* Each valid envelope source becomes one output file. Structural sources
|
|
503
|
+
* (`agent`, `supervisor`, `facilitator`) classify into the matching role and
|
|
504
|
+
* use their own name as participant; profile-named sources (e.g.
|
|
505
|
+
* `staff-engineer`) classify as agents with the profile in the participant
|
|
506
|
+
* slot. Orchestrator events and invalid source names are dropped.
|
|
507
|
+
*
|
|
508
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
509
|
+
*/
|
|
510
|
+
export async function runSplitCommand(ctx) {
|
|
511
|
+
const { runtime } = ctx.deps;
|
|
512
|
+
const file = ctx.args.file;
|
|
513
|
+
if (!file) return { ok: false, code: 1, error: "split: missing input file" };
|
|
514
|
+
|
|
515
|
+
const mode = ctx.options.mode;
|
|
516
|
+
if (!mode) return { ok: false, code: 1, error: "split: --mode is required" };
|
|
517
|
+
if (!["run", "supervise", "facilitate"].includes(mode)) {
|
|
518
|
+
return { ok: false, code: 1, error: `split: invalid --mode "${mode}"` };
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
const caseId = ctx.options.case ?? "default";
|
|
522
|
+
const outputDir = ctx.options["output-dir"] || dirname(file);
|
|
523
|
+
runtime.fsSync.mkdirSync(outputDir, { recursive: true });
|
|
524
|
+
|
|
525
|
+
const buckets = parseBuckets(runtime.fsSync.readFileSync(file, "utf8"));
|
|
526
|
+
|
|
527
|
+
for (const [source, lines] of buckets.entries()) {
|
|
528
|
+
if (!VALID_SOURCE_NAME.test(source)) continue;
|
|
529
|
+
const role = STRUCTURAL_ROLES.has(source) ? source : "agent";
|
|
530
|
+
const outPath = join(
|
|
531
|
+
outputDir,
|
|
532
|
+
`trace--${caseId}--${source}.${role}.ndjson`,
|
|
533
|
+
);
|
|
534
|
+
runtime.fsSync.writeFileSync(outPath, lines.join("\n") + "\n");
|
|
535
|
+
}
|
|
536
|
+
return { ok: true };
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* Parse NDJSON content into per-source buckets of unwrapped event lines.
|
|
541
|
+
* Skips empty lines, malformed JSON, non-envelope lines, and orchestrator events.
|
|
542
|
+
* @param {string} content - Raw NDJSON file content
|
|
543
|
+
* @returns {Map<string, string[]>} source name -> array of unwrapped JSON lines
|
|
544
|
+
*/
|
|
545
|
+
function parseBuckets(content) {
|
|
546
|
+
const buckets = new Map();
|
|
547
|
+
|
|
548
|
+
for (const raw of content.split("\n")) {
|
|
549
|
+
const trimmed = raw.trim();
|
|
550
|
+
if (!trimmed) continue;
|
|
551
|
+
|
|
552
|
+
let envelope;
|
|
553
|
+
try {
|
|
554
|
+
envelope = JSON.parse(trimmed);
|
|
555
|
+
} catch {
|
|
556
|
+
continue;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
if (!envelope.event || typeof envelope.source !== "string") continue;
|
|
560
|
+
if (envelope.source === "orchestrator") continue;
|
|
561
|
+
|
|
562
|
+
if (!buckets.has(envelope.source)) {
|
|
563
|
+
buckets.set(envelope.source, []);
|
|
564
|
+
}
|
|
565
|
+
buckets.get(envelope.source).push(JSON.stringify(envelope.event));
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
return buckets;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
// --- Shared helpers ---
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Compute total + per-source cost from raw file content. A structured JSON
|
|
575
|
+
* trace (from `fit-trace download`) carries its total in `summary.totalCostUsd`
|
|
576
|
+
* but no per-source split; raw NDJSON is summed via `sumTraceCost`.
|
|
577
|
+
* @param {string} content - Raw file content (structured JSON or NDJSON).
|
|
578
|
+
* @returns {{totalCostUsd: number, bySource: Record<string, number>}}
|
|
579
|
+
*/
|
|
580
|
+
function computeTraceCost(content) {
|
|
581
|
+
try {
|
|
582
|
+
const parsed = JSON.parse(content);
|
|
583
|
+
if (parsed && typeof parsed.summary?.totalCostUsd === "number") {
|
|
584
|
+
return { totalCostUsd: parsed.summary.totalCostUsd, bySource: {} };
|
|
585
|
+
}
|
|
586
|
+
} catch {
|
|
587
|
+
// Not a single JSON object — treat as NDJSON below.
|
|
588
|
+
}
|
|
589
|
+
return sumTraceCost(content.split("\n"));
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* Load a trace file. Supports structured JSON and raw NDJSON.
|
|
594
|
+
* @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
|
|
595
|
+
* @param {string} file
|
|
596
|
+
* @returns {import("../trace-query.js").TraceQuery}
|
|
597
|
+
*/
|
|
598
|
+
export function loadTrace(runtime, file) {
|
|
599
|
+
const content = runtime.fsSync.readFileSync(file, "utf8");
|
|
600
|
+
|
|
601
|
+
try {
|
|
602
|
+
const parsed = JSON.parse(content);
|
|
603
|
+
if (parsed.turns) {
|
|
604
|
+
return createTraceQuery(parsed);
|
|
605
|
+
}
|
|
606
|
+
} catch {
|
|
607
|
+
// Not valid JSON — fall through to NDJSON.
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
const collector = createTraceCollector({
|
|
611
|
+
now: () => isoTimestamp(runtime.clock.now()),
|
|
612
|
+
});
|
|
613
|
+
for (const line of content.split("\n")) {
|
|
614
|
+
collector.addLine(line);
|
|
615
|
+
}
|
|
616
|
+
return createTraceQuery(collector.toJSON());
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* Write JSON output to stdout. By default strips `thinking.signature`
|
|
621
|
+
* base64 blobs from the payload so they don't dominate terminal output;
|
|
622
|
+
* pass `--signatures` (surfaced as `values.signatures`) to keep them.
|
|
623
|
+
* @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
|
|
624
|
+
* @param {*} data
|
|
625
|
+
* @param {object} [values]
|
|
626
|
+
*/
|
|
627
|
+
function writeJSON(runtime, data, values = {}) {
|
|
628
|
+
const output = values.signatures ? data : stripSignatures(data);
|
|
629
|
+
runtime.proc.stdout.write(JSON.stringify(output, null, 2) + "\n");
|
|
630
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The active work-item tracker selects which column of the work-trackers
|
|
3
|
+
* matrix realizes each coordination operation (see the agent reference
|
|
4
|
+
* `work-trackers.md`). `github` is the production binding; the offline
|
|
5
|
+
* coordination benchmark runs under `filesystem`.
|
|
6
|
+
*/
|
|
7
|
+
export const DEFAULT_WORK_TRACKER = "github";
|
|
8
|
+
|
|
9
|
+
/** Trackers the harness knows how to select. */
|
|
10
|
+
export const KNOWN_WORK_TRACKERS = ["github", "filesystem"];
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Resolve the active work tracker. Precedence: the explicit `--work-tracker`
|
|
14
|
+
* flag, then an inherited `LIBHARNESS_WORK_TRACKER` on the environment (so a CI
|
|
15
|
+
* job or harness can select it without the flag), then the `github` default.
|
|
16
|
+
* The harness writes the result to `LIBHARNESS_WORK_TRACKER` on the agent
|
|
17
|
+
* environment, mirroring `--agent-profile` → `LIBHARNESS_AGENT_PROFILE`.
|
|
18
|
+
* @param {Record<string, string|undefined>} values - Parsed option values
|
|
19
|
+
* @param {Record<string, string|undefined>} [env] - Process environment
|
|
20
|
+
* (e.g. `runtime.proc.env`); read for the `LIBHARNESS_WORK_TRACKER` fallback.
|
|
21
|
+
* @returns {string}
|
|
22
|
+
* @throws {Error} if the resolved tracker is unknown
|
|
23
|
+
*/
|
|
24
|
+
export function resolveWorkTracker(values, env = {}) {
|
|
25
|
+
const tracker =
|
|
26
|
+
values["work-tracker"] ||
|
|
27
|
+
env.LIBHARNESS_WORK_TRACKER ||
|
|
28
|
+
DEFAULT_WORK_TRACKER;
|
|
29
|
+
if (!KNOWN_WORK_TRACKERS.includes(tracker)) {
|
|
30
|
+
throw new Error(
|
|
31
|
+
`unknown work tracker '${tracker}'; expected one of: ${KNOWN_WORK_TRACKERS.join(", ")}`,
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
return tracker;
|
|
35
|
+
}
|