@forwardimpact/libharness 0.1.22 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/LICENSE +21 -201
  2. package/README.md +196 -80
  3. package/bin/fit-benchmark.js +44 -0
  4. package/bin/fit-harness.js +358 -0
  5. package/bin/fit-selfedit.js +165 -0
  6. package/bin/fit-trace.js +510 -0
  7. package/package.json +41 -11
  8. package/src/agent-runner.js +256 -0
  9. package/src/benchmark/apm-installer.js +207 -0
  10. package/src/benchmark/env-loader.js +158 -0
  11. package/src/benchmark/hook-env.js +40 -0
  12. package/src/benchmark/invariants.js +141 -0
  13. package/src/benchmark/judge.js +187 -0
  14. package/src/benchmark/npm-installer.js +87 -0
  15. package/src/benchmark/report.js +604 -0
  16. package/src/benchmark/result.js +127 -0
  17. package/src/benchmark/runner.js +688 -0
  18. package/src/benchmark/scheduler.js +78 -0
  19. package/src/benchmark/task-family.js +260 -0
  20. package/src/benchmark/workdir.js +344 -0
  21. package/src/commands/assert.js +153 -0
  22. package/src/commands/benchmark-definition.js +175 -0
  23. package/src/commands/benchmark-invariants.js +73 -0
  24. package/src/commands/benchmark-report.js +51 -0
  25. package/src/commands/benchmark-run.js +175 -0
  26. package/src/commands/by-discussion.js +94 -0
  27. package/src/commands/callback.js +119 -0
  28. package/src/commands/discuss.js +132 -0
  29. package/src/commands/facilitate.js +123 -0
  30. package/src/commands/output.js +36 -0
  31. package/src/commands/run.js +152 -0
  32. package/src/commands/supervise.js +136 -0
  33. package/src/commands/task-input.js +54 -0
  34. package/src/commands/tee.js +53 -0
  35. package/src/commands/trace.js +630 -0
  36. package/src/commands/work-tracker.js +35 -0
  37. package/src/cost.js +79 -0
  38. package/src/discuss-tools.js +173 -0
  39. package/src/discusser.js +394 -0
  40. package/src/events/github.js +161 -0
  41. package/src/facilitator.js +205 -0
  42. package/src/inbox-poller.js +81 -0
  43. package/src/index.js +72 -2
  44. package/src/judge.js +210 -0
  45. package/src/message-bus.js +118 -0
  46. package/src/orchestration-loop.js +330 -0
  47. package/src/orchestration-toolkit.js +441 -0
  48. package/src/orchestrator-helpers.js +23 -0
  49. package/src/profile-prompt.js +266 -0
  50. package/src/redaction.js +253 -0
  51. package/src/render/line-renderer.js +54 -0
  52. package/src/render/orchestrator-filter.js +19 -0
  53. package/src/render/palette.js +63 -0
  54. package/src/render/tool-hints.js +154 -0
  55. package/src/render/turn-renderer.js +96 -0
  56. package/src/reply-emitter.js +47 -0
  57. package/src/sequence-counter.js +21 -0
  58. package/src/signature-filter.js +27 -0
  59. package/src/supervisor.js +236 -0
  60. package/src/tee-writer.js +150 -0
  61. package/src/trace-collector.js +444 -0
  62. package/src/trace-github.js +473 -0
  63. package/src/trace-multi.js +101 -0
  64. package/src/trace-query.js +748 -0
  65. package/src/trace-render.js +211 -0
  66. package/src/trace-usage.js +249 -0
  67. package/src/fixture/assertions.js +0 -42
  68. package/src/fixture/cache.js +0 -50
  69. package/src/fixture/eval.js +0 -146
  70. package/src/fixture/index.js +0 -9
  71. package/src/fixture/pathway.js +0 -451
  72. package/src/fixture/services.js +0 -56
  73. package/src/mock/clients.js +0 -135
  74. package/src/mock/config.js +0 -45
  75. package/src/mock/data.js +0 -46
  76. package/src/mock/fs.js +0 -111
  77. package/src/mock/grpc.js +0 -94
  78. package/src/mock/http.js +0 -60
  79. package/src/mock/index.js +0 -36
  80. package/src/mock/infra.js +0 -219
  81. package/src/mock/logger.js +0 -42
  82. package/src/mock/observer.js +0 -74
  83. package/src/mock/resource-index.js +0 -95
  84. package/src/mock/service-callbacks.js +0 -39
  85. package/src/mock/services.js +0 -79
  86. package/src/mock/spy.js +0 -44
  87. package/src/mock/storage.js +0 -118
@@ -0,0 +1,136 @@
1
+ import { resolve, join } from "node:path";
2
+ import { isoTimestamp } from "@forwardimpact/libutil";
3
+ import { createSupervisor } from "../supervisor.js";
4
+ import { createRedactor } from "../redaction.js";
5
+ import { createTeeWriter } from "../tee-writer.js";
6
+ import { resolveTaskContent } from "./task-input.js";
7
+ import { resolveWorkTracker } from "./work-tracker.js";
8
+ import { createServiceConfig } from "@forwardimpact/libconfig";
9
+ import { AGENT_MODEL, LEAD_MODEL } from "@forwardimpact/libutil/models";
10
+
11
+ /**
12
+ * Parse all supervise flags from parsed values into an options object.
13
+ * @param {object} values - Parsed option values from cli.parse()
14
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
15
+ * @returns {Promise<object>}
16
+ */
17
+ export async function parseSuperviseOptions(values, runtime) {
18
+ const { task: taskContent, amend: taskAmend } = resolveTaskContent(
19
+ values,
20
+ runtime,
21
+ );
22
+ const supervisorAllowedToolsRaw = values["supervisor-allowed-tools"];
23
+
24
+ const tmpRoot = runtime.proc.env.TMPDIR ?? "/tmp";
25
+ const agentCwd = resolve(
26
+ values["agent-cwd"] ??
27
+ (await runtime.fs.mkdtemp(join(tmpRoot, "fit-harness-agent-"))),
28
+ );
29
+
30
+ return {
31
+ taskContent,
32
+ taskAmend,
33
+ supervisorCwd: resolve(values["supervisor-cwd"] ?? "."),
34
+ agentCwd,
35
+ agentModel: values["agent-model"] || AGENT_MODEL,
36
+ supervisorModel: values["lead-model"] || LEAD_MODEL,
37
+ maxTurns: (() => {
38
+ const raw = values["max-turns"] ?? "200";
39
+ return raw === "0" ? 0 : parseInt(raw, 10);
40
+ })(),
41
+ outputPath: values.output,
42
+ supervisorProfile: values["lead-profile"] ?? undefined,
43
+ agentProfile: values["agent-profile"] ?? undefined,
44
+ workTracker: resolveWorkTracker(values, runtime?.proc?.env),
45
+ allowedTools: (
46
+ values["allowed-tools"] ??
47
+ "Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite"
48
+ ).split(","),
49
+ supervisorAllowedTools: supervisorAllowedToolsRaw
50
+ ? supervisorAllowedToolsRaw.split(",")
51
+ : undefined,
52
+ mcpServer: values["mcp-server"] ?? undefined,
53
+ };
54
+ }
55
+
56
+ /**
57
+ * Supervise command — run one agent under a supervisor via the
58
+ * orchestration loop. The supervisor delegates work through Ask, sees
59
+ * each reply on its next turn, and ends with Conclude.
60
+ *
61
+ * Usage: fit-harness supervise [options]
62
+ *
63
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
64
+ * @returns {Promise<{ok: boolean, code?: number, error?: string}>}
65
+ */
66
+ export async function runSuperviseCommand(ctx) {
67
+ const runtime = ctx.deps.runtime;
68
+ const opts = await parseSuperviseOptions(ctx.options, runtime);
69
+
70
+ // Build the redactor as the first observable side-effect after option
71
+ // parsing — the env snapshot must freeze BEFORE any in-process
72
+ // env writes the command performs (e.g. LIBHARNESS_AGENT_PROFILE).
73
+ const redactor = createRedactor({ runtime });
74
+
75
+ // When --output is specified, stream text to stdout while writing NDJSON to file.
76
+ // Otherwise, write NDJSON directly to stdout (backwards-compatible).
77
+ const fileStream = opts.outputPath
78
+ ? runtime.fs.createWriteStream(opts.outputPath)
79
+ : null;
80
+ const output = fileStream
81
+ ? createTeeWriter({
82
+ fileStream,
83
+ textStream: runtime.proc.stdout,
84
+ mode: "supervised",
85
+ now: () => isoTimestamp(runtime.clock.now()),
86
+ })
87
+ : runtime.proc.stdout;
88
+
89
+ let agentMcpServers = null;
90
+ if (opts.mcpServer) {
91
+ const mcpConfig = await createServiceConfig("mcp");
92
+ agentMcpServers = {
93
+ [opts.mcpServer]: {
94
+ type: "http",
95
+ url: mcpConfig.url,
96
+ headers: { Authorization: `Bearer ${mcpConfig.mcpToken()}` },
97
+ },
98
+ };
99
+ opts.allowedTools.push(`mcp__${opts.mcpServer}__*`);
100
+ }
101
+
102
+ if (opts.agentProfile) {
103
+ runtime.proc.env.LIBHARNESS_AGENT_PROFILE = opts.agentProfile;
104
+ }
105
+ // Unconditional so the default "github" is observable to the agent's
106
+ // active-tracker resolution, mirroring --agent-profile's env write above.
107
+ runtime.proc.env.LIBHARNESS_WORK_TRACKER = opts.workTracker;
108
+
109
+ const { query } = await import("@anthropic-ai/claude-agent-sdk");
110
+ const supervisor = createSupervisor({
111
+ supervisorCwd: opts.supervisorCwd,
112
+ agentCwd: opts.agentCwd,
113
+ query,
114
+ output,
115
+ agentModel: opts.agentModel,
116
+ supervisorModel: opts.supervisorModel,
117
+ maxTurns: opts.maxTurns,
118
+ allowedTools: opts.allowedTools,
119
+ supervisorAllowedTools: opts.supervisorAllowedTools,
120
+ supervisorProfile: opts.supervisorProfile,
121
+ agentProfile: opts.agentProfile,
122
+ taskAmend: opts.taskAmend,
123
+ agentMcpServers,
124
+ redactor,
125
+ runtime,
126
+ });
127
+
128
+ const result = await supervisor.run(opts.taskContent);
129
+
130
+ if (fileStream) {
131
+ await new Promise((r) => output.end(r));
132
+ await new Promise((r) => fileStream.end(r));
133
+ }
134
+
135
+ return result.success ? { ok: true } : { ok: false, code: 1, error: "" };
136
+ }
@@ -0,0 +1,54 @@
1
+ import { composeTaskFromGitHubEvent } from "../events/github.js";
2
+
3
+ /**
4
+ * Resolve `--task-file` / `--task-text` / `--task-event` into the task pair the
5
+ * runner consumes. Exactly one of the three must be set. For `--task-event`,
6
+ * libharness reads the event payload and extracts both the main task (from the
7
+ * template that matches `$GITHUB_EVENT_NAME` + `payload.action`) and the
8
+ * amendment (from `payload.inputs?.prompt`) — so the workflow doesn't need to
9
+ * wire `--task-amend` separately. For the other two modes, `--task-amend`
10
+ * works as before.
11
+ *
12
+ * @param {object} values - Parsed option values from cli.parse()
13
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime - Ambient
14
+ * collaborators; `fsSync.readFileSync` loads `--task-file`/`--task-event`
15
+ * and `proc.env` resolves `GITHUB_EVENT_NAME`.
16
+ * @returns {{ task: string, amend: string | undefined }}
17
+ */
18
+ export function resolveTaskContent(values, runtime) {
19
+ const taskFile = values["task-file"];
20
+ const taskText = values["task-text"];
21
+ const taskEvent = values["task-event"];
22
+
23
+ const set = [taskFile, taskText, taskEvent].filter(Boolean).length;
24
+ if (set === 0) {
25
+ throw new Error(
26
+ "one of --task-file, --task-text, --task-event is required",
27
+ );
28
+ }
29
+ if (set > 1) {
30
+ throw new Error(
31
+ "--task-file, --task-text, --task-event are mutually exclusive",
32
+ );
33
+ }
34
+
35
+ const amendFlag = values["task-amend"] ?? undefined;
36
+
37
+ if (taskFile) {
38
+ return {
39
+ task: runtime.fsSync.readFileSync(taskFile, "utf8"),
40
+ amend: amendFlag,
41
+ };
42
+ }
43
+ if (taskText) {
44
+ return { task: taskText, amend: amendFlag };
45
+ }
46
+
47
+ const eventName = runtime.proc.env.GITHUB_EVENT_NAME;
48
+ if (!eventName) {
49
+ throw new Error("--task-event requires GITHUB_EVENT_NAME to be set");
50
+ }
51
+ const payload = JSON.parse(runtime.fsSync.readFileSync(taskEvent, "utf8"));
52
+ const composed = composeTaskFromGitHubEvent(payload, eventName);
53
+ return { task: composed.task, amend: amendFlag ?? composed.amend };
54
+ }
@@ -0,0 +1,53 @@
1
+ import { PassThrough } from "node:stream";
2
+ import { pipeline } from "node:stream/promises";
3
+ import { isoTimestamp } from "@forwardimpact/libutil";
4
+ import { createTeeWriter } from "../tee-writer.js";
5
+
6
+ /**
7
+ * Tee command — stream text output to stdout while optionally saving the raw
8
+ * NDJSON to a file. Reads stdin line-by-line through the injected runtime and
9
+ * re-delimits each record with a newline so the TeeWriter's line splitter sees
10
+ * the same framing the raw byte stream produced.
11
+ *
12
+ * Usage: fit-harness tee [output.ndjson] < trace.ndjson
13
+ *
14
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
15
+ * @returns {Promise<{ok: boolean, code?: number, error?: string}>}
16
+ */
17
+ export async function runTeeCommand(ctx) {
18
+ const runtime = ctx.deps.runtime;
19
+ const outputPath = ctx.args.output ?? null;
20
+ const fileStream = outputPath
21
+ ? runtime.fs.createWriteStream(outputPath)
22
+ : null;
23
+
24
+ // TeeWriter requires a fileStream; when no output file is specified,
25
+ // use a PassThrough as a no-op sink (NDJSON is not saved).
26
+ const sink = fileStream ?? new PassThrough();
27
+ const tee = createTeeWriter({
28
+ fileStream: sink,
29
+ textStream: runtime.proc.stdout,
30
+ mode: "raw",
31
+ now: () => isoTimestamp(runtime.clock.now()),
32
+ });
33
+
34
+ try {
35
+ // `runtime.proc.stdin` yields newline-stripped lines; re-append `\n` so the
36
+ // TeeWriter's `_write` line splitter frames records exactly as it did when
37
+ // piped the raw byte stream.
38
+ const lines = (async function* () {
39
+ for await (const line of runtime.proc.stdin) yield `${line}\n`;
40
+ })();
41
+ await pipeline(lines, tee);
42
+ return { ok: true };
43
+ } catch (error) {
44
+ return { ok: false, code: 1, error: error.message };
45
+ } finally {
46
+ if (fileStream) {
47
+ await new Promise((resolve, reject) => {
48
+ fileStream.end(() => resolve());
49
+ fileStream.on("error", reject);
50
+ });
51
+ }
52
+ }
53
+ }