@forwardimpact/libharness 0.1.22 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -201
- package/README.md +196 -80
- package/bin/fit-benchmark.js +44 -0
- package/bin/fit-harness.js +358 -0
- package/bin/fit-selfedit.js +165 -0
- package/bin/fit-trace.js +510 -0
- package/package.json +41 -11
- package/src/agent-runner.js +256 -0
- package/src/benchmark/apm-installer.js +207 -0
- package/src/benchmark/env-loader.js +158 -0
- package/src/benchmark/hook-env.js +40 -0
- package/src/benchmark/invariants.js +141 -0
- package/src/benchmark/judge.js +187 -0
- package/src/benchmark/npm-installer.js +87 -0
- package/src/benchmark/report.js +604 -0
- package/src/benchmark/result.js +127 -0
- package/src/benchmark/runner.js +688 -0
- package/src/benchmark/scheduler.js +78 -0
- package/src/benchmark/task-family.js +260 -0
- package/src/benchmark/workdir.js +344 -0
- package/src/commands/assert.js +153 -0
- package/src/commands/benchmark-definition.js +175 -0
- package/src/commands/benchmark-invariants.js +73 -0
- package/src/commands/benchmark-report.js +51 -0
- package/src/commands/benchmark-run.js +175 -0
- package/src/commands/by-discussion.js +94 -0
- package/src/commands/callback.js +119 -0
- package/src/commands/discuss.js +132 -0
- package/src/commands/facilitate.js +123 -0
- package/src/commands/output.js +36 -0
- package/src/commands/run.js +152 -0
- package/src/commands/supervise.js +136 -0
- package/src/commands/task-input.js +54 -0
- package/src/commands/tee.js +53 -0
- package/src/commands/trace.js +630 -0
- package/src/commands/work-tracker.js +35 -0
- package/src/cost.js +79 -0
- package/src/discuss-tools.js +173 -0
- package/src/discusser.js +394 -0
- package/src/events/github.js +161 -0
- package/src/facilitator.js +205 -0
- package/src/inbox-poller.js +81 -0
- package/src/index.js +72 -2
- package/src/judge.js +210 -0
- package/src/message-bus.js +118 -0
- package/src/orchestration-loop.js +330 -0
- package/src/orchestration-toolkit.js +441 -0
- package/src/orchestrator-helpers.js +23 -0
- package/src/profile-prompt.js +266 -0
- package/src/redaction.js +253 -0
- package/src/render/line-renderer.js +54 -0
- package/src/render/orchestrator-filter.js +19 -0
- package/src/render/palette.js +63 -0
- package/src/render/tool-hints.js +154 -0
- package/src/render/turn-renderer.js +96 -0
- package/src/reply-emitter.js +47 -0
- package/src/sequence-counter.js +21 -0
- package/src/signature-filter.js +27 -0
- package/src/supervisor.js +236 -0
- package/src/tee-writer.js +150 -0
- package/src/trace-collector.js +444 -0
- package/src/trace-github.js +473 -0
- package/src/trace-multi.js +101 -0
- package/src/trace-query.js +748 -0
- package/src/trace-render.js +211 -0
- package/src/trace-usage.js +249 -0
- package/src/fixture/assertions.js +0 -42
- package/src/fixture/cache.js +0 -50
- package/src/fixture/eval.js +0 -146
- package/src/fixture/index.js +0 -9
- package/src/fixture/pathway.js +0 -451
- package/src/fixture/services.js +0 -56
- package/src/mock/clients.js +0 -135
- package/src/mock/config.js +0 -45
- package/src/mock/data.js +0 -46
- package/src/mock/fs.js +0 -111
- package/src/mock/grpc.js +0 -94
- package/src/mock/http.js +0 -60
- package/src/mock/index.js +0 -36
- package/src/mock/infra.js +0 -219
- package/src/mock/logger.js +0 -42
- package/src/mock/observer.js +0 -74
- package/src/mock/resource-index.js +0 -95
- package/src/mock/service-callbacks.js +0 -39
- package/src/mock/services.js +0 -79
- package/src/mock/spy.js +0 -44
- package/src/mock/storage.js +0 -118
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { resolve, join } from "node:path";
|
|
2
|
+
import { isoTimestamp } from "@forwardimpact/libutil";
|
|
3
|
+
import { createSupervisor } from "../supervisor.js";
|
|
4
|
+
import { createRedactor } from "../redaction.js";
|
|
5
|
+
import { createTeeWriter } from "../tee-writer.js";
|
|
6
|
+
import { resolveTaskContent } from "./task-input.js";
|
|
7
|
+
import { resolveWorkTracker } from "./work-tracker.js";
|
|
8
|
+
import { createServiceConfig } from "@forwardimpact/libconfig";
|
|
9
|
+
import { AGENT_MODEL, LEAD_MODEL } from "@forwardimpact/libutil/models";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Parse all supervise flags from parsed values into an options object.
|
|
13
|
+
* @param {object} values - Parsed option values from cli.parse()
|
|
14
|
+
* @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
|
|
15
|
+
* @returns {Promise<object>}
|
|
16
|
+
*/
|
|
17
|
+
export async function parseSuperviseOptions(values, runtime) {
|
|
18
|
+
const { task: taskContent, amend: taskAmend } = resolveTaskContent(
|
|
19
|
+
values,
|
|
20
|
+
runtime,
|
|
21
|
+
);
|
|
22
|
+
const supervisorAllowedToolsRaw = values["supervisor-allowed-tools"];
|
|
23
|
+
|
|
24
|
+
const tmpRoot = runtime.proc.env.TMPDIR ?? "/tmp";
|
|
25
|
+
const agentCwd = resolve(
|
|
26
|
+
values["agent-cwd"] ??
|
|
27
|
+
(await runtime.fs.mkdtemp(join(tmpRoot, "fit-harness-agent-"))),
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
return {
|
|
31
|
+
taskContent,
|
|
32
|
+
taskAmend,
|
|
33
|
+
supervisorCwd: resolve(values["supervisor-cwd"] ?? "."),
|
|
34
|
+
agentCwd,
|
|
35
|
+
agentModel: values["agent-model"] || AGENT_MODEL,
|
|
36
|
+
supervisorModel: values["lead-model"] || LEAD_MODEL,
|
|
37
|
+
maxTurns: (() => {
|
|
38
|
+
const raw = values["max-turns"] ?? "200";
|
|
39
|
+
return raw === "0" ? 0 : parseInt(raw, 10);
|
|
40
|
+
})(),
|
|
41
|
+
outputPath: values.output,
|
|
42
|
+
supervisorProfile: values["lead-profile"] ?? undefined,
|
|
43
|
+
agentProfile: values["agent-profile"] ?? undefined,
|
|
44
|
+
workTracker: resolveWorkTracker(values, runtime?.proc?.env),
|
|
45
|
+
allowedTools: (
|
|
46
|
+
values["allowed-tools"] ??
|
|
47
|
+
"Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite"
|
|
48
|
+
).split(","),
|
|
49
|
+
supervisorAllowedTools: supervisorAllowedToolsRaw
|
|
50
|
+
? supervisorAllowedToolsRaw.split(",")
|
|
51
|
+
: undefined,
|
|
52
|
+
mcpServer: values["mcp-server"] ?? undefined,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Supervise command — run one agent under a supervisor via the
|
|
58
|
+
* orchestration loop. The supervisor delegates work through Ask, sees
|
|
59
|
+
* each reply on its next turn, and ends with Conclude.
|
|
60
|
+
*
|
|
61
|
+
* Usage: fit-harness supervise [options]
|
|
62
|
+
*
|
|
63
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
64
|
+
* @returns {Promise<{ok: boolean, code?: number, error?: string}>}
|
|
65
|
+
*/
|
|
66
|
+
export async function runSuperviseCommand(ctx) {
|
|
67
|
+
const runtime = ctx.deps.runtime;
|
|
68
|
+
const opts = await parseSuperviseOptions(ctx.options, runtime);
|
|
69
|
+
|
|
70
|
+
// Build the redactor as the first observable side-effect after option
|
|
71
|
+
// parsing — the env snapshot must freeze BEFORE any in-process
|
|
72
|
+
// env writes the command performs (e.g. LIBHARNESS_AGENT_PROFILE).
|
|
73
|
+
const redactor = createRedactor({ runtime });
|
|
74
|
+
|
|
75
|
+
// When --output is specified, stream text to stdout while writing NDJSON to file.
|
|
76
|
+
// Otherwise, write NDJSON directly to stdout (backwards-compatible).
|
|
77
|
+
const fileStream = opts.outputPath
|
|
78
|
+
? runtime.fs.createWriteStream(opts.outputPath)
|
|
79
|
+
: null;
|
|
80
|
+
const output = fileStream
|
|
81
|
+
? createTeeWriter({
|
|
82
|
+
fileStream,
|
|
83
|
+
textStream: runtime.proc.stdout,
|
|
84
|
+
mode: "supervised",
|
|
85
|
+
now: () => isoTimestamp(runtime.clock.now()),
|
|
86
|
+
})
|
|
87
|
+
: runtime.proc.stdout;
|
|
88
|
+
|
|
89
|
+
let agentMcpServers = null;
|
|
90
|
+
if (opts.mcpServer) {
|
|
91
|
+
const mcpConfig = await createServiceConfig("mcp");
|
|
92
|
+
agentMcpServers = {
|
|
93
|
+
[opts.mcpServer]: {
|
|
94
|
+
type: "http",
|
|
95
|
+
url: mcpConfig.url,
|
|
96
|
+
headers: { Authorization: `Bearer ${mcpConfig.mcpToken()}` },
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
opts.allowedTools.push(`mcp__${opts.mcpServer}__*`);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
if (opts.agentProfile) {
|
|
103
|
+
runtime.proc.env.LIBHARNESS_AGENT_PROFILE = opts.agentProfile;
|
|
104
|
+
}
|
|
105
|
+
// Unconditional so the default "github" is observable to the agent's
|
|
106
|
+
// active-tracker resolution, mirroring --agent-profile's env write above.
|
|
107
|
+
runtime.proc.env.LIBHARNESS_WORK_TRACKER = opts.workTracker;
|
|
108
|
+
|
|
109
|
+
const { query } = await import("@anthropic-ai/claude-agent-sdk");
|
|
110
|
+
const supervisor = createSupervisor({
|
|
111
|
+
supervisorCwd: opts.supervisorCwd,
|
|
112
|
+
agentCwd: opts.agentCwd,
|
|
113
|
+
query,
|
|
114
|
+
output,
|
|
115
|
+
agentModel: opts.agentModel,
|
|
116
|
+
supervisorModel: opts.supervisorModel,
|
|
117
|
+
maxTurns: opts.maxTurns,
|
|
118
|
+
allowedTools: opts.allowedTools,
|
|
119
|
+
supervisorAllowedTools: opts.supervisorAllowedTools,
|
|
120
|
+
supervisorProfile: opts.supervisorProfile,
|
|
121
|
+
agentProfile: opts.agentProfile,
|
|
122
|
+
taskAmend: opts.taskAmend,
|
|
123
|
+
agentMcpServers,
|
|
124
|
+
redactor,
|
|
125
|
+
runtime,
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
const result = await supervisor.run(opts.taskContent);
|
|
129
|
+
|
|
130
|
+
if (fileStream) {
|
|
131
|
+
await new Promise((r) => output.end(r));
|
|
132
|
+
await new Promise((r) => fileStream.end(r));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
return result.success ? { ok: true } : { ok: false, code: 1, error: "" };
|
|
136
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { composeTaskFromGitHubEvent } from "../events/github.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Resolve `--task-file` / `--task-text` / `--task-event` into the task pair the
|
|
5
|
+
* runner consumes. Exactly one of the three must be set. For `--task-event`,
|
|
6
|
+
* libharness reads the event payload and extracts both the main task (from the
|
|
7
|
+
* template that matches `$GITHUB_EVENT_NAME` + `payload.action`) and the
|
|
8
|
+
* amendment (from `payload.inputs?.prompt`) — so the workflow doesn't need to
|
|
9
|
+
* wire `--task-amend` separately. For the other two modes, `--task-amend`
|
|
10
|
+
* works as before.
|
|
11
|
+
*
|
|
12
|
+
* @param {object} values - Parsed option values from cli.parse()
|
|
13
|
+
* @param {import("@forwardimpact/libutil/runtime").Runtime} runtime - Ambient
|
|
14
|
+
* collaborators; `fsSync.readFileSync` loads `--task-file`/`--task-event`
|
|
15
|
+
* and `proc.env` resolves `GITHUB_EVENT_NAME`.
|
|
16
|
+
* @returns {{ task: string, amend: string | undefined }}
|
|
17
|
+
*/
|
|
18
|
+
export function resolveTaskContent(values, runtime) {
|
|
19
|
+
const taskFile = values["task-file"];
|
|
20
|
+
const taskText = values["task-text"];
|
|
21
|
+
const taskEvent = values["task-event"];
|
|
22
|
+
|
|
23
|
+
const set = [taskFile, taskText, taskEvent].filter(Boolean).length;
|
|
24
|
+
if (set === 0) {
|
|
25
|
+
throw new Error(
|
|
26
|
+
"one of --task-file, --task-text, --task-event is required",
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
if (set > 1) {
|
|
30
|
+
throw new Error(
|
|
31
|
+
"--task-file, --task-text, --task-event are mutually exclusive",
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const amendFlag = values["task-amend"] ?? undefined;
|
|
36
|
+
|
|
37
|
+
if (taskFile) {
|
|
38
|
+
return {
|
|
39
|
+
task: runtime.fsSync.readFileSync(taskFile, "utf8"),
|
|
40
|
+
amend: amendFlag,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
if (taskText) {
|
|
44
|
+
return { task: taskText, amend: amendFlag };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const eventName = runtime.proc.env.GITHUB_EVENT_NAME;
|
|
48
|
+
if (!eventName) {
|
|
49
|
+
throw new Error("--task-event requires GITHUB_EVENT_NAME to be set");
|
|
50
|
+
}
|
|
51
|
+
const payload = JSON.parse(runtime.fsSync.readFileSync(taskEvent, "utf8"));
|
|
52
|
+
const composed = composeTaskFromGitHubEvent(payload, eventName);
|
|
53
|
+
return { task: composed.task, amend: amendFlag ?? composed.amend };
|
|
54
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { PassThrough } from "node:stream";
|
|
2
|
+
import { pipeline } from "node:stream/promises";
|
|
3
|
+
import { isoTimestamp } from "@forwardimpact/libutil";
|
|
4
|
+
import { createTeeWriter } from "../tee-writer.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Tee command — stream text output to stdout while optionally saving the raw
|
|
8
|
+
* NDJSON to a file. Reads stdin line-by-line through the injected runtime and
|
|
9
|
+
* re-delimits each record with a newline so the TeeWriter's line splitter sees
|
|
10
|
+
* the same framing the raw byte stream produced.
|
|
11
|
+
*
|
|
12
|
+
* Usage: fit-harness tee [output.ndjson] < trace.ndjson
|
|
13
|
+
*
|
|
14
|
+
* @param {import("@forwardimpact/libcli").InvocationContext} ctx
|
|
15
|
+
* @returns {Promise<{ok: boolean, code?: number, error?: string}>}
|
|
16
|
+
*/
|
|
17
|
+
export async function runTeeCommand(ctx) {
|
|
18
|
+
const runtime = ctx.deps.runtime;
|
|
19
|
+
const outputPath = ctx.args.output ?? null;
|
|
20
|
+
const fileStream = outputPath
|
|
21
|
+
? runtime.fs.createWriteStream(outputPath)
|
|
22
|
+
: null;
|
|
23
|
+
|
|
24
|
+
// TeeWriter requires a fileStream; when no output file is specified,
|
|
25
|
+
// use a PassThrough as a no-op sink (NDJSON is not saved).
|
|
26
|
+
const sink = fileStream ?? new PassThrough();
|
|
27
|
+
const tee = createTeeWriter({
|
|
28
|
+
fileStream: sink,
|
|
29
|
+
textStream: runtime.proc.stdout,
|
|
30
|
+
mode: "raw",
|
|
31
|
+
now: () => isoTimestamp(runtime.clock.now()),
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
try {
|
|
35
|
+
// `runtime.proc.stdin` yields newline-stripped lines; re-append `\n` so the
|
|
36
|
+
// TeeWriter's `_write` line splitter frames records exactly as it did when
|
|
37
|
+
// piped the raw byte stream.
|
|
38
|
+
const lines = (async function* () {
|
|
39
|
+
for await (const line of runtime.proc.stdin) yield `${line}\n`;
|
|
40
|
+
})();
|
|
41
|
+
await pipeline(lines, tee);
|
|
42
|
+
return { ok: true };
|
|
43
|
+
} catch (error) {
|
|
44
|
+
return { ok: false, code: 1, error: error.message };
|
|
45
|
+
} finally {
|
|
46
|
+
if (fileStream) {
|
|
47
|
+
await new Promise((resolve, reject) => {
|
|
48
|
+
fileStream.end(() => resolve());
|
|
49
|
+
fileStream.on("error", reject);
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|