@forwardimpact/libharness 3.0.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -57
- package/package.json +2 -2
- package/src/advisor.js +47 -41
- package/src/agent-runner.js +57 -47
- package/src/benchmark/apm-installer.js +28 -28
- package/src/benchmark/env-loader.js +24 -16
- package/src/benchmark/grade.js +44 -41
- package/src/benchmark/hidden-tests.js +25 -24
- package/src/benchmark/hook-env.js +11 -9
- package/src/benchmark/invariants.js +20 -17
- package/src/benchmark/judge.js +29 -28
- package/src/benchmark/npm-installer.js +9 -8
- package/src/benchmark/report.js +53 -50
- package/src/benchmark/result.js +24 -23
- package/src/benchmark/runner.js +75 -69
- package/src/benchmark/scheduler.js +17 -16
- package/src/benchmark/task-family.js +28 -26
- package/src/benchmark/trace-split.js +8 -7
- package/src/benchmark/workdir.js +27 -25
- package/src/claude-code-executable.js +11 -11
- package/src/commands/advisor-flags.js +8 -7
- package/src/commands/assert.js +16 -15
- package/src/commands/benchmark-definition.js +11 -11
- package/src/commands/benchmark-grade.js +12 -11
- package/src/commands/benchmark-report.js +5 -5
- package/src/commands/benchmark-run.js +31 -28
- package/src/commands/by-discussion.js +10 -10
- package/src/commands/callback.js +11 -11
- package/src/commands/discuss.js +8 -7
- package/src/commands/facilitate.js +15 -13
- package/src/commands/output.js +3 -2
- package/src/commands/run.js +14 -14
- package/src/commands/scan-logs.js +21 -19
- package/src/commands/selfedit.js +14 -14
- package/src/commands/supervise.js +11 -9
- package/src/commands/task-input.js +9 -9
- package/src/commands/tee.js +10 -9
- package/src/commands/trace.js +55 -42
- package/src/commands/work-tracker.js +4 -3
- package/src/cost.js +17 -17
- package/src/discuss-tools.js +16 -16
- package/src/discusser.js +39 -38
- package/src/events/github.js +54 -37
- package/src/facilitator.js +21 -21
- package/src/inbox-poller.js +4 -4
- package/src/judge.js +32 -30
- package/src/message-bus.js +12 -11
- package/src/orchestration-loop.js +35 -36
- package/src/orchestration-toolkit.js +58 -53
- package/src/orchestrator-helpers.js +2 -2
- package/src/profile-prompt.js +54 -53
- package/src/redaction.js +63 -57
- package/src/render/line-renderer.js +5 -5
- package/src/render/orchestrator-filter.js +3 -3
- package/src/render/palette.js +11 -9
- package/src/render/tool-hints.js +18 -15
- package/src/render/turn-renderer.js +4 -4
- package/src/reply-emitter.js +2 -2
- package/src/sequence-counter.js +4 -3
- package/src/signature-filter.js +7 -6
- package/src/supervisor.js +19 -18
- package/src/tee-writer.js +25 -25
- package/src/trace-collector.js +53 -48
- package/src/trace-github.js +53 -44
- package/src/trace-multi.js +15 -13
- package/src/trace-query.js +61 -52
- package/src/trace-render.js +18 -18
- package/src/trace-usage.js +31 -28
- package/src/transcript-recorder.js +24 -20
package/src/sequence-counter.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SequenceCounter — global monotonic counter
|
|
3
|
-
*
|
|
2
|
+
* SequenceCounter — global monotonic counter. All participants in a session
|
|
3
|
+
* share one counter. Single-threaded JS means the counter needs no
|
|
4
|
+
* synchronization.
|
|
4
5
|
*/
|
|
5
6
|
/** Monotonic counter that assigns globally ordered sequence numbers within a session. */
|
|
6
7
|
export class SequenceCounter {
|
|
@@ -15,7 +16,7 @@ export class SequenceCounter {
|
|
|
15
16
|
}
|
|
16
17
|
}
|
|
17
18
|
|
|
18
|
-
/** Create a new SequenceCounter
|
|
19
|
+
/** Create a new SequenceCounter that starts at zero. */
|
|
19
20
|
export function createSequenceCounter() {
|
|
20
21
|
return new SequenceCounter();
|
|
21
22
|
}
|
package/src/signature-filter.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Strip `thinking.signature` base64 blobs from a JSON-serializable value.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* signatures intact (lossless storage)
|
|
6
|
-
* by default because they dominate output
|
|
4
|
+
* The CLI applies this filter at the output boundary. The stored structured
|
|
5
|
+
* trace keeps signatures intact (lossless storage). The display filter drops
|
|
6
|
+
* them by default, because they dominate the output and do not help
|
|
7
|
+
* analysis.
|
|
7
8
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* any other type
|
|
9
|
+
* This function walks the input recursively. For any object whose
|
|
10
|
+
* `type === "thinking"`, it copies the object and then removes the
|
|
11
|
+
* `signature` field. It keeps signatures on objects of any other type.
|
|
11
12
|
*
|
|
12
13
|
* @param {*} value - Any JSON-serializable value
|
|
13
14
|
* @returns {*} A deep-copy with thinking signatures removed
|
package/src/supervisor.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Supervisor — supervise-mode wrapper around `OrchestrationLoop`.
|
|
3
|
-
*
|
|
4
|
-
* (`"
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* Supervisor — supervise-mode wrapper around `OrchestrationLoop`. A lead
|
|
3
|
+
* participant (`"supervisor"`) coordinates one named participant
|
|
4
|
+
* (`"agent"`). The structure is the same as `Facilitator` with a single
|
|
5
|
+
* agent. Only the role names, the prompts, and the pass-through accessors
|
|
6
|
+
* differ.
|
|
7
7
|
*
|
|
8
|
-
* Ask is async
|
|
9
|
-
* `{askIds:[N]}` immediately
|
|
8
|
+
* Ask is async, with the same contract as facilitate and discuss. It
|
|
9
|
+
* returns `{askIds:[N]}` immediately. The agent's reply arrives on the
|
|
10
10
|
* supervisor's next turn as `[answer#N] agent: <text>`. The supervisor
|
|
11
|
-
* sees the agent at each Ask boundary
|
|
11
|
+
* sees the agent at each Ask boundary. It plans the next step. It
|
|
12
12
|
* eventually calls Conclude.
|
|
13
13
|
*
|
|
14
14
|
* For tighter feedback loops, size the agent's per-turn budget down
|
|
@@ -34,7 +34,7 @@ import {
|
|
|
34
34
|
} from "./advisor.js";
|
|
35
35
|
import { createTranscriptRecorder } from "./transcript-recorder.js";
|
|
36
36
|
|
|
37
|
-
/** System prompt for the supervisor lead. L0 mechanics only per
|
|
37
|
+
/** System prompt for the supervisor lead. L0 mechanics only per JIDOKA. */
|
|
38
38
|
export const SUPERVISOR_SYSTEM_PROMPT =
|
|
39
39
|
"You supervise one agent.\n" +
|
|
40
40
|
"Use `Ask` to delegate the agent's task to the agent.\n" +
|
|
@@ -42,9 +42,9 @@ export const SUPERVISOR_SYSTEM_PROMPT =
|
|
|
42
42
|
"The reply arrives on your next turn as `[answer#N] agent: <text>` in your inbox.\n" +
|
|
43
43
|
"End your turn while Asks are pending. The system resumes you when an answer arrives.\n" +
|
|
44
44
|
"If the agent goes off-track, send a corrective `Ask`.\n" +
|
|
45
|
-
"
|
|
45
|
+
"Call `Conclude` with a verdict and summary to end every session.";
|
|
46
46
|
|
|
47
|
-
/** System prompt for the supervised agent. L0 mechanics only per
|
|
47
|
+
/** System prompt for the supervised agent. L0 mechanics only per JIDOKA. */
|
|
48
48
|
export const AGENT_SYSTEM_PROMPT =
|
|
49
49
|
"A supervisor directs your work.\n" +
|
|
50
50
|
"Each question arrives as `[ask#N] supervisor: <text>` in your inbox.\n" +
|
|
@@ -54,7 +54,8 @@ export const AGENT_SYSTEM_PROMPT =
|
|
|
54
54
|
|
|
55
55
|
/**
|
|
56
56
|
* Supervise-mode wrapper around `OrchestrationLoop`. The lead is
|
|
57
|
-
* `"supervisor"
|
|
57
|
+
* `"supervisor"`. One participant is `"agent"`. The mode tag is
|
|
58
|
+
* `"supervised"`.
|
|
58
59
|
*/
|
|
59
60
|
export class Supervisor extends OrchestrationLoop {
|
|
60
61
|
/**
|
|
@@ -135,7 +136,7 @@ const devNull = new Writable({
|
|
|
135
136
|
* @param {string} [deps.profilesDir]
|
|
136
137
|
* @param {string} [deps.taskAmend]
|
|
137
138
|
* @param {Record<string, object>} [deps.agentMcpServers]
|
|
138
|
-
* @param {string} [deps.advisorModel] - Claude model for advisor consults
|
|
139
|
+
* @param {string} [deps.advisorModel] - Claude model for advisor consults. When absent, the factory offers no Advisor tool.
|
|
139
140
|
* @param {number} [deps.advisorMaxUses] - Session-wide consult budget (default 3).
|
|
140
141
|
* @returns {Supervisor}
|
|
141
142
|
*/
|
|
@@ -181,9 +182,9 @@ export function createSupervisor({
|
|
|
181
182
|
const perRunBudget = maxTurns ?? 200;
|
|
182
183
|
const abortController = new AbortController();
|
|
183
184
|
|
|
184
|
-
//
|
|
185
|
-
//
|
|
186
|
-
// to today's.
|
|
185
|
+
// Everything below wires the advisor. It runs only when advisorModel is
|
|
186
|
+
// set. When advisorModel is unset, the composed prompt and the tool
|
|
187
|
+
// surface stay byte-identical to today's.
|
|
187
188
|
const budget = advisorModel ? createAdvisorBudget(advisorMaxUses ?? 3) : null;
|
|
188
189
|
const agentSystemPrompt = composeSystemPrompt({
|
|
189
190
|
role: "agent",
|
|
@@ -201,8 +202,8 @@ export function createSupervisor({
|
|
|
201
202
|
systemPrompt: agentSystemPrompt,
|
|
202
203
|
redactor,
|
|
203
204
|
});
|
|
204
|
-
//
|
|
205
|
-
//
|
|
205
|
+
// The `let supervisor` closure binds this late. The instance does not
|
|
206
|
+
// exist yet when the factory builds the advisor and the tool.
|
|
206
207
|
const advisor = createAdvisor({
|
|
207
208
|
model: advisorModel,
|
|
208
209
|
cwd: agentCwd,
|
package/src/tee-writer.js
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* TeeWriter — a Writable stream that writes raw NDJSON to a file
|
|
3
|
-
*
|
|
2
|
+
* TeeWriter — a Writable stream that writes raw NDJSON to a file. At the
|
|
3
|
+
* same time it sends human-readable text to a separate stream (e.g.
|
|
4
4
|
* process.stdout).
|
|
5
5
|
*
|
|
6
6
|
* All modes emit the same { source, seq, event } envelope. The `mode`
|
|
7
|
-
* parameter controls display
|
|
7
|
+
* parameter controls the display format. Multi-participant modes show
|
|
8
8
|
* source labels on content lines.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
10
|
+
* The pure modules under `./render/` render the human text, so the live
|
|
11
|
+
* stream and the offline `TraceCollector.toText()` replay share one format
|
|
12
|
+
* path. The NDJSON that goes to `fileStream` stays untouched. Only what
|
|
13
|
+
* reaches `textStream` changes.
|
|
14
14
|
*
|
|
15
15
|
* Follows OO+DI: constructor injection, factory function, tests bypass factory.
|
|
16
16
|
*/
|
|
@@ -20,15 +20,16 @@ import { TraceCollector } from "./trace-collector.js";
|
|
|
20
20
|
import { renderTurnLines } from "./render/turn-renderer.js";
|
|
21
21
|
import { isSuppressedOrchestratorEvent } from "./render/orchestrator-filter.js";
|
|
22
22
|
|
|
23
|
-
/** Writable stream that saves raw NDJSON to a file
|
|
23
|
+
/** Writable stream that saves raw NDJSON to a file and sends human-readable text to a display stream. */
|
|
24
24
|
export class TeeWriter extends Writable {
|
|
25
25
|
/**
|
|
26
26
|
* @param {object} deps
|
|
27
27
|
* @param {import("stream").Writable} deps.fileStream - Stream to write raw NDJSON to
|
|
28
28
|
* @param {import("stream").Writable} deps.textStream - Stream to write human-readable text to
|
|
29
29
|
* @param {"raw"|"supervised"} [deps.mode] - Display mode: "raw" (no source labels) or "supervised" (source labels) (default: "raw")
|
|
30
|
-
* @param {function} [deps.now] - Injected ISO-timestamp source
|
|
31
|
-
* the internal `TraceCollector`
|
|
30
|
+
* @param {function} [deps.now] - Injected ISO-timestamp source. The class
|
|
31
|
+
* threads it into the internal `TraceCollector`
|
|
32
|
+
* (`() => isoTimestamp(runtime.clock.now())`).
|
|
32
33
|
*/
|
|
33
34
|
constructor({ fileStream, textStream, mode, now }) {
|
|
34
35
|
super();
|
|
@@ -68,18 +69,17 @@ export class TeeWriter extends Writable {
|
|
|
68
69
|
this.processLine(this.partial);
|
|
69
70
|
}
|
|
70
71
|
|
|
71
|
-
// Emit the trailing `--- Result: ... ---` footer
|
|
72
|
-
// humans want.
|
|
73
|
-
//
|
|
74
|
-
//
|
|
72
|
+
// Emit the trailing `--- Result: ... ---` footer. It is the one summary
|
|
73
|
+
// line humans want. TraceCollector.toText() appends the same tail, so
|
|
74
|
+
// the live stream and the offline replay stay in sync. No mode emits
|
|
75
|
+
// the superseded `--- Evaluation ... ---` footer.
|
|
75
76
|
if (this.collector.result) {
|
|
76
77
|
const text = this.collector.toText();
|
|
77
78
|
const idx = text.lastIndexOf("\n---");
|
|
78
79
|
if (idx !== -1) {
|
|
79
|
-
// Slice past the leading `\n
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
// the offline replay.
|
|
80
|
+
// Slice past the leading `\n`. The body that already streamed
|
|
81
|
+
// ended with its own newline. A second `\n---` here would add a
|
|
82
|
+
// blank line before the footer and desync from the offline replay.
|
|
83
83
|
this.textStream.write(text.slice(idx + 1) + "\n");
|
|
84
84
|
}
|
|
85
85
|
}
|
|
@@ -88,7 +88,7 @@ export class TeeWriter extends Writable {
|
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
/**
|
|
91
|
-
* Process a single NDJSON line
|
|
91
|
+
* Process a single NDJSON line. The same envelope logic covers all modes.
|
|
92
92
|
* @param {string} line
|
|
93
93
|
*/
|
|
94
94
|
processLine(line) {
|
|
@@ -102,14 +102,14 @@ export class TeeWriter extends Writable {
|
|
|
102
102
|
// Universal envelope: { source, seq, event }
|
|
103
103
|
if (parsed.event) {
|
|
104
104
|
// Always forward to the collector so it can capture orchestrator
|
|
105
|
-
// metadata (e.g. the summary verdict for the result footer)
|
|
106
|
-
// collector adds no turn for suppressed events
|
|
105
|
+
// metadata (e.g. the summary verdict for the result footer). The
|
|
106
|
+
// collector adds no turn for suppressed events. So flushTurns stays
|
|
107
107
|
// a no-op when we skip it below.
|
|
108
108
|
this.collector.addLine(line);
|
|
109
109
|
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
110
|
+
// The text stream drops orchestrator lifecycle events entirely.
|
|
111
|
+
// Humans only want agent-visible content. These events still reached
|
|
112
|
+
// fileStream above.
|
|
113
113
|
if (
|
|
114
114
|
parsed.source === "orchestrator" &&
|
|
115
115
|
isSuppressedOrchestratorEvent(parsed.event)
|
|
@@ -126,7 +126,7 @@ export class TeeWriter extends Writable {
|
|
|
126
126
|
}
|
|
127
127
|
|
|
128
128
|
/**
|
|
129
|
-
* Emit text for any new turns
|
|
129
|
+
* Emit text for any new turns the collector accumulated.
|
|
130
130
|
*/
|
|
131
131
|
flushTurns() {
|
|
132
132
|
const turns = this.collector.turns;
|
package/src/trace-collector.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Collects Claude Code stream-json NDJSON events into structured traces.
|
|
3
3
|
*
|
|
4
|
-
* Accepts one NDJSON line at a time
|
|
5
|
-
* structured JSON trace (toJSON) or human-readable text (toText).
|
|
4
|
+
* Accepts one NDJSON line at a time through addLine(). Then produces either
|
|
5
|
+
* a structured JSON trace (toJSON) or human-readable text (toText).
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* The pure modules under `./render/` render the human text. So the live
|
|
8
|
+
* `TeeWriter` stream and the offline `toText()` replay use one shared path
|
|
9
|
+
* to format the text.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { isoTimestamp } from "@forwardimpact/libutil";
|
|
@@ -18,12 +18,12 @@ import { isSuppressedOrchestratorEvent } from "./render/orchestrator-filter.js";
|
|
|
18
18
|
export class TraceCollector {
|
|
19
19
|
/**
|
|
20
20
|
* @param {object} [deps]
|
|
21
|
-
* @param {function} [deps.now] - Returns an ISO timestamp string.
|
|
22
|
-
* so the collector never reads the wall clock directly
|
|
23
|
-
* `() => isoTimestamp(runtime.clock.now())`.
|
|
24
|
-
* structural
|
|
25
|
-
*
|
|
26
|
-
* read.
|
|
21
|
+
* @param {function} [deps.now] - Returns an ISO timestamp string. Inject it
|
|
22
|
+
* so the collector never reads the wall clock directly. Construct it as
|
|
23
|
+
* `() => isoTimestamp(runtime.clock.now())`. Omit it for pure
|
|
24
|
+
* structural or replay use, where every event already carries a
|
|
25
|
+
* `timestamp`. The fallback then formats the epoch. The epoch is a
|
|
26
|
+
* deterministic sentinel. The fallback does not read a clock.
|
|
27
27
|
*/
|
|
28
28
|
constructor(deps = {}) {
|
|
29
29
|
/** @type {function} */
|
|
@@ -44,7 +44,7 @@ export class TraceCollector {
|
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
46
|
* Parse one NDJSON line and accumulate state.
|
|
47
|
-
*
|
|
47
|
+
* This method silently skips malformed lines.
|
|
48
48
|
* @param {string} line - A single JSON line from stream-json output
|
|
49
49
|
*/
|
|
50
50
|
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: NDJSON envelope unwrap + orchestrator/system/assistant/user dispatch
|
|
@@ -60,8 +60,8 @@ export class TraceCollector {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
// Unwrap combined supervised trace format {source, seq, event}. The
|
|
63
|
-
// Supervisor / Facilitator emits this wrapper
|
|
64
|
-
// addLine the inner event is the one we care about. Carry the envelope
|
|
63
|
+
// Supervisor / Facilitator emits this wrapper. On replay through
|
|
64
|
+
// addLine, the inner event is the one we care about. Carry the envelope
|
|
65
65
|
// `source` onto each new turn so the renderer can color it correctly.
|
|
66
66
|
let source = null;
|
|
67
67
|
if (event.event && !event.type && typeof event.source === "string") {
|
|
@@ -69,11 +69,11 @@ export class TraceCollector {
|
|
|
69
69
|
event = event.event;
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
-
// Orchestrator lifecycle events carry no content
|
|
73
|
-
// from turns entirely
|
|
72
|
+
// Orchestrator lifecycle events carry no content. The collector drops
|
|
73
|
+
// them from turns entirely. The NDJSON artifact keeps them separately.
|
|
74
74
|
if (source === "orchestrator" && isSuppressedOrchestratorEvent(event)) {
|
|
75
|
-
// The summary event carries the supervisor/facilitator verdict
|
|
76
|
-
//
|
|
75
|
+
// The summary event carries the supervisor/facilitator verdict.
|
|
76
|
+
// Capture it before you drop the event, so the result footer can
|
|
77
77
|
// surface verdict="failure" instead of the SDK's per-runner status.
|
|
78
78
|
if (event.type === "summary") {
|
|
79
79
|
this.orchestratorSummary = {
|
|
@@ -219,11 +219,11 @@ export class TraceCollector {
|
|
|
219
219
|
}
|
|
220
220
|
|
|
221
221
|
/**
|
|
222
|
-
* Accumulate a result event into the
|
|
222
|
+
* Accumulate a result event into the summary so far. Facilitated and
|
|
223
223
|
* supervised sessions emit one result event per runner invocation, so a
|
|
224
|
-
* single trace can carry several
|
|
225
|
-
*
|
|
226
|
-
*
|
|
224
|
+
* single trace can carry several. Cost, duration, turn, and token figures
|
|
225
|
+
* sum across all of them. `result` reflects the latest event. `isError`
|
|
226
|
+
* is true once any event errored.
|
|
227
227
|
* @param {object} event
|
|
228
228
|
*/
|
|
229
229
|
handleResult(event) {
|
|
@@ -270,15 +270,17 @@ export class TraceCollector {
|
|
|
270
270
|
}
|
|
271
271
|
|
|
272
272
|
/**
|
|
273
|
-
* Render the accumulated turns as human-readable text
|
|
274
|
-
* live `TeeWriter` stream uses
|
|
275
|
-
*
|
|
273
|
+
* Render the accumulated turns as human-readable text. This is the same
|
|
274
|
+
* path the live `TeeWriter` stream uses. So
|
|
275
|
+
* `gemba-harness output --format=text` over a captured trace reproduces
|
|
276
|
+
* what the live workflow log showed.
|
|
276
277
|
*
|
|
277
|
-
*
|
|
278
|
-
* source (supervised / facilitated traces). A pure `run` trace
|
|
279
|
-
* envelope
|
|
278
|
+
* The renderer emits source prefixes whenever at least one turn has a
|
|
279
|
+
* non-null source (supervised / facilitated traces). A pure `run` trace
|
|
280
|
+
* has no envelope. All turn sources are null. The renderer drops the
|
|
281
|
+
* prefix.
|
|
280
282
|
*
|
|
281
|
-
* @returns {string} Formatted text output
|
|
283
|
+
* @returns {string} Formatted text output with ANSI escapes
|
|
282
284
|
*/
|
|
283
285
|
toText() {
|
|
284
286
|
const withPrefix = this.turns.some((t) => t.source);
|
|
@@ -290,22 +292,22 @@ export class TraceCollector {
|
|
|
290
292
|
|
|
291
293
|
const tail = this.#formatResultTail();
|
|
292
294
|
|
|
293
|
-
// Each rendered line already ends with `\n
|
|
294
|
-
// trailing newline
|
|
295
|
-
// compatible with existing consumers
|
|
296
|
-
// the result footer
|
|
297
|
-
//
|
|
295
|
+
// Each rendered line already ends with `\n`. Concatenate the lines.
|
|
296
|
+
// Drop the trailing newline. Then append the tail so the output shape
|
|
297
|
+
// stays compatible with existing consumers. With turns, there is no
|
|
298
|
+
// double-blank line before the result footer. Without turns, there is
|
|
299
|
+
// no leading blank.
|
|
298
300
|
const body = out.join("").replace(/\n$/, "");
|
|
299
301
|
return body + tail;
|
|
300
302
|
}
|
|
301
303
|
|
|
302
304
|
/**
|
|
303
|
-
* Format the trailing result summary line.
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
305
|
+
* Format the trailing result summary line. With an orchestrator summary
|
|
306
|
+
* (supervised / facilitated mode), the headline word is the supervisor's
|
|
307
|
+
* verdict ("success" / "failure"). It is not the SDK's per-runner
|
|
308
|
+
* subtype. The footer then aligns with the CI exit code. Turn, cost, and
|
|
309
|
+
* duration figures are the accumulated totals across every result event
|
|
310
|
+
* in the trace. They are not the last event's figures.
|
|
309
311
|
* @returns {string}
|
|
310
312
|
*/
|
|
311
313
|
#formatResultTail() {
|
|
@@ -320,7 +322,7 @@ export class TraceCollector {
|
|
|
320
322
|
}
|
|
321
323
|
}
|
|
322
324
|
|
|
323
|
-
/** Identity element
|
|
325
|
+
/** Identity element that handleResult uses to accumulate result events. */
|
|
324
326
|
const EMPTY_RESULT = {
|
|
325
327
|
isError: false,
|
|
326
328
|
totalCostUsd: 0,
|
|
@@ -347,7 +349,7 @@ function normalizeUsage(usage) {
|
|
|
347
349
|
|
|
348
350
|
/**
|
|
349
351
|
* Sum two token-usage records field-by-field. Either side may be null
|
|
350
|
-
* (a result event without usage)
|
|
352
|
+
* (a result event without usage). The sum is null only when both are null.
|
|
351
353
|
* @param {object|null} a
|
|
352
354
|
* @param {object|null} b
|
|
353
355
|
* @returns {object|null}
|
|
@@ -365,9 +367,10 @@ function sumTokenUsage(a, b) {
|
|
|
365
367
|
}
|
|
366
368
|
|
|
367
369
|
/**
|
|
368
|
-
* Per-model fields that sum additively across result events
|
|
369
|
-
* per-model cost, and request counters.
|
|
370
|
-
* context-window size)
|
|
370
|
+
* Per-model fields that sum additively across result events: token counts,
|
|
371
|
+
* per-model cost, and request counters. The merge carries every other
|
|
372
|
+
* per-model field (e.g. a context-window size) from the first event that
|
|
373
|
+
* set it. The merge never sums those fields.
|
|
371
374
|
*/
|
|
372
375
|
const ADDITIVE_MODEL_FIELDS = [
|
|
373
376
|
"inputTokens",
|
|
@@ -380,8 +383,9 @@ const ADDITIVE_MODEL_FIELDS = [
|
|
|
380
383
|
|
|
381
384
|
/**
|
|
382
385
|
* Merge two per-model usage maps across result events. Additive fields
|
|
383
|
-
* (token counts, cost, request counters) sum
|
|
384
|
-
* from the first event that set them (prev wins).
|
|
386
|
+
* (token counts, cost, request counters) sum. The merge carries
|
|
387
|
+
* non-additive fields from the first event that set them (prev wins).
|
|
388
|
+
* Either side may be null.
|
|
385
389
|
* @param {object|null} prevMU
|
|
386
390
|
* @param {object|null} nextMU
|
|
387
391
|
* @returns {object|null}
|
|
@@ -401,7 +405,8 @@ function mergeModelUsage(prevMU, nextMU) {
|
|
|
401
405
|
}
|
|
402
406
|
|
|
403
407
|
/**
|
|
404
|
-
* Merge one model's usage
|
|
408
|
+
* Merge one model's usage. Additive fields sum. Other fields carry the
|
|
409
|
+
* first-seen value (a).
|
|
405
410
|
* @param {object} a - First-seen (prev) per-model usage.
|
|
406
411
|
* @param {object} b - Next per-model usage.
|
|
407
412
|
* @returns {object}
|
package/src/trace-github.js
CHANGED
|
@@ -7,8 +7,8 @@ import { isoTimestamp } from "@forwardimpact/libutil";
|
|
|
7
7
|
const API = "https://api.github.com";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
|
-
* GitHub API client for trace-related operations
|
|
11
|
-
* and
|
|
10
|
+
* GitHub API client for trace-related operations. It lists workflow runs
|
|
11
|
+
* and downloads trace artifacts.
|
|
12
12
|
*/
|
|
13
13
|
export class TraceGitHub {
|
|
14
14
|
/**
|
|
@@ -17,7 +17,7 @@ export class TraceGitHub {
|
|
|
17
17
|
* @param {string} deps.owner - Repository owner
|
|
18
18
|
* @param {string} deps.repo - Repository name
|
|
19
19
|
* @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime -
|
|
20
|
-
* Ambient collaborators
|
|
20
|
+
* Ambient collaborators. The class uses `fs`, `subprocess`, `clock`.
|
|
21
21
|
*/
|
|
22
22
|
constructor({ token, owner, repo, runtime }) {
|
|
23
23
|
if (!runtime) throw new Error("runtime is required");
|
|
@@ -28,27 +28,30 @@ export class TraceGitHub {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
/**
|
|
31
|
-
* List recent workflow runs
|
|
32
|
-
* participant whose trace lane a run carries.
|
|
31
|
+
* List recent workflow runs. Filter them by name pattern, and by the
|
|
32
|
+
* participant whose trace lane a run carries. Both filters are optional.
|
|
33
33
|
*
|
|
34
|
-
* Without `participant`, behaviour
|
|
35
|
-
* the only filter. With `participant`,
|
|
36
|
-
* against its trace lane
|
|
37
|
-
* with a
|
|
34
|
+
* Without `participant`, the behaviour does not change. The workflow-name
|
|
35
|
+
* pattern is the only filter. With `participant`, the method resolves each
|
|
36
|
+
* name-matched run against its trace lane
|
|
37
|
+
* (see {@link runMatchesParticipant}). It then annotates the run with a
|
|
38
|
+
* `match` field:
|
|
38
39
|
* - `"confirmed"` — the participant's lane is present in the run's
|
|
39
40
|
* artifacts (matrix artifact name, or a member filename in the shared
|
|
40
41
|
* dispatch artifact).
|
|
41
42
|
* - `"unconfirmed-pending-artifacts"` — the run's workflow mints trace
|
|
42
43
|
* artifacts but none exist yet (still running, or completed-but-not-yet
|
|
43
|
-
* uploaded)
|
|
44
|
-
*
|
|
45
|
-
*
|
|
44
|
+
* uploaded). The method reports the run as a candidate. It never drops
|
|
45
|
+
* the run silently.
|
|
46
|
+
* The method omits runs that have artifacts but no matching lane. It reads
|
|
47
|
+
* participant identity from artifact/file *names* only. It never reads
|
|
48
|
+
* trace content.
|
|
46
49
|
*
|
|
47
50
|
* @param {object} [opts]
|
|
48
|
-
* @param {string} [opts.pattern] - Case-insensitive regex to match workflow name (default: "kata|agent"
|
|
51
|
+
* @param {string} [opts.pattern] - Case-insensitive regex to match workflow name (default: "kata|agent", which covers `Kata: Shift`, `Kata: Dispatch`, and any `agent`-named workflow)
|
|
49
52
|
* @param {number} [opts.limit=50] - Max runs to return from GitHub API
|
|
50
53
|
* @param {string} [opts.lookback="7d"] - How far back to search (e.g. "7d", "24h", "2w")
|
|
51
|
-
* @param {string} [opts.participant] - Participant name
|
|
54
|
+
* @param {string} [opts.participant] - Participant name. When set, the method filters and annotates runs by trace lane
|
|
52
55
|
* @returns {Promise<object[]>} Array of {workflow, runId, status, conclusion, createdAt, branch, url[, match]}
|
|
53
56
|
*/
|
|
54
57
|
async listRuns(opts = {}) {
|
|
@@ -97,15 +100,17 @@ export class TraceGitHub {
|
|
|
97
100
|
* Decide whether a run carries a participant's trace lane.
|
|
98
101
|
*
|
|
99
102
|
* Matrix hosts name the participant in an artifact name
|
|
100
|
-
* (`trace--<participant>`)
|
|
101
|
-
* (`trace--<case>--<participant>.<role>.ndjson`) inside one shared
|
|
102
|
-
* artifact. The GitHub artifacts API exposes only
|
|
103
|
-
* a matrix lane confirms from the inventory
|
|
104
|
-
*
|
|
105
|
-
* filenames
|
|
103
|
+
* (`trace--<participant>`). Dispatch hosts name it in a member filename
|
|
104
|
+
* (`trace--<case>--<participant>.<role>.ndjson`) inside one shared
|
|
105
|
+
* `trace--*` artifact. The GitHub artifacts API exposes only
|
|
106
|
+
* artifact-level metadata. So a matrix lane confirms from the inventory
|
|
107
|
+
* alone. For a dispatch lane, the method downloads the shared artifact.
|
|
108
|
+
* Then it lists the extracted member filenames. It reads names only. It
|
|
109
|
+
* never reads trace content.
|
|
106
110
|
*
|
|
107
111
|
* A run whose trace artifacts are absent (still running, or
|
|
108
|
-
* completed-but-not-yet-uploaded) is a candidate
|
|
112
|
+
* completed-but-not-yet-uploaded) is a candidate. The method does not
|
|
113
|
+
* drop it.
|
|
109
114
|
*
|
|
110
115
|
* @param {number|string} runId
|
|
111
116
|
* @param {string} participant
|
|
@@ -119,8 +124,9 @@ export class TraceGitHub {
|
|
|
119
124
|
a.name.startsWith("trace--"),
|
|
120
125
|
);
|
|
121
126
|
|
|
122
|
-
// No trace artifacts yet
|
|
123
|
-
// the lane may upload when the
|
|
127
|
+
// No trace artifacts yet. The matcher must report this run as a
|
|
128
|
+
// candidate. It must not drop the run. The lane may upload when the
|
|
129
|
+
// host completes.
|
|
124
130
|
if (traceArtifacts.length === 0) return "unconfirmed-pending-artifacts";
|
|
125
131
|
|
|
126
132
|
// Matrix host: the participant is an artifact name. No download.
|
|
@@ -146,10 +152,11 @@ export class TraceGitHub {
|
|
|
146
152
|
|
|
147
153
|
/**
|
|
148
154
|
* Resolve a participant's lane trace path for a known run in one keyed
|
|
149
|
-
* lookup
|
|
155
|
+
* lookup. The method does not enumerate runs. It does not inspect trace
|
|
156
|
+
* content.
|
|
150
157
|
*
|
|
151
158
|
* Matrix host: the artifact name carries the participant (no download).
|
|
152
|
-
* Dispatch host: download the shared `trace--*` artifact
|
|
159
|
+
* Dispatch host: download the shared `trace--*` artifact. Return the
|
|
153
160
|
* extracted member file whose name carries the participant.
|
|
154
161
|
*
|
|
155
162
|
* @param {number|string} runId
|
|
@@ -208,12 +215,12 @@ export class TraceGitHub {
|
|
|
208
215
|
}
|
|
209
216
|
|
|
210
217
|
/**
|
|
211
|
-
* Download a trace artifact from a workflow run
|
|
218
|
+
* Download a trace artifact from a workflow run. Extract it.
|
|
212
219
|
*
|
|
213
|
-
*
|
|
214
|
-
* the single `trace--*` artifact
|
|
215
|
-
*
|
|
216
|
-
*
|
|
220
|
+
* With `opts.name` set, the method looks up that exact artifact.
|
|
221
|
+
* Otherwise it picks the single `trace--*` artifact when exactly one
|
|
222
|
+
* exists. When matrix workflows emit multiple per-participant artifacts,
|
|
223
|
+
* it throws with a disambiguation list (see {@link pickTraceArtifact}).
|
|
217
224
|
*
|
|
218
225
|
* @param {number|string} runId
|
|
219
226
|
* @param {object} [opts]
|
|
@@ -296,13 +303,14 @@ export class TraceGitHub {
|
|
|
296
303
|
/**
|
|
297
304
|
* Test whether a participant's trace lane is present in a list of names.
|
|
298
305
|
*
|
|
299
|
-
*
|
|
306
|
+
* This function matches the two trace-name shapes by *name* only (never by
|
|
307
|
+
* content):
|
|
300
308
|
* - matrix artifact name: `trace--<participant>`
|
|
301
309
|
* - dispatch member filename: `trace--<case>--<participant>.<role>.ndjson`
|
|
302
310
|
*
|
|
303
|
-
* The
|
|
304
|
-
* or end
|
|
305
|
-
* `release-engineer` and vice versa.
|
|
311
|
+
* The `--` separator delimits the participant segment. The segment ends at
|
|
312
|
+
* the next `--`, at a `.`, or at the end of the string. So a substring like
|
|
313
|
+
* `release` does not match `release-engineer` and vice versa.
|
|
306
314
|
*
|
|
307
315
|
* @param {string[]} names - Artifact names or extracted member filenames.
|
|
308
316
|
* @param {string} participant - Participant name to look for.
|
|
@@ -326,11 +334,12 @@ export function participantInNames(names, participant) {
|
|
|
326
334
|
/**
|
|
327
335
|
* Pick the trace artifact to download from a workflow run's artifact list.
|
|
328
336
|
*
|
|
329
|
-
*
|
|
330
|
-
* names.
|
|
331
|
-
*
|
|
332
|
-
* `kata-shift.yml` emit one `trace--<participant>`
|
|
333
|
-
* lists them
|
|
337
|
+
* With `name` given, this function returns the exact match. If it finds no
|
|
338
|
+
* match, it throws with the available names. Without `name`, it returns the
|
|
339
|
+
* only `trace--*` artifact when exactly one exists. With more than one
|
|
340
|
+
* (matrix workflows like `kata-shift.yml` emit one `trace--<participant>`
|
|
341
|
+
* per cell), it throws and lists them. The caller can then pass `--name` to
|
|
342
|
+
* disambiguate.
|
|
334
343
|
*
|
|
335
344
|
* @param {Array<{name: string}>} artifacts - Artifact list from the GitHub API.
|
|
336
345
|
* @param {string} [name] - Exact artifact name to match.
|
|
@@ -407,7 +416,7 @@ export function parseGitRemote(remote) {
|
|
|
407
416
|
* Detect the current GitHub repository slug as `{owner, repo}`.
|
|
408
417
|
*
|
|
409
418
|
* Resolution order:
|
|
410
|
-
* 1. `GITHUB_REPOSITORY` env var (
|
|
419
|
+
* 1. `GITHUB_REPOSITORY` env var (GitHub Actions sets it automatically).
|
|
411
420
|
* 2. `git remote get-url origin` in the current working directory.
|
|
412
421
|
*
|
|
413
422
|
* @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
|
|
@@ -442,12 +451,12 @@ export async function detectRepoSlug(runtime) {
|
|
|
442
451
|
}
|
|
443
452
|
|
|
444
453
|
/**
|
|
445
|
-
* Create a TraceGitHub instance. The caller
|
|
446
|
-
*
|
|
447
|
-
*
|
|
454
|
+
* Create a TraceGitHub instance. The caller must resolve the GitHub token,
|
|
455
|
+
* typically with `Config.ghToken()`. So the CLI entry point loads the
|
|
456
|
+
* credentials.
|
|
448
457
|
*
|
|
449
458
|
* Breaking change from the prior signature: `token` is now a required
|
|
450
|
-
* caller input. Construct a `Config`
|
|
459
|
+
* caller input. Construct a `Config` with `@forwardimpact/libconfig`. Then
|
|
451
460
|
* pass `config.ghToken()`.
|
|
452
461
|
*
|
|
453
462
|
* @param {object} opts
|