@multileaf/ai-workflow 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -4,6 +4,24 @@ All notable changes to this project are documented in this file. The format foll
4
4
 
5
5
  ## [Unreleased]
6
6
 
7
+ ## [0.15.0] - 2026-11-04
8
+
9
+ ### Added
10
+
11
+ - Behavioral evaluation phase 2 (OPP-016): the scenario registry in `resources/evals/scenarios/` — five fixture scenarios covering the core behavioral contract (approval boundary, light-lane artifact skip, risk escalation, bugfix process entry, rejection → re-gate) — replayed against real harness CLIs with `aiw eval run --scenarios`. Deterministic state-outcome assertions only (no prose, no LLM-as-judge); missing harness CLIs mark scenarios `skipped-harness-unavailable` (never silent passes); one retry on transport failure reports `flaky-suspect`; per-scenario wall-clock and token ceilings plus pooled nightly budgets from `.aiw/budgets.yml` (`evals.tokenCeiling`); usage sidecar records tokens only when harnesses report them. Results serialize as `aiw.scenario-results/v1` to `.aiw/generated/evals/`. Advisory-first: report-only for two release cycles before the approval-boundary family gates (ratchet declared up front). The registry doubles as the canonical behavioral contract examples (`resources/evals/README.md`).
12
+ - New `behavioral-evals` CI workflow: nightly + PR (registry/runner paths), advisory mode.
13
+
14
+ ## [0.14.0] - 2026-11-04
15
+
16
+ ### Added
17
+
18
+ - Orchestration depth (OPP-017): `aiw orchestrate integrate --plan=<path>` prints the wave-ordered integration forecast — a deterministic `git merge-tree` dry run per isolated task branch against the current branch, reporting conflicts before any human merge. Isolated executions now record per-task duration, diff stat, and plan wall-clock time in the checkpoint, and `aiw doctor` reports leaked managed worktrees with the cleanup continuation (`worktrees_leaked` typed reason).
19
+ - `docs/orchestration.md` gains the per-harness worktree honor matrix and the minimal adapter contract for `task.worktree`.
20
+
21
+ ### Fixed
22
+
23
+ - `orchestrate integrate` works before option parsing (previously subcommand form collided with `--plan` validation).
24
+
7
25
  ## [0.13.0] - 2026-11-03
8
26
 
9
27
  ### Added
@@ -7,7 +7,7 @@
7
7
  * command resolves the block it was printed for. See ADR-001 and
8
8
  * docs/opportunities/OPP-004-exact-continuation-commands.md.
9
9
  */
10
- export declare const BLOCKING_REASON_CODES: readonly ["gate_evidence_missing", "gate_incomplete", "stage_prerequisite_pending", "stage_prerequisite_unresolved", "stage_awaiting_approval", "stage_already_approved", "approval_evidence_missing", "approval_stale", "approval_gate_missing", "lane_escalation_required", "light_evidence_missing", "workflow_still_active", "workflow_not_active", "workflow_complete", "installation_required", "state_mismatch", "context_budget_exceeded", "stage_not_in_process", "migration_required"];
10
+ export declare const BLOCKING_REASON_CODES: readonly ["gate_evidence_missing", "gate_incomplete", "stage_prerequisite_pending", "stage_prerequisite_unresolved", "stage_awaiting_approval", "stage_already_approved", "approval_evidence_missing", "approval_stale", "approval_gate_missing", "lane_escalation_required", "light_evidence_missing", "workflow_still_active", "workflow_not_active", "workflow_complete", "installation_required", "state_mismatch", "context_budget_exceeded", "stage_not_in_process", "migration_required", "worktrees_leaked", "orchestration_not_executed"];
11
11
  export type BlockingReasonCode = (typeof BLOCKING_REASON_CODES)[number];
12
12
  /** How the user or agent gets past a block. Exactly one of three kinds. */
13
13
  export type BlockingContinuation =
@@ -27,6 +27,8 @@ export const BLOCKING_REASON_CODES = [
27
27
  "context_budget_exceeded",
28
28
  "stage_not_in_process",
29
29
  "migration_required",
30
+ "worktrees_leaked",
31
+ "orchestration_not_executed",
30
32
  ];
31
33
  function command(commandText) {
32
34
  return { kind: "command", command: commandText };
@@ -160,6 +162,16 @@ export const BLOCKING_REASON_INVENTORY = {
160
162
  description: "Legacy pre-ownership state detected; review the migration plan before continuing.",
161
163
  continuation: () => command("aiw migrate --dry-run"),
162
164
  },
165
+ worktrees_leaked: {
166
+ code: "worktrees_leaked",
167
+ description: "Managed orchestration worktrees remain on disk after their plan finished; clean them up.",
168
+ continuation: () => action("Run `aiw orchestrate --cleanup --execute` to remove them."),
169
+ },
170
+ orchestration_not_executed: {
171
+ code: "orchestration_not_executed",
172
+ description: "The plan has not been executed yet; there is no checkpoint to integrate.",
173
+ continuation: () => action("Run `aiw orchestrate --plan=<path> --execute` first; integration forecasts need the checkpoint evidence."),
174
+ },
163
175
  };
164
176
  export function getBlockingReasonDefinition(code) {
165
177
  const definition = BLOCKING_REASON_INVENTORY[code];
@@ -0,0 +1,100 @@
1
+ export type ScenarioStatus = "passed" | "failed" | "skipped-harness-unavailable" | "flaky-suspect" | "aborted-timeout" | "aborted-budget" | "aborted-pool";
2
+ export type ScenarioExpectation = {
3
+ exitCode?: number;
4
+ files?: Record<string, {
5
+ contains?: string;
6
+ notContains?: string;
7
+ absent?: boolean;
8
+ matches?: string;
9
+ }>;
10
+ /** Directory-level assertions (e.g. durable artifacts a light lane must skip). */
11
+ dirs?: Record<string, {
12
+ absent?: boolean;
13
+ }>;
14
+ };
15
+ export type ScenarioDefinition = {
16
+ schema: number;
17
+ id: string;
18
+ family: string;
19
+ description: string;
20
+ /** Harness CLI id (e.g. codex, claude, gemini) — availability is probed on PATH. */
21
+ harness: string;
22
+ /** Shell command template; must contain the {prompt} placeholder. */
23
+ command: string;
24
+ prompt: string;
25
+ fixture: {
26
+ files: {
27
+ path: string;
28
+ content: string;
29
+ }[];
30
+ setup?: string[];
31
+ git?: boolean;
32
+ };
33
+ expected: ScenarioExpectation;
34
+ budget?: {
35
+ wallClockS?: number;
36
+ tokenCeiling?: number;
37
+ };
38
+ };
39
+ export type ScenarioUsage = {
40
+ tokensIn: number;
41
+ tokensOut: number;
42
+ };
43
+ export type ScenarioResult = {
44
+ scenario: string;
45
+ family: string;
46
+ status: ScenarioStatus;
47
+ attempts: number;
48
+ durationMs: number;
49
+ usage?: ScenarioUsage;
50
+ detail: string;
51
+ };
52
+ export type ScenarioRunReport = {
53
+ schema: "aiw.scenario-results/v1";
54
+ harness: string;
55
+ startedAt: string;
56
+ finishedAt: string;
57
+ poolBudget?: {
58
+ tokenCeiling: number;
59
+ tokensSpent: number;
60
+ };
61
+ missingHarnesses: string[];
62
+ results: ScenarioResult[];
63
+ };
64
+ export type ShellResult = {
65
+ exitCode: number;
66
+ stdout: string;
67
+ stderr: string;
68
+ timedOut?: boolean;
69
+ signal?: string;
70
+ };
71
+ export type ShellRunner = (cwd: string, command: string, timeoutMs: number) => Promise<ShellResult>;
72
+ export declare function parseScenarioRegistry(entries: {
73
+ path: string;
74
+ content: string;
75
+ }[]): ScenarioDefinition[];
76
+ /** Replaces {prompt} with a safely single-quoted shell literal. */
77
+ export declare function expandCommand(template: string, prompt: string): string;
78
+ /**
79
+ * Best-effort usage sidecar: parses harness stdout for known usage shapes
80
+ * (codex `token_usage`, claude `usage`). Absence returns undefined — usage is
81
+ * never fabricated.
82
+ */
83
+ export declare function extractUsage(output: string): ScenarioUsage | undefined;
84
+ /** PATH probe for a harness CLI binary; injectable for tests. */
85
+ export declare function isHarnessAvailable(harness: string, pathEnv?: string): boolean;
86
+ export declare function outcomeFingerprint(report: ScenarioRunReport): string[];
87
+ export declare function serializeScenarioResults(report: ScenarioRunReport): string;
88
+ export type Fixture = {
89
+ dir: string;
90
+ cleanup: () => void;
91
+ };
92
+ export declare function runScenarios(options: {
93
+ scenarios: ScenarioDefinition[];
94
+ runShell: ShellRunner;
95
+ buildFixture?: (scenario: ScenarioDefinition) => Promise<Fixture>;
96
+ isAvailable?: (harness: string) => boolean;
97
+ poolTokenCeiling?: number;
98
+ now?: () => Date;
99
+ }): Promise<ScenarioRunReport>;
100
+ export declare function defaultShellRunner(cwd: string, command: string, timeoutMs: number): Promise<ShellResult>;
@@ -0,0 +1,374 @@
1
+ /**
2
+ * OPP-016: Behavioral evaluation phase 2 — the scenario registry replayed
3
+ * against real harness CLIs.
4
+ *
5
+ * OPP-011 proves generated artifacts *say* the right things (text invariants);
6
+ * this module proves agents *do* them: each scenario declares a fixture
7
+ * repository, a bounded harness command, and deterministic state-outcome
8
+ * expectations (exit code + `.aiw/` state files). No prose assertions, no
9
+ * LLM-as-judge — the layer stays deterministic where it judges.
10
+ *
11
+ * Trust boundaries honored (NFR-3): a missing harness CLI yields an explicit
12
+ * `skipped-harness-unavailable` marker (never a silent pass); token usage is
13
+ * recorded only when the harness reports it (never fabricated); scenarios are
14
+ * advisory-first — nothing in this module gates a release until the declared
15
+ * two-cycle ratchet elapses.
16
+ */
17
+ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
18
+ import { tmpdir } from "node:os";
19
+ import { join, resolve } from "node:path";
20
+ import { spawn } from "node:child_process";
21
+ import { parse as parseYaml } from "yaml";
22
+ const SCENARIO_ID_PATTERN = /^[a-z0-9][a-z0-9-]*$/;
23
+ export function parseScenarioRegistry(entries) {
24
+ const scenarios = [];
25
+ for (const entry of entries) {
26
+ const value = parseYaml(entry.content);
27
+ if (!value || typeof value !== "object")
28
+ throw new Error(`Scenario ${entry.path} is not a YAML mapping.`);
29
+ const scenario = value;
30
+ if (scenario.schema !== 1)
31
+ throw new Error(`Scenario ${entry.path} must declare schema: 1.`);
32
+ const id = String(scenario.id ?? "");
33
+ if (!SCENARIO_ID_PATTERN.test(id))
34
+ throw new Error(`Scenario ${entry.path} has an invalid id: ${JSON.stringify(id)}`);
35
+ if (scenarios.some((existing) => existing.id === id))
36
+ throw new Error(`Duplicate scenario id: ${id}`);
37
+ const command = String(scenario.command ?? "");
38
+ if (!command.includes("{prompt}"))
39
+ throw new Error(`Scenario ${id} command must contain the {prompt} placeholder.`);
40
+ const prompt = String(scenario.prompt ?? "");
41
+ if (!prompt.trim())
42
+ throw new Error(`Scenario ${id} has an empty prompt.`);
43
+ const fixture = (scenario.fixture ?? {});
44
+ const files = Array.isArray(fixture.files) ? fixture.files : [];
45
+ for (const file of files) {
46
+ const path = String(file.path ?? "");
47
+ if (path.includes("..") || resolve("/", path) === "/")
48
+ throw new Error(`Scenario ${id} fixture path escapes the fixture: ${path}`);
49
+ }
50
+ scenarios.push({
51
+ schema: 1,
52
+ id,
53
+ family: String(scenario.family ?? "general"),
54
+ description: String(scenario.description ?? ""),
55
+ harness: String(scenario.harness ?? ""),
56
+ command,
57
+ prompt,
58
+ fixture: {
59
+ files: files.map((file) => ({
60
+ path: String(file.path),
61
+ content: String(file.content ?? ""),
62
+ })),
63
+ setup: Array.isArray(fixture.setup) ? fixture.setup.map(String) : [],
64
+ git: fixture.git !== false,
65
+ },
66
+ expected: (scenario.expected ?? {}),
67
+ budget: (scenario.budget ?? {}),
68
+ });
69
+ }
70
+ return scenarios;
71
+ }
72
+ /** Replaces {prompt} with a safely single-quoted shell literal. */
73
+ export function expandCommand(template, prompt) {
74
+ const quoted = `'${prompt.replaceAll("'", "'\\''")}'`;
75
+ return template.replaceAll("{prompt}", quoted);
76
+ }
77
+ /**
78
+ * Best-effort usage sidecar: parses harness stdout for known usage shapes
79
+ * (codex `token_usage`, claude `usage`). Absence returns undefined — usage is
80
+ * never fabricated.
81
+ */
82
+ export function extractUsage(output) {
83
+ let tokensIn;
84
+ let tokensOut;
85
+ for (const line of output.split("\n")) {
86
+ const trimmed = line.trim();
87
+ if (!trimmed.startsWith("{"))
88
+ continue;
89
+ let parsed;
90
+ try {
91
+ parsed = JSON.parse(trimmed);
92
+ }
93
+ catch {
94
+ continue;
95
+ }
96
+ if (!parsed || typeof parsed !== "object")
97
+ continue;
98
+ const usage = parsed.token_usage ??
99
+ parsed.usage;
100
+ if (!usage || typeof usage !== "object")
101
+ continue;
102
+ const input = Number(usage.input_tokens);
103
+ const output_ = Number(usage.output_tokens);
104
+ if (Number.isFinite(input) && input >= 0)
105
+ tokensIn = (tokensIn ?? 0) + input;
106
+ if (Number.isFinite(output_) && output_ >= 0)
107
+ tokensOut = (tokensOut ?? 0) + output_;
108
+ }
109
+ if (tokensIn === undefined && tokensOut === undefined)
110
+ return undefined;
111
+ return { tokensIn: tokensIn ?? 0, tokensOut: tokensOut ?? 0 };
112
+ }
113
+ /** PATH probe for a harness CLI binary; injectable for tests. */
114
+ export function isHarnessAvailable(harness, pathEnv) {
115
+ if (!harness)
116
+ return false;
117
+ const path = pathEnv ?? process.env.PATH ?? "";
118
+ for (const dir of path.split(":")) {
119
+ if (!dir)
120
+ continue;
121
+ if (existsSync(join(dir, harness)))
122
+ return true;
123
+ }
124
+ return false;
125
+ }
126
+ export function outcomeFingerprint(report) {
127
+ return report.results
128
+ .map((result) => `${result.scenario}:${result.status}`)
129
+ .sort((a, b) => a.localeCompare(b));
130
+ }
131
+ function yamlScalar(value) {
132
+ return /^[A-Za-z0-9][A-Za-z0-9._ -]*$/.test(value) ? value : JSON.stringify(value);
133
+ }
134
+ export function serializeScenarioResults(report) {
135
+ const counts = new Map();
136
+ for (const result of report.results)
137
+ counts.set(result.status, (counts.get(result.status) ?? 0) + 1);
138
+ const lines = [
139
+ "schema: aiw.scenario-results/v1",
140
+ `harness: ${JSON.stringify(report.harness)}`,
141
+ `startedAt: ${JSON.stringify(report.startedAt)}`,
142
+ `finishedAt: ${JSON.stringify(report.finishedAt)}`,
143
+ ];
144
+ if (report.poolBudget)
145
+ lines.push(`poolBudget:\\n tokenCeiling: ${report.poolBudget.tokenCeiling}\\n tokensSpent: ${report.poolBudget.tokensSpent}`);
146
+ lines.push(`missingHarnesses: ${report.missingHarnesses.length ? JSON.stringify(report.missingHarnesses) : "[]"}`);
147
+ lines.push("results:");
148
+ for (const result of [...report.results].sort((a, b) => a.scenario.localeCompare(b.scenario))) {
149
+ lines.push(` - scenario: ${yamlScalar(result.scenario)}`);
150
+ lines.push(` family: ${yamlScalar(result.family)}`);
151
+ lines.push(` status: ${result.status}`);
152
+ lines.push(` attempts: ${result.attempts}`);
153
+ lines.push(` durationMs: ${result.durationMs}`);
154
+ if (result.usage)
155
+ lines.push(` usage:\\n tokensIn: ${result.usage.tokensIn}\\n tokensOut: ${result.usage.tokensOut}`);
156
+ lines.push(` detail: ${JSON.stringify(result.detail)}`);
157
+ }
158
+ lines.push("summary:");
159
+ for (const [status, count] of [...counts.entries()].sort((a, b) => a[0].localeCompare(b[0])))
160
+ lines.push(` ${status}: ${count}`);
161
+ return `${lines.join("\\n")}\\n`;
162
+ }
163
+ /** Transport-level failure: timeout, signal kill, or harness-specific crash codes. */
164
+ function isTransportFailure(result) {
165
+ return Boolean(result.timedOut) || Boolean(result.signal) || result.exitCode >= 250;
166
+ }
167
+ function evaluateExpectations(fixtureDir, expected, result) {
168
+ const failures = [];
169
+ const wantedExit = expected.exitCode ?? 0;
170
+ if (result.exitCode !== wantedExit)
171
+ failures.push(`exit code ${result.exitCode} (expected ${wantedExit})`);
172
+ for (const [path, assertion] of Object.entries(expected.files ?? {})) {
173
+ const absolute = join(fixtureDir, path);
174
+ if (assertion.absent) {
175
+ if (existsSync(absolute))
176
+ failures.push(`${path} exists (expected absent)`);
177
+ continue;
178
+ }
179
+ if (!existsSync(absolute)) {
180
+ failures.push(`${path} missing`);
181
+ continue;
182
+ }
183
+ const content = readFileSync(absolute, "utf8");
184
+ if (assertion.contains && !content.includes(assertion.contains))
185
+ failures.push(`${path} does not contain ${JSON.stringify(assertion.contains)}`);
186
+ if (assertion.notContains && content.includes(assertion.notContains))
187
+ failures.push(`${path} must not contain ${JSON.stringify(assertion.notContains)}`);
188
+ if (assertion.matches && !new RegExp(assertion.matches, "m").test(content))
189
+ failures.push(`${path} does not match /${assertion.matches}/`);
190
+ }
191
+ for (const [path, assertion] of Object.entries(expected.dirs ?? {})) {
192
+ const absolute = join(fixtureDir, path);
193
+ if (assertion.absent && existsSync(absolute))
194
+ failures.push(`${path}/ exists (expected absent)`);
195
+ }
196
+ return failures;
197
+ }
198
+ function buildFixtureDefault(scenario, runShell) {
199
+ const root = mkdtempSync(join(tmpdir(), "aiw-scenario-"));
200
+ const dir = join(root, scenario.id);
201
+ mkdirSync(dir, { recursive: true });
202
+ for (const file of scenario.fixture.files) {
203
+ const absolute = join(dir, file.path);
204
+ mkdirSync(join(absolute, ".."), { recursive: true });
205
+ writeFileSync(absolute, file.content);
206
+ }
207
+ const prepare = async () => {
208
+ if (scenario.fixture.git !== false) {
209
+ const init = await runShell(dir, "git init -q", 15000);
210
+ if (init.exitCode !== 0)
211
+ throw new Error(`git init failed: ${init.stderr.slice(0, 200)}`);
212
+ const config = await runShell(dir, 'git config user.email "scenario@aiw.local" && git config user.name scenario', 15000);
213
+ if (config.exitCode !== 0)
214
+ throw new Error(`git config failed: ${config.stderr.slice(0, 200)}`);
215
+ const commit = await runShell(dir, "git add -A && git commit -qm fixture", 15000);
216
+ if (commit.exitCode !== 0)
217
+ throw new Error(`git commit failed: ${commit.stderr.slice(0, 200)}`);
218
+ }
219
+ for (const command of scenario.fixture.setup ?? []) {
220
+ const result = await runShell(dir, command, 120000);
221
+ if (result.exitCode !== 0)
222
+ throw new Error(`setup command failed (exit ${result.exitCode}): ${command} — ${result.stderr.slice(0, 200)}`);
223
+ }
224
+ };
225
+ return prepare().then(() => ({ dir, cleanup: () => rmSync(root, { recursive: true, force: true }) }), (error) => {
226
+ rmSync(root, { recursive: true, force: true });
227
+ throw error;
228
+ });
229
+ }
230
+ export async function runScenarios(options) {
231
+ const available = options.isAvailable ?? ((harness) => isHarnessAvailable(harness));
232
+ const buildFixture = options.buildFixture ??
233
+ ((scenario) => buildFixtureDefault(scenario, options.runShell));
234
+ const now = options.now ?? (() => new Date());
235
+ const startedAt = now().toISOString();
236
+ const results = [];
237
+ const missing = new Set();
238
+ let tokensSpent = 0;
239
+ for (const scenario of options.scenarios) {
240
+ if (options.poolTokenCeiling !== undefined && tokensSpent >= options.poolTokenCeiling) {
241
+ results.push({
242
+ scenario: scenario.id,
243
+ family: scenario.family,
244
+ status: "aborted-pool",
245
+ attempts: 0,
246
+ durationMs: 0,
247
+ detail: `pooled token budget exhausted (${tokensSpent}/${options.poolTokenCeiling})`,
248
+ });
249
+ continue;
250
+ }
251
+ if (!available(scenario.harness)) {
252
+ missing.add(scenario.harness);
253
+ results.push({
254
+ scenario: scenario.id,
255
+ family: scenario.family,
256
+ status: "skipped-harness-unavailable",
257
+ attempts: 0,
258
+ durationMs: 0,
259
+ detail: `harness CLI "${scenario.harness}" not found on PATH`,
260
+ });
261
+ continue;
262
+ }
263
+ const started = now();
264
+ let fixture;
265
+ try {
266
+ fixture = await buildFixture(scenario);
267
+ }
268
+ catch (error) {
269
+ results.push({
270
+ scenario: scenario.id,
271
+ family: scenario.family,
272
+ status: "failed",
273
+ attempts: 0,
274
+ durationMs: 0,
275
+ detail: `fixture setup failed: ${error instanceof Error ? error.message : String(error)}`,
276
+ });
277
+ continue;
278
+ }
279
+ let status = "failed";
280
+ let usage;
281
+ let detail = "";
282
+ let attempts = 0;
283
+ const wallClockMs = (scenario.budget?.wallClockS ?? 300) * 1000;
284
+ try {
285
+ for (let attempt = 1; attempt <= 2; attempt += 1) {
286
+ attempts = attempt;
287
+ const command = expandCommand(scenario.command, scenario.prompt);
288
+ const result = await options.runShell(fixture.dir, command, wallClockMs);
289
+ usage = extractUsage(result.stdout) ?? usage;
290
+ const expectationFailures = evaluateExpectations(fixture.dir, scenario.expected, result);
291
+ const transport = isTransportFailure(result);
292
+ if (expectationFailures.length === 0 && !transport) {
293
+ status = attempt === 1 ? "passed" : "flaky-suspect";
294
+ detail =
295
+ attempt === 1
296
+ ? "outcome matched expectations"
297
+ : "passed on retry after transport failure";
298
+ break;
299
+ }
300
+ if (result.timedOut) {
301
+ status = "aborted-timeout";
302
+ detail = `wall clock exceeded ${scenario.budget?.wallClockS ?? 300}s`;
303
+ }
304
+ else if (transport) {
305
+ status = "failed";
306
+ detail = `harness transport failure (exit ${result.exitCode}${result.signal ? `, signal ${result.signal}` : ""})`;
307
+ }
308
+ else {
309
+ status = "failed";
310
+ detail = expectationFailures.join("; ");
311
+ }
312
+ if (attempt === 2)
313
+ break;
314
+ if (!transport && status === "failed" && !result.timedOut)
315
+ break; // expectation failure: no retry
316
+ }
317
+ }
318
+ finally {
319
+ fixture?.cleanup();
320
+ }
321
+ const durationMs = Math.max(0, now().getTime() - started.getTime());
322
+ if (usage)
323
+ tokensSpent += usage.tokensIn + usage.tokensOut;
324
+ if (usage &&
325
+ scenario.budget?.tokenCeiling !== undefined &&
326
+ usage.tokensIn + usage.tokensOut > scenario.budget.tokenCeiling) {
327
+ status = "aborted-budget";
328
+ detail = `token usage ${usage.tokensIn + usage.tokensOut} exceeded per-scenario ceiling ${scenario.budget.tokenCeiling}`;
329
+ }
330
+ results.push({
331
+ scenario: scenario.id,
332
+ family: scenario.family,
333
+ status,
334
+ attempts,
335
+ durationMs,
336
+ usage,
337
+ detail,
338
+ });
339
+ }
340
+ const finishedAt = now().toISOString();
341
+ return {
342
+ schema: "aiw.scenario-results/v1",
343
+ harness: [...new Set(options.scenarios.map((scenario) => scenario.harness))].join(","),
344
+ startedAt,
345
+ finishedAt,
346
+ poolBudget: options.poolTokenCeiling !== undefined
347
+ ? { tokenCeiling: options.poolTokenCeiling, tokensSpent }
348
+ : undefined,
349
+ missingHarnesses: [...missing].sort(),
350
+ results,
351
+ };
352
+ }
353
+ export function defaultShellRunner(cwd, command, timeoutMs) {
354
+ return new Promise((resolvePromise) => {
355
+ const child = spawn("sh", ["-c", command], { cwd, timeout: timeoutMs, env: process.env });
356
+ let stdout = "";
357
+ let stderr = "";
358
+ child.stdout.on("data", (chunk) => {
359
+ stdout += String(chunk);
360
+ });
361
+ child.stderr.on("data", (chunk) => {
362
+ stderr += String(chunk);
363
+ });
364
+ const finish = (result) => resolvePromise(result);
365
+ child.on("error", (error) => finish({ exitCode: 255, stdout, stderr: `${stderr}${error}` }));
366
+ child.on("close", (code, signal) => finish({
367
+ exitCode: code ?? (signal ? 255 : 1),
368
+ stdout,
369
+ stderr,
370
+ timedOut: Boolean(signal && signal === "SIGTERM"),
371
+ signal: signal ?? undefined,
372
+ }));
373
+ });
374
+ }
@@ -21,6 +21,9 @@ export type OrchestrationTaskResult = {
21
21
  worktree?: string;
22
22
  baseCommit?: string;
23
23
  finalCommit?: string;
24
+ /** OPP-017 execution evidence. */
25
+ durationMs?: number;
26
+ diffStat?: string;
24
27
  };
25
28
  export type IsolationOptions = {
26
29
  /** Repository root the plan runs in. */
@@ -41,6 +44,23 @@ export declare function createTaskWorktrees(root: string, aiw: string, plan: Orc
41
44
  }>;
42
45
  /** Final commit recorded from an isolated worktree (evidence for the checkpoint). */
43
46
  export declare function worktreeFinalCommit(worktree: string): string | undefined;
47
+ /**
48
+ * OPP-017 integration forecast: wave-ordered merge preview for the isolated
49
+ * task branches against the current branch. Deterministic `git merge-tree`
50
+ * dry runs — nothing is merged. Conflicts surface per task before any human
51
+ * merge is attempted.
52
+ */
53
+ export type IntegrationForecastLine = {
54
+ wave: number;
55
+ taskId: string;
56
+ branch: string;
57
+ state: "clean" | "conflict" | "missing" | "identical";
58
+ conflicts?: string[];
59
+ };
60
+ export declare function currentBranch(root: string): string;
61
+ export declare function forecastIntegration(root: string, plan: OrchestrationPlan, results: OrchestrationTaskResult[]): IntegrationForecastLine[];
62
+ /** Parse a serialized orchestration checkpoint back into task results. */
63
+ export declare function parseOrchestrationResults(content: string): OrchestrationTaskResult[];
44
64
  /** Remove isolated worktrees; failed-task worktrees are kept unless forced. */
45
65
  export declare function removeWorktrees(root: string, paths: Iterable<string>, options?: {
46
66
  force?: boolean;
@@ -64,4 +84,4 @@ export declare function executeOrchestrationPlan(plan: OrchestrationPlan, orches
64
84
  }): Promise<OrchestrationTaskResult[]>;
65
85
  export declare function validateOrchestrationParallelism(value: number): number;
66
86
  export declare function serializeOrchestrationPreview(plan: OrchestrationPlan): string;
67
- export declare function serializeOrchestrationResults(plan: OrchestrationPlan, results: OrchestrationTaskResult[]): string;
87
+ export declare function serializeOrchestrationResults(plan: OrchestrationPlan, results: OrchestrationTaskResult[], wallClockMs?: number): string;