@multileaf/ai-workflow 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/blocking-reasons.d.ts +1 -1
- package/dist/blocking-reasons.js +12 -0
- package/dist/eval/scenarios.d.ts +100 -0
- package/dist/eval/scenarios.js +374 -0
- package/dist/orchestration.d.ts +21 -1
- package/dist/orchestration.js +118 -5
- package/dist/workflow.js +103 -3
- package/docs/backlog.md +2 -2
- package/docs/continuation-inventory.md +2 -0
- package/docs/opportunities/OPP-016-behavioral-eval-scenarios.md +1 -1
- package/docs/opportunities/OPP-017-orchestration-depth.md +1 -1
- package/docs/orchestration.md +16 -0
- package/docs/quality.md +15 -0
- package/package.json +1 -1
- package/resources/evals/README.md +59 -0
- package/resources/evals/scenarios/approval-boundary-full.yml +42 -0
- package/resources/evals/scenarios/bugfix-process-entry.yml +41 -0
- package/resources/evals/scenarios/light-lane-skips-durable-artifacts.yml +40 -0
- package/resources/evals/scenarios/rejection-regate.yml +74 -0
- package/resources/evals/scenarios/risk-escalation-high-risk-area.yml +45 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,24 @@ All notable changes to this project are documented in this file. The format foll
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.15.0] - 2026-11-04
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Behavioral evaluation phase 2 (OPP-016): the scenario registry in `resources/evals/scenarios/` — five fixture scenarios covering the core behavioral contract (approval boundary, light-lane artifact skip, risk escalation, bugfix process entry, rejection → re-gate) — replayed against real harness CLIs with `aiw eval run --scenarios`. Deterministic state-outcome assertions only (no prose, no LLM-as-judge); missing harness CLIs mark scenarios `skipped-harness-unavailable` (never silent passes); one retry on transport failure reports `flaky-suspect`; per-scenario wall-clock and token ceilings plus pooled nightly budgets from `.aiw/budgets.yml` (`evals.tokenCeiling`); usage sidecar records tokens only when harnesses report them. Results serialize as `aiw.scenario-results/v1` to `.aiw/generated/evals/`. Advisory-first: report-only for two release cycles before the approval-boundary family gates (ratchet declared up front). The registry doubles as the canonical behavioral contract examples (`resources/evals/README.md`).
|
|
12
|
+
- New `behavioral-evals` CI workflow: nightly + PR (registry/runner paths), advisory mode.
|
|
13
|
+
|
|
14
|
+
## [0.14.0] - 2026-11-04
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
- Orchestration depth (OPP-017): `aiw orchestrate integrate --plan=<path>` prints the wave-ordered integration forecast — a deterministic `git merge-tree` dry run per isolated task branch against the current branch, reporting conflicts before any human merge. Isolated executions now record per-task duration, diff stat, and plan wall-clock time in the checkpoint, and `aiw doctor` reports leaked managed worktrees with the cleanup continuation (`worktrees_leaked` typed reason).
|
|
19
|
+
- `docs/orchestration.md` gains the per-harness worktree honor matrix and the minimal adapter contract for `task.worktree`.
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
|
|
23
|
+
- `orchestrate integrate` works before option parsing (previously subcommand form collided with `--plan` validation).
|
|
24
|
+
|
|
7
25
|
## [0.13.0] - 2026-11-03
|
|
8
26
|
|
|
9
27
|
### Added
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* command resolves the block it was printed for. See ADR-001 and
|
|
8
8
|
* docs/opportunities/OPP-004-exact-continuation-commands.md.
|
|
9
9
|
*/
|
|
10
|
-
export declare const BLOCKING_REASON_CODES: readonly ["gate_evidence_missing", "gate_incomplete", "stage_prerequisite_pending", "stage_prerequisite_unresolved", "stage_awaiting_approval", "stage_already_approved", "approval_evidence_missing", "approval_stale", "approval_gate_missing", "lane_escalation_required", "light_evidence_missing", "workflow_still_active", "workflow_not_active", "workflow_complete", "installation_required", "state_mismatch", "context_budget_exceeded", "stage_not_in_process", "migration_required"];
|
|
10
|
+
export declare const BLOCKING_REASON_CODES: readonly ["gate_evidence_missing", "gate_incomplete", "stage_prerequisite_pending", "stage_prerequisite_unresolved", "stage_awaiting_approval", "stage_already_approved", "approval_evidence_missing", "approval_stale", "approval_gate_missing", "lane_escalation_required", "light_evidence_missing", "workflow_still_active", "workflow_not_active", "workflow_complete", "installation_required", "state_mismatch", "context_budget_exceeded", "stage_not_in_process", "migration_required", "worktrees_leaked", "orchestration_not_executed"];
|
|
11
11
|
export type BlockingReasonCode = (typeof BLOCKING_REASON_CODES)[number];
|
|
12
12
|
/** How the user or agent gets past a block. Exactly one of three kinds. */
|
|
13
13
|
export type BlockingContinuation =
|
package/dist/blocking-reasons.js
CHANGED
|
@@ -27,6 +27,8 @@ export const BLOCKING_REASON_CODES = [
|
|
|
27
27
|
"context_budget_exceeded",
|
|
28
28
|
"stage_not_in_process",
|
|
29
29
|
"migration_required",
|
|
30
|
+
"worktrees_leaked",
|
|
31
|
+
"orchestration_not_executed",
|
|
30
32
|
];
|
|
31
33
|
function command(commandText) {
|
|
32
34
|
return { kind: "command", command: commandText };
|
|
@@ -160,6 +162,16 @@ export const BLOCKING_REASON_INVENTORY = {
|
|
|
160
162
|
description: "Legacy pre-ownership state detected; review the migration plan before continuing.",
|
|
161
163
|
continuation: () => command("aiw migrate --dry-run"),
|
|
162
164
|
},
|
|
165
|
+
worktrees_leaked: {
|
|
166
|
+
code: "worktrees_leaked",
|
|
167
|
+
description: "Managed orchestration worktrees remain on disk after their plan finished; clean them up.",
|
|
168
|
+
continuation: () => action("Run `aiw orchestrate --cleanup --execute` to remove them."),
|
|
169
|
+
},
|
|
170
|
+
orchestration_not_executed: {
|
|
171
|
+
code: "orchestration_not_executed",
|
|
172
|
+
description: "The plan has not been executed yet; there is no checkpoint to integrate.",
|
|
173
|
+
continuation: () => action("Run `aiw orchestrate --plan=<path> --execute` first; integration forecasts need the checkpoint evidence."),
|
|
174
|
+
},
|
|
163
175
|
};
|
|
164
176
|
export function getBlockingReasonDefinition(code) {
|
|
165
177
|
const definition = BLOCKING_REASON_INVENTORY[code];
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
export type ScenarioStatus = "passed" | "failed" | "skipped-harness-unavailable" | "flaky-suspect" | "aborted-timeout" | "aborted-budget" | "aborted-pool";
|
|
2
|
+
export type ScenarioExpectation = {
|
|
3
|
+
exitCode?: number;
|
|
4
|
+
files?: Record<string, {
|
|
5
|
+
contains?: string;
|
|
6
|
+
notContains?: string;
|
|
7
|
+
absent?: boolean;
|
|
8
|
+
matches?: string;
|
|
9
|
+
}>;
|
|
10
|
+
/** Directory-level assertions (e.g. durable artifacts a light lane must skip). */
|
|
11
|
+
dirs?: Record<string, {
|
|
12
|
+
absent?: boolean;
|
|
13
|
+
}>;
|
|
14
|
+
};
|
|
15
|
+
export type ScenarioDefinition = {
|
|
16
|
+
schema: number;
|
|
17
|
+
id: string;
|
|
18
|
+
family: string;
|
|
19
|
+
description: string;
|
|
20
|
+
/** Harness CLI id (e.g. codex, claude, gemini) — availability is probed on PATH. */
|
|
21
|
+
harness: string;
|
|
22
|
+
/** Shell command template; must contain the {prompt} placeholder. */
|
|
23
|
+
command: string;
|
|
24
|
+
prompt: string;
|
|
25
|
+
fixture: {
|
|
26
|
+
files: {
|
|
27
|
+
path: string;
|
|
28
|
+
content: string;
|
|
29
|
+
}[];
|
|
30
|
+
setup?: string[];
|
|
31
|
+
git?: boolean;
|
|
32
|
+
};
|
|
33
|
+
expected: ScenarioExpectation;
|
|
34
|
+
budget?: {
|
|
35
|
+
wallClockS?: number;
|
|
36
|
+
tokenCeiling?: number;
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
export type ScenarioUsage = {
|
|
40
|
+
tokensIn: number;
|
|
41
|
+
tokensOut: number;
|
|
42
|
+
};
|
|
43
|
+
export type ScenarioResult = {
|
|
44
|
+
scenario: string;
|
|
45
|
+
family: string;
|
|
46
|
+
status: ScenarioStatus;
|
|
47
|
+
attempts: number;
|
|
48
|
+
durationMs: number;
|
|
49
|
+
usage?: ScenarioUsage;
|
|
50
|
+
detail: string;
|
|
51
|
+
};
|
|
52
|
+
export type ScenarioRunReport = {
|
|
53
|
+
schema: "aiw.scenario-results/v1";
|
|
54
|
+
harness: string;
|
|
55
|
+
startedAt: string;
|
|
56
|
+
finishedAt: string;
|
|
57
|
+
poolBudget?: {
|
|
58
|
+
tokenCeiling: number;
|
|
59
|
+
tokensSpent: number;
|
|
60
|
+
};
|
|
61
|
+
missingHarnesses: string[];
|
|
62
|
+
results: ScenarioResult[];
|
|
63
|
+
};
|
|
64
|
+
export type ShellResult = {
|
|
65
|
+
exitCode: number;
|
|
66
|
+
stdout: string;
|
|
67
|
+
stderr: string;
|
|
68
|
+
timedOut?: boolean;
|
|
69
|
+
signal?: string;
|
|
70
|
+
};
|
|
71
|
+
export type ShellRunner = (cwd: string, command: string, timeoutMs: number) => Promise<ShellResult>;
|
|
72
|
+
export declare function parseScenarioRegistry(entries: {
|
|
73
|
+
path: string;
|
|
74
|
+
content: string;
|
|
75
|
+
}[]): ScenarioDefinition[];
|
|
76
|
+
/** Replaces {prompt} with a safely single-quoted shell literal. */
|
|
77
|
+
export declare function expandCommand(template: string, prompt: string): string;
|
|
78
|
+
/**
|
|
79
|
+
* Best-effort usage sidecar: parses harness stdout for known usage shapes
|
|
80
|
+
* (codex `token_usage`, claude `usage`). Absence returns undefined — usage is
|
|
81
|
+
* never fabricated.
|
|
82
|
+
*/
|
|
83
|
+
export declare function extractUsage(output: string): ScenarioUsage | undefined;
|
|
84
|
+
/** PATH probe for a harness CLI binary; injectable for tests. */
|
|
85
|
+
export declare function isHarnessAvailable(harness: string, pathEnv?: string): boolean;
|
|
86
|
+
export declare function outcomeFingerprint(report: ScenarioRunReport): string[];
|
|
87
|
+
export declare function serializeScenarioResults(report: ScenarioRunReport): string;
|
|
88
|
+
export type Fixture = {
|
|
89
|
+
dir: string;
|
|
90
|
+
cleanup: () => void;
|
|
91
|
+
};
|
|
92
|
+
export declare function runScenarios(options: {
|
|
93
|
+
scenarios: ScenarioDefinition[];
|
|
94
|
+
runShell: ShellRunner;
|
|
95
|
+
buildFixture?: (scenario: ScenarioDefinition) => Promise<Fixture>;
|
|
96
|
+
isAvailable?: (harness: string) => boolean;
|
|
97
|
+
poolTokenCeiling?: number;
|
|
98
|
+
now?: () => Date;
|
|
99
|
+
}): Promise<ScenarioRunReport>;
|
|
100
|
+
export declare function defaultShellRunner(cwd: string, command: string, timeoutMs: number): Promise<ShellResult>;
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OPP-016: Behavioral evaluation phase 2 — the scenario registry replayed
|
|
3
|
+
* against real harness CLIs.
|
|
4
|
+
*
|
|
5
|
+
* OPP-011 proves generated artifacts *say* the right things (text invariants);
|
|
6
|
+
* this module proves agents *do* them: each scenario declares a fixture
|
|
7
|
+
* repository, a bounded harness command, and deterministic state-outcome
|
|
8
|
+
* expectations (exit code + `.aiw/` state files). No prose assertions, no
|
|
9
|
+
* LLM-as-judge — the layer stays deterministic where it judges.
|
|
10
|
+
*
|
|
11
|
+
* Trust boundaries honored (NFR-3): a missing harness CLI yields an explicit
|
|
12
|
+
* `skipped-harness-unavailable` marker (never a silent pass); token usage is
|
|
13
|
+
* recorded only when the harness reports it (never fabricated); scenarios are
|
|
14
|
+
* advisory-first — nothing in this module gates a release until the declared
|
|
15
|
+
* two-cycle ratchet elapses.
|
|
16
|
+
*/
|
|
17
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
18
|
+
import { tmpdir } from "node:os";
|
|
19
|
+
import { join, resolve } from "node:path";
|
|
20
|
+
import { spawn } from "node:child_process";
|
|
21
|
+
import { parse as parseYaml } from "yaml";
|
|
22
|
+
const SCENARIO_ID_PATTERN = /^[a-z0-9][a-z0-9-]*$/;
|
|
23
|
+
export function parseScenarioRegistry(entries) {
|
|
24
|
+
const scenarios = [];
|
|
25
|
+
for (const entry of entries) {
|
|
26
|
+
const value = parseYaml(entry.content);
|
|
27
|
+
if (!value || typeof value !== "object")
|
|
28
|
+
throw new Error(`Scenario ${entry.path} is not a YAML mapping.`);
|
|
29
|
+
const scenario = value;
|
|
30
|
+
if (scenario.schema !== 1)
|
|
31
|
+
throw new Error(`Scenario ${entry.path} must declare schema: 1.`);
|
|
32
|
+
const id = String(scenario.id ?? "");
|
|
33
|
+
if (!SCENARIO_ID_PATTERN.test(id))
|
|
34
|
+
throw new Error(`Scenario ${entry.path} has an invalid id: ${JSON.stringify(id)}`);
|
|
35
|
+
if (scenarios.some((existing) => existing.id === id))
|
|
36
|
+
throw new Error(`Duplicate scenario id: ${id}`);
|
|
37
|
+
const command = String(scenario.command ?? "");
|
|
38
|
+
if (!command.includes("{prompt}"))
|
|
39
|
+
throw new Error(`Scenario ${id} command must contain the {prompt} placeholder.`);
|
|
40
|
+
const prompt = String(scenario.prompt ?? "");
|
|
41
|
+
if (!prompt.trim())
|
|
42
|
+
throw new Error(`Scenario ${id} has an empty prompt.`);
|
|
43
|
+
const fixture = (scenario.fixture ?? {});
|
|
44
|
+
const files = Array.isArray(fixture.files) ? fixture.files : [];
|
|
45
|
+
for (const file of files) {
|
|
46
|
+
const path = String(file.path ?? "");
|
|
47
|
+
if (path.includes("..") || resolve("/", path) === "/")
|
|
48
|
+
throw new Error(`Scenario ${id} fixture path escapes the fixture: ${path}`);
|
|
49
|
+
}
|
|
50
|
+
scenarios.push({
|
|
51
|
+
schema: 1,
|
|
52
|
+
id,
|
|
53
|
+
family: String(scenario.family ?? "general"),
|
|
54
|
+
description: String(scenario.description ?? ""),
|
|
55
|
+
harness: String(scenario.harness ?? ""),
|
|
56
|
+
command,
|
|
57
|
+
prompt,
|
|
58
|
+
fixture: {
|
|
59
|
+
files: files.map((file) => ({
|
|
60
|
+
path: String(file.path),
|
|
61
|
+
content: String(file.content ?? ""),
|
|
62
|
+
})),
|
|
63
|
+
setup: Array.isArray(fixture.setup) ? fixture.setup.map(String) : [],
|
|
64
|
+
git: fixture.git !== false,
|
|
65
|
+
},
|
|
66
|
+
expected: (scenario.expected ?? {}),
|
|
67
|
+
budget: (scenario.budget ?? {}),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
return scenarios;
|
|
71
|
+
}
|
|
72
|
+
/** Replaces {prompt} with a safely single-quoted shell literal. */
|
|
73
|
+
export function expandCommand(template, prompt) {
|
|
74
|
+
const quoted = `'${prompt.replaceAll("'", "'\\''")}'`;
|
|
75
|
+
return template.replaceAll("{prompt}", quoted);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Best-effort usage sidecar: parses harness stdout for known usage shapes
|
|
79
|
+
* (codex `token_usage`, claude `usage`). Absence returns undefined — usage is
|
|
80
|
+
* never fabricated.
|
|
81
|
+
*/
|
|
82
|
+
export function extractUsage(output) {
|
|
83
|
+
let tokensIn;
|
|
84
|
+
let tokensOut;
|
|
85
|
+
for (const line of output.split("\n")) {
|
|
86
|
+
const trimmed = line.trim();
|
|
87
|
+
if (!trimmed.startsWith("{"))
|
|
88
|
+
continue;
|
|
89
|
+
let parsed;
|
|
90
|
+
try {
|
|
91
|
+
parsed = JSON.parse(trimmed);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (!parsed || typeof parsed !== "object")
|
|
97
|
+
continue;
|
|
98
|
+
const usage = parsed.token_usage ??
|
|
99
|
+
parsed.usage;
|
|
100
|
+
if (!usage || typeof usage !== "object")
|
|
101
|
+
continue;
|
|
102
|
+
const input = Number(usage.input_tokens);
|
|
103
|
+
const output_ = Number(usage.output_tokens);
|
|
104
|
+
if (Number.isFinite(input) && input >= 0)
|
|
105
|
+
tokensIn = (tokensIn ?? 0) + input;
|
|
106
|
+
if (Number.isFinite(output_) && output_ >= 0)
|
|
107
|
+
tokensOut = (tokensOut ?? 0) + output_;
|
|
108
|
+
}
|
|
109
|
+
if (tokensIn === undefined && tokensOut === undefined)
|
|
110
|
+
return undefined;
|
|
111
|
+
return { tokensIn: tokensIn ?? 0, tokensOut: tokensOut ?? 0 };
|
|
112
|
+
}
|
|
113
|
+
/** PATH probe for a harness CLI binary; injectable for tests. */
|
|
114
|
+
export function isHarnessAvailable(harness, pathEnv) {
|
|
115
|
+
if (!harness)
|
|
116
|
+
return false;
|
|
117
|
+
const path = pathEnv ?? process.env.PATH ?? "";
|
|
118
|
+
for (const dir of path.split(":")) {
|
|
119
|
+
if (!dir)
|
|
120
|
+
continue;
|
|
121
|
+
if (existsSync(join(dir, harness)))
|
|
122
|
+
return true;
|
|
123
|
+
}
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
export function outcomeFingerprint(report) {
|
|
127
|
+
return report.results
|
|
128
|
+
.map((result) => `${result.scenario}:${result.status}`)
|
|
129
|
+
.sort((a, b) => a.localeCompare(b));
|
|
130
|
+
}
|
|
131
|
+
function yamlScalar(value) {
|
|
132
|
+
return /^[A-Za-z0-9][A-Za-z0-9._ -]*$/.test(value) ? value : JSON.stringify(value);
|
|
133
|
+
}
|
|
134
|
+
export function serializeScenarioResults(report) {
|
|
135
|
+
const counts = new Map();
|
|
136
|
+
for (const result of report.results)
|
|
137
|
+
counts.set(result.status, (counts.get(result.status) ?? 0) + 1);
|
|
138
|
+
const lines = [
|
|
139
|
+
"schema: aiw.scenario-results/v1",
|
|
140
|
+
`harness: ${JSON.stringify(report.harness)}`,
|
|
141
|
+
`startedAt: ${JSON.stringify(report.startedAt)}`,
|
|
142
|
+
`finishedAt: ${JSON.stringify(report.finishedAt)}`,
|
|
143
|
+
];
|
|
144
|
+
if (report.poolBudget)
|
|
145
|
+
lines.push(`poolBudget:\\n tokenCeiling: ${report.poolBudget.tokenCeiling}\\n tokensSpent: ${report.poolBudget.tokensSpent}`);
|
|
146
|
+
lines.push(`missingHarnesses: ${report.missingHarnesses.length ? JSON.stringify(report.missingHarnesses) : "[]"}`);
|
|
147
|
+
lines.push("results:");
|
|
148
|
+
for (const result of [...report.results].sort((a, b) => a.scenario.localeCompare(b.scenario))) {
|
|
149
|
+
lines.push(` - scenario: ${yamlScalar(result.scenario)}`);
|
|
150
|
+
lines.push(` family: ${yamlScalar(result.family)}`);
|
|
151
|
+
lines.push(` status: ${result.status}`);
|
|
152
|
+
lines.push(` attempts: ${result.attempts}`);
|
|
153
|
+
lines.push(` durationMs: ${result.durationMs}`);
|
|
154
|
+
if (result.usage)
|
|
155
|
+
lines.push(` usage:\\n tokensIn: ${result.usage.tokensIn}\\n tokensOut: ${result.usage.tokensOut}`);
|
|
156
|
+
lines.push(` detail: ${JSON.stringify(result.detail)}`);
|
|
157
|
+
}
|
|
158
|
+
lines.push("summary:");
|
|
159
|
+
for (const [status, count] of [...counts.entries()].sort((a, b) => a[0].localeCompare(b[0])))
|
|
160
|
+
lines.push(` ${status}: ${count}`);
|
|
161
|
+
return `${lines.join("\\n")}\\n`;
|
|
162
|
+
}
|
|
163
|
+
/** Transport-level failure: timeout, signal kill, or harness-specific crash codes. */
|
|
164
|
+
function isTransportFailure(result) {
|
|
165
|
+
return Boolean(result.timedOut) || Boolean(result.signal) || result.exitCode >= 250;
|
|
166
|
+
}
|
|
167
|
+
function evaluateExpectations(fixtureDir, expected, result) {
|
|
168
|
+
const failures = [];
|
|
169
|
+
const wantedExit = expected.exitCode ?? 0;
|
|
170
|
+
if (result.exitCode !== wantedExit)
|
|
171
|
+
failures.push(`exit code ${result.exitCode} (expected ${wantedExit})`);
|
|
172
|
+
for (const [path, assertion] of Object.entries(expected.files ?? {})) {
|
|
173
|
+
const absolute = join(fixtureDir, path);
|
|
174
|
+
if (assertion.absent) {
|
|
175
|
+
if (existsSync(absolute))
|
|
176
|
+
failures.push(`${path} exists (expected absent)`);
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (!existsSync(absolute)) {
|
|
180
|
+
failures.push(`${path} missing`);
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
const content = readFileSync(absolute, "utf8");
|
|
184
|
+
if (assertion.contains && !content.includes(assertion.contains))
|
|
185
|
+
failures.push(`${path} does not contain ${JSON.stringify(assertion.contains)}`);
|
|
186
|
+
if (assertion.notContains && content.includes(assertion.notContains))
|
|
187
|
+
failures.push(`${path} must not contain ${JSON.stringify(assertion.notContains)}`);
|
|
188
|
+
if (assertion.matches && !new RegExp(assertion.matches, "m").test(content))
|
|
189
|
+
failures.push(`${path} does not match /${assertion.matches}/`);
|
|
190
|
+
}
|
|
191
|
+
for (const [path, assertion] of Object.entries(expected.dirs ?? {})) {
|
|
192
|
+
const absolute = join(fixtureDir, path);
|
|
193
|
+
if (assertion.absent && existsSync(absolute))
|
|
194
|
+
failures.push(`${path}/ exists (expected absent)`);
|
|
195
|
+
}
|
|
196
|
+
return failures;
|
|
197
|
+
}
|
|
198
|
+
function buildFixtureDefault(scenario, runShell) {
|
|
199
|
+
const root = mkdtempSync(join(tmpdir(), "aiw-scenario-"));
|
|
200
|
+
const dir = join(root, scenario.id);
|
|
201
|
+
mkdirSync(dir, { recursive: true });
|
|
202
|
+
for (const file of scenario.fixture.files) {
|
|
203
|
+
const absolute = join(dir, file.path);
|
|
204
|
+
mkdirSync(join(absolute, ".."), { recursive: true });
|
|
205
|
+
writeFileSync(absolute, file.content);
|
|
206
|
+
}
|
|
207
|
+
const prepare = async () => {
|
|
208
|
+
if (scenario.fixture.git !== false) {
|
|
209
|
+
const init = await runShell(dir, "git init -q", 15000);
|
|
210
|
+
if (init.exitCode !== 0)
|
|
211
|
+
throw new Error(`git init failed: ${init.stderr.slice(0, 200)}`);
|
|
212
|
+
const config = await runShell(dir, 'git config user.email "scenario@aiw.local" && git config user.name scenario', 15000);
|
|
213
|
+
if (config.exitCode !== 0)
|
|
214
|
+
throw new Error(`git config failed: ${config.stderr.slice(0, 200)}`);
|
|
215
|
+
const commit = await runShell(dir, "git add -A && git commit -qm fixture", 15000);
|
|
216
|
+
if (commit.exitCode !== 0)
|
|
217
|
+
throw new Error(`git commit failed: ${commit.stderr.slice(0, 200)}`);
|
|
218
|
+
}
|
|
219
|
+
for (const command of scenario.fixture.setup ?? []) {
|
|
220
|
+
const result = await runShell(dir, command, 120000);
|
|
221
|
+
if (result.exitCode !== 0)
|
|
222
|
+
throw new Error(`setup command failed (exit ${result.exitCode}): ${command} — ${result.stderr.slice(0, 200)}`);
|
|
223
|
+
}
|
|
224
|
+
};
|
|
225
|
+
return prepare().then(() => ({ dir, cleanup: () => rmSync(root, { recursive: true, force: true }) }), (error) => {
|
|
226
|
+
rmSync(root, { recursive: true, force: true });
|
|
227
|
+
throw error;
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
export async function runScenarios(options) {
|
|
231
|
+
const available = options.isAvailable ?? ((harness) => isHarnessAvailable(harness));
|
|
232
|
+
const buildFixture = options.buildFixture ??
|
|
233
|
+
((scenario) => buildFixtureDefault(scenario, options.runShell));
|
|
234
|
+
const now = options.now ?? (() => new Date());
|
|
235
|
+
const startedAt = now().toISOString();
|
|
236
|
+
const results = [];
|
|
237
|
+
const missing = new Set();
|
|
238
|
+
let tokensSpent = 0;
|
|
239
|
+
for (const scenario of options.scenarios) {
|
|
240
|
+
if (options.poolTokenCeiling !== undefined && tokensSpent >= options.poolTokenCeiling) {
|
|
241
|
+
results.push({
|
|
242
|
+
scenario: scenario.id,
|
|
243
|
+
family: scenario.family,
|
|
244
|
+
status: "aborted-pool",
|
|
245
|
+
attempts: 0,
|
|
246
|
+
durationMs: 0,
|
|
247
|
+
detail: `pooled token budget exhausted (${tokensSpent}/${options.poolTokenCeiling})`,
|
|
248
|
+
});
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
if (!available(scenario.harness)) {
|
|
252
|
+
missing.add(scenario.harness);
|
|
253
|
+
results.push({
|
|
254
|
+
scenario: scenario.id,
|
|
255
|
+
family: scenario.family,
|
|
256
|
+
status: "skipped-harness-unavailable",
|
|
257
|
+
attempts: 0,
|
|
258
|
+
durationMs: 0,
|
|
259
|
+
detail: `harness CLI "${scenario.harness}" not found on PATH`,
|
|
260
|
+
});
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
const started = now();
|
|
264
|
+
let fixture;
|
|
265
|
+
try {
|
|
266
|
+
fixture = await buildFixture(scenario);
|
|
267
|
+
}
|
|
268
|
+
catch (error) {
|
|
269
|
+
results.push({
|
|
270
|
+
scenario: scenario.id,
|
|
271
|
+
family: scenario.family,
|
|
272
|
+
status: "failed",
|
|
273
|
+
attempts: 0,
|
|
274
|
+
durationMs: 0,
|
|
275
|
+
detail: `fixture setup failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
276
|
+
});
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
let status = "failed";
|
|
280
|
+
let usage;
|
|
281
|
+
let detail = "";
|
|
282
|
+
let attempts = 0;
|
|
283
|
+
const wallClockMs = (scenario.budget?.wallClockS ?? 300) * 1000;
|
|
284
|
+
try {
|
|
285
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
286
|
+
attempts = attempt;
|
|
287
|
+
const command = expandCommand(scenario.command, scenario.prompt);
|
|
288
|
+
const result = await options.runShell(fixture.dir, command, wallClockMs);
|
|
289
|
+
usage = extractUsage(result.stdout) ?? usage;
|
|
290
|
+
const expectationFailures = evaluateExpectations(fixture.dir, scenario.expected, result);
|
|
291
|
+
const transport = isTransportFailure(result);
|
|
292
|
+
if (expectationFailures.length === 0 && !transport) {
|
|
293
|
+
status = attempt === 1 ? "passed" : "flaky-suspect";
|
|
294
|
+
detail =
|
|
295
|
+
attempt === 1
|
|
296
|
+
? "outcome matched expectations"
|
|
297
|
+
: "passed on retry after transport failure";
|
|
298
|
+
break;
|
|
299
|
+
}
|
|
300
|
+
if (result.timedOut) {
|
|
301
|
+
status = "aborted-timeout";
|
|
302
|
+
detail = `wall clock exceeded ${scenario.budget?.wallClockS ?? 300}s`;
|
|
303
|
+
}
|
|
304
|
+
else if (transport) {
|
|
305
|
+
status = "failed";
|
|
306
|
+
detail = `harness transport failure (exit ${result.exitCode}${result.signal ? `, signal ${result.signal}` : ""})`;
|
|
307
|
+
}
|
|
308
|
+
else {
|
|
309
|
+
status = "failed";
|
|
310
|
+
detail = expectationFailures.join("; ");
|
|
311
|
+
}
|
|
312
|
+
if (attempt === 2)
|
|
313
|
+
break;
|
|
314
|
+
if (!transport && status === "failed" && !result.timedOut)
|
|
315
|
+
break; // expectation failure: no retry
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
finally {
|
|
319
|
+
fixture?.cleanup();
|
|
320
|
+
}
|
|
321
|
+
const durationMs = Math.max(0, now().getTime() - started.getTime());
|
|
322
|
+
if (usage)
|
|
323
|
+
tokensSpent += usage.tokensIn + usage.tokensOut;
|
|
324
|
+
if (usage &&
|
|
325
|
+
scenario.budget?.tokenCeiling !== undefined &&
|
|
326
|
+
usage.tokensIn + usage.tokensOut > scenario.budget.tokenCeiling) {
|
|
327
|
+
status = "aborted-budget";
|
|
328
|
+
detail = `token usage ${usage.tokensIn + usage.tokensOut} exceeded per-scenario ceiling ${scenario.budget.tokenCeiling}`;
|
|
329
|
+
}
|
|
330
|
+
results.push({
|
|
331
|
+
scenario: scenario.id,
|
|
332
|
+
family: scenario.family,
|
|
333
|
+
status,
|
|
334
|
+
attempts,
|
|
335
|
+
durationMs,
|
|
336
|
+
usage,
|
|
337
|
+
detail,
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
const finishedAt = now().toISOString();
|
|
341
|
+
return {
|
|
342
|
+
schema: "aiw.scenario-results/v1",
|
|
343
|
+
harness: [...new Set(options.scenarios.map((scenario) => scenario.harness))].join(","),
|
|
344
|
+
startedAt,
|
|
345
|
+
finishedAt,
|
|
346
|
+
poolBudget: options.poolTokenCeiling !== undefined
|
|
347
|
+
? { tokenCeiling: options.poolTokenCeiling, tokensSpent }
|
|
348
|
+
: undefined,
|
|
349
|
+
missingHarnesses: [...missing].sort(),
|
|
350
|
+
results,
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
export function defaultShellRunner(cwd, command, timeoutMs) {
|
|
354
|
+
return new Promise((resolvePromise) => {
|
|
355
|
+
const child = spawn("sh", ["-c", command], { cwd, timeout: timeoutMs, env: process.env });
|
|
356
|
+
let stdout = "";
|
|
357
|
+
let stderr = "";
|
|
358
|
+
child.stdout.on("data", (chunk) => {
|
|
359
|
+
stdout += String(chunk);
|
|
360
|
+
});
|
|
361
|
+
child.stderr.on("data", (chunk) => {
|
|
362
|
+
stderr += String(chunk);
|
|
363
|
+
});
|
|
364
|
+
const finish = (result) => resolvePromise(result);
|
|
365
|
+
child.on("error", (error) => finish({ exitCode: 255, stdout, stderr: `${stderr}${error}` }));
|
|
366
|
+
child.on("close", (code, signal) => finish({
|
|
367
|
+
exitCode: code ?? (signal ? 255 : 1),
|
|
368
|
+
stdout,
|
|
369
|
+
stderr,
|
|
370
|
+
timedOut: Boolean(signal && signal === "SIGTERM"),
|
|
371
|
+
signal: signal ?? undefined,
|
|
372
|
+
}));
|
|
373
|
+
});
|
|
374
|
+
}
|
package/dist/orchestration.d.ts
CHANGED
|
@@ -21,6 +21,9 @@ export type OrchestrationTaskResult = {
|
|
|
21
21
|
worktree?: string;
|
|
22
22
|
baseCommit?: string;
|
|
23
23
|
finalCommit?: string;
|
|
24
|
+
/** OPP-017 execution evidence. */
|
|
25
|
+
durationMs?: number;
|
|
26
|
+
diffStat?: string;
|
|
24
27
|
};
|
|
25
28
|
export type IsolationOptions = {
|
|
26
29
|
/** Repository root the plan runs in. */
|
|
@@ -41,6 +44,23 @@ export declare function createTaskWorktrees(root: string, aiw: string, plan: Orc
|
|
|
41
44
|
}>;
|
|
42
45
|
/** Final commit recorded from an isolated worktree (evidence for the checkpoint). */
|
|
43
46
|
export declare function worktreeFinalCommit(worktree: string): string | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* OPP-017 integration forecast: wave-ordered merge preview for the isolated
|
|
49
|
+
* task branches against the current branch. Deterministic `git merge-tree`
|
|
50
|
+
* dry runs — nothing is merged. Conflicts surface per task before any human
|
|
51
|
+
* merge is attempted.
|
|
52
|
+
*/
|
|
53
|
+
export type IntegrationForecastLine = {
|
|
54
|
+
wave: number;
|
|
55
|
+
taskId: string;
|
|
56
|
+
branch: string;
|
|
57
|
+
state: "clean" | "conflict" | "missing" | "identical";
|
|
58
|
+
conflicts?: string[];
|
|
59
|
+
};
|
|
60
|
+
export declare function currentBranch(root: string): string;
|
|
61
|
+
export declare function forecastIntegration(root: string, plan: OrchestrationPlan, results: OrchestrationTaskResult[]): IntegrationForecastLine[];
|
|
62
|
+
/** Parse a serialized orchestration checkpoint back into task results. */
|
|
63
|
+
export declare function parseOrchestrationResults(content: string): OrchestrationTaskResult[];
|
|
44
64
|
/** Remove isolated worktrees; failed-task worktrees are kept unless forced. */
|
|
45
65
|
export declare function removeWorktrees(root: string, paths: Iterable<string>, options?: {
|
|
46
66
|
force?: boolean;
|
|
@@ -64,4 +84,4 @@ export declare function executeOrchestrationPlan(plan: OrchestrationPlan, orches
|
|
|
64
84
|
}): Promise<OrchestrationTaskResult[]>;
|
|
65
85
|
export declare function validateOrchestrationParallelism(value: number): number;
|
|
66
86
|
export declare function serializeOrchestrationPreview(plan: OrchestrationPlan): string;
|
|
67
|
-
export declare function serializeOrchestrationResults(plan: OrchestrationPlan, results: OrchestrationTaskResult[]): string;
|
|
87
|
+
export declare function serializeOrchestrationResults(plan: OrchestrationPlan, results: OrchestrationTaskResult[], wallClockMs?: number): string;
|