@cleocode/playbooks 2026.6.13 → 2026.6.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +3 -1
- package/dist/parser.js +33 -0
- package/dist/runtime.d.ts +101 -1
- package/dist/runtime.js +84 -3
- package/package.json +3 -3
- package/src/__tests__/fixtures/pi-skill-only.cantbook +26 -0
- package/src/__tests__/guarded-deterministic-dispatch.test.ts +225 -0
- package/src/__tests__/inputs-bindings.test.ts +250 -0
- package/src/__tests__/node-allowlist.test.ts +186 -0
- package/src/__tests__/node-profile.test.ts +142 -0
- package/src/__tests__/parser.test.ts +80 -0
- package/src/__tests__/pi-cantbook-e2e.test.ts +294 -0
- package/src/__tests__/rcasd-one-envelope.test.ts +239 -0
- package/src/index.ts +2 -0
- package/src/parser.ts +35 -0
- package/src/runtime.ts +161 -2
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* T11945 (M4) — deterministic in-process e2e: a skill-only `.cantbook` executes
|
|
3
|
+
* THROUGH the PiAgentAdapter when the Pi runner is wired into the production
|
|
4
|
+
* dispatcher.
|
|
5
|
+
*
|
|
6
|
+
* ## What this proves (the M4 keystone)
|
|
7
|
+
*
|
|
8
|
+
* The two production cantbook dispatchers (`buildDefaultDispatcher` in
|
|
9
|
+
* `packages/cleo/src/dispatch/domains/playbook.ts` and `buildGoDispatcher` in
|
|
10
|
+
* `packages/cleo/src/cli/commands/go-ivtr-runner.ts`) now pass
|
|
11
|
+
* `runner: await maybeCreatePiRunner({ system, projectRoot })` into
|
|
12
|
+
* {@link runSkillNodeOrSpawn}. This test reconstructs that EXACT production
|
|
13
|
+
* dispatcher shape (`createToolGuard` + `runSkillNodeOrSpawn` + the
|
|
14
|
+
* `maybeCreatePiRunner` wiring helper) and drives a single skill-only node
|
|
15
|
+
* through the real {@link executePlaybook} runtime state machine.
|
|
16
|
+
*
|
|
17
|
+
* With `CLEO_PI_RUNNER_ENABLED=1` the in-process skill node routes THROUGH the
|
|
18
|
+
* PiAgentAdapter. There is no working LLM backend on this box, so we do NOT
|
|
19
|
+
* assert a live round-trip — we assert the DISPATCH PATH reached the Pi adapter,
|
|
20
|
+
* proven by the **cleo-owned no-credential failure string**
|
|
21
|
+
* (`no credential resolved for system "task-executor"`) emitted by the Cleo
|
|
22
|
+
* streamFn (`createPiStreamFn`, the E9 chokepoint). A pi-ai-internal failure
|
|
23
|
+
* shape would not carry that exact cleo string. The Pi adapter NEVER throws —
|
|
24
|
+
* it returns a contained `{ status: 'failure' }` (pi-agent-adapter.test.ts:197
|
|
25
|
+
* precedent), so the run terminates cleanly.
|
|
26
|
+
*
|
|
27
|
+
* ## Determinism + isolation contract (AC3 · AC4)
|
|
28
|
+
*
|
|
29
|
+
* - IN-PROCESS only — vitest source, NO subprocess (tsx unresolvable in CI).
|
|
30
|
+
* - daemon-OFF — the runtime is the in-memory `node:sqlite` DB, no daemon.
|
|
31
|
+
* - `CLEO_SESSION_ID` is stamped so the Pi runner's ZERO-authority guard passes
|
|
32
|
+
* (production relies on the daemon stamping it; here the test stamps it).
|
|
33
|
+
* - A real `ct-*` skill is materialised under a temp project root's
|
|
34
|
+
* `.agents/skills/` dir so `findSkill` resolves it and the in-process skill
|
|
35
|
+
* path (not subprocess spawn) is taken. `CLEO_PROJECT_ROOT` pins resolution
|
|
36
|
+
* to that temp dir.
|
|
37
|
+
* - `maxIterationsDefault: 1` so the failing node runs exactly once → the run
|
|
38
|
+
* terminates as `exceeded_iteration_cap` with the Pi error in `errorContext`.
|
|
39
|
+
*
|
|
40
|
+
* @epic T10403
|
|
41
|
+
* @task T11761
|
|
42
|
+
* @task T11945
|
|
43
|
+
*/
|
|
44
|
+
|
|
45
|
+
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
46
|
+
import { createRequire } from 'node:module';
|
|
47
|
+
import { tmpdir } from 'node:os';
|
|
48
|
+
import { dirname, join, resolve } from 'node:path';
|
|
49
|
+
import type { DatabaseSync as _DatabaseSyncType } from 'node:sqlite';
|
|
50
|
+
import { fileURLToPath } from 'node:url';
|
|
51
|
+
import {
|
|
52
|
+
type AgentDispatcher,
|
|
53
|
+
type AgentDispatchInput,
|
|
54
|
+
type AgentDispatchResult,
|
|
55
|
+
createToolGuard,
|
|
56
|
+
maybeCreatePiRunner,
|
|
57
|
+
runSkillNodeOrSpawn,
|
|
58
|
+
} from '@cleocode/core/internal';
|
|
59
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
60
|
+
import { parsePlaybook } from '../parser.js';
|
|
61
|
+
import { executePlaybook } from '../runtime.js';
|
|
62
|
+
import { getPlaybookRun } from '../state.js';
|
|
63
|
+
|
|
64
|
+
const _require = createRequire(import.meta.url);
|
|
65
|
+
type DatabaseSync = _DatabaseSyncType;
|
|
66
|
+
const { DatabaseSync } = _require('node:sqlite') as {
|
|
67
|
+
DatabaseSync: new (...args: ConstructorParameters<typeof _DatabaseSyncType>) => DatabaseSync;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
71
|
+
|
|
72
|
+
/** Absolute path to the T889 playbook-tables migration SQL. */
|
|
73
|
+
const MIGRATION_SQL_PATH = resolve(
|
|
74
|
+
__dirname,
|
|
75
|
+
'../../../core/migrations/drizzle-tasks/20260417220000_t889-playbook-tables/migration.sql',
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
/** The skill-only fixture cantbook materialised for this suite. */
|
|
79
|
+
const FIXTURE_PATH = resolve(__dirname, 'fixtures/pi-skill-only.cantbook');
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Cleo-owned Pi failure markers that ONLY appear when a dispatch reached the
|
|
83
|
+
* PiAgentAdapter → Cleo streamFn (E9 chokepoint). A `defaultSkillRunner` would
|
|
84
|
+
* have SUCCEEDED (it does no LLM work), so seeing any of these proves the Pi
|
|
85
|
+
* route was taken:
|
|
86
|
+
* - `no credential resolved for system` — the Cleo streamFn's no-credential
|
|
87
|
+
* terminal error (`createPiStreamFn`), when NO key is reachable.
|
|
88
|
+
* - `Pi loop failed:` — the Cleo `wrapPiCall` containment prefix
|
|
89
|
+
* (`pi-errors.ts`), when a credential DID resolve and the real LLM transport
|
|
90
|
+
* failed (e.g. an expired `sk-ant-oat` token → 401). Either way the call went
|
|
91
|
+
* through the Pi adapter, never `defaultSkillRunner`.
|
|
92
|
+
*
|
|
93
|
+
* Mirrors the `pi-agent-adapter.test.ts:197` precedent
|
|
94
|
+
* (`/no credential resolved for system|pi loop error|error/i`).
|
|
95
|
+
*/
|
|
96
|
+
const PI_DISPATCH_MARKERS = /no credential resolved for system|Pi loop failed:/i;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Apply a multi-statement Drizzle migration file (split on the
|
|
100
|
+
* `--> statement-breakpoint` token). Mirrors starter.e2e.test.ts.
|
|
101
|
+
*/
|
|
102
|
+
function applyMigration(db: DatabaseSync, sql: string): void {
|
|
103
|
+
const statements = sql
|
|
104
|
+
.split(/--> statement-breakpoint/)
|
|
105
|
+
.map((s) => s.trim())
|
|
106
|
+
.filter((s) => s.length > 0);
|
|
107
|
+
for (const stmt of statements) {
|
|
108
|
+
const lines = stmt.split('\n');
|
|
109
|
+
const hasSql = lines.some((l) => l.trim().length > 0 && !l.trim().startsWith('--'));
|
|
110
|
+
if (hasSql) db.exec(stmt);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Materialise a real `ct-task-executor` skill under `<projectRoot>/.agents/skills/`
|
|
116
|
+
* so {@link runSkillNodeOrSpawn}'s `findSkill(agentId, projectRoot)` resolves it
|
|
117
|
+
* via the `project-custom` search scope, taking the in-process skill path.
|
|
118
|
+
*/
|
|
119
|
+
function writeFixtureSkill(projectRoot: string): void {
|
|
120
|
+
const skillDir = join(projectRoot, '.agents', 'skills', 'ct-task-executor');
|
|
121
|
+
mkdirSync(skillDir, { recursive: true });
|
|
122
|
+
writeFileSync(
|
|
123
|
+
join(skillDir, 'SKILL.md'),
|
|
124
|
+
[
|
|
125
|
+
'---',
|
|
126
|
+
'name: ct-task-executor',
|
|
127
|
+
'description: T11945 e2e fixture skill — drives the in-process Pi dispatch path.',
|
|
128
|
+
'---',
|
|
129
|
+
'',
|
|
130
|
+
'Execute the assigned task.',
|
|
131
|
+
'',
|
|
132
|
+
].join('\n'),
|
|
133
|
+
'utf8',
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Build the EXACT production dispatcher shape from `buildDefaultDispatcher`:
|
|
139
|
+
* in-process `ct-*` skill nodes run over a deny-first guarded tool surface via
|
|
140
|
+
* {@link runSkillNodeOrSpawn} with the Pi runner injected; isolation/agent nodes
|
|
141
|
+
* fall back to the (here-unused) subprocess spawn. The `runner` is obtained from
|
|
142
|
+
* the M4 wiring helper {@link maybeCreatePiRunner}, exactly as production does.
|
|
143
|
+
*/
|
|
144
|
+
async function buildProductionShapedDispatcher(projectRoot: string): Promise<AgentDispatcher> {
|
|
145
|
+
const tools = createToolGuard({ allowedRoots: [projectRoot] });
|
|
146
|
+
// The M4 keystone wiring under test — with CLEO_PI_RUNNER_ENABLED=1 this is
|
|
147
|
+
// the Pi `SkillRunner`; with the flag unset it is `undefined`.
|
|
148
|
+
const runner = await maybeCreatePiRunner({ system: 'task-executor', projectRoot });
|
|
149
|
+
|
|
150
|
+
/** Subprocess fallback — never exercised by the skill-only fixture. */
|
|
151
|
+
const spawn = async (input: AgentDispatchInput): Promise<AgentDispatchResult> => ({
|
|
152
|
+
status: 'failure',
|
|
153
|
+
output: {},
|
|
154
|
+
error: `unexpected subprocess spawn for ${input.agentId}`,
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
async dispatch(input: AgentDispatchInput): Promise<AgentDispatchResult> {
|
|
159
|
+
return runSkillNodeOrSpawn(
|
|
160
|
+
{ nodeId: input.nodeId, agentId: input.agentId, context: input.context },
|
|
161
|
+
{
|
|
162
|
+
tools,
|
|
163
|
+
cwd: projectRoot,
|
|
164
|
+
subprocessSpawn: () => spawn(input),
|
|
165
|
+
...(runner !== undefined ? { runner } : {}),
|
|
166
|
+
},
|
|
167
|
+
);
|
|
168
|
+
},
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ---------------------------------------------------------------------------
|
|
173
|
+
|
|
174
|
+
describe('T11945 (M4): skill-only .cantbook routes through the PiAgentAdapter', () => {
|
|
175
|
+
let db: DatabaseSync;
|
|
176
|
+
let projectRoot: string;
|
|
177
|
+
|
|
178
|
+
const SAVED = {
|
|
179
|
+
session: process.env['CLEO_SESSION_ID'],
|
|
180
|
+
flag: process.env['CLEO_PI_RUNNER_ENABLED'],
|
|
181
|
+
projectRoot: process.env['CLEO_PROJECT_ROOT'],
|
|
182
|
+
cleoRoot: process.env['CLEO_ROOT'],
|
|
183
|
+
anthropic: process.env['ANTHROPIC_API_KEY'],
|
|
184
|
+
anthropicOauth: process.env['ANTHROPIC_OAUTH_TOKEN'],
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
beforeEach(() => {
|
|
188
|
+
db = new DatabaseSync(':memory:');
|
|
189
|
+
db.exec('PRAGMA foreign_keys=ON');
|
|
190
|
+
applyMigration(db, readFileSync(MIGRATION_SQL_PATH, 'utf8'));
|
|
191
|
+
|
|
192
|
+
projectRoot = mkdtempSync(join(tmpdir(), 't11945-pi-e2e-'));
|
|
193
|
+
writeFixtureSkill(projectRoot);
|
|
194
|
+
|
|
195
|
+
// Pin skill + project resolution to the temp root (getProjectRoot honours
|
|
196
|
+
// CLEO_ROOT / CLEO_PROJECT_ROOT before any walk).
|
|
197
|
+
process.env['CLEO_PROJECT_ROOT'] = projectRoot;
|
|
198
|
+
process.env['CLEO_ROOT'] = projectRoot;
|
|
199
|
+
// Daemon-stamped identity — the Pi runner fails closed without it (ZERO authority).
|
|
200
|
+
process.env['CLEO_SESSION_ID'] = 'sess-t11945-e2e';
|
|
201
|
+
// No credential reachable → deterministic cleo-owned no-credential failure.
|
|
202
|
+
delete process.env['ANTHROPIC_API_KEY'];
|
|
203
|
+
delete process.env['ANTHROPIC_OAUTH_TOKEN'];
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
afterEach(() => {
|
|
207
|
+
db.close();
|
|
208
|
+
rmSync(projectRoot, { recursive: true, force: true });
|
|
209
|
+
restoreEnv('CLEO_SESSION_ID', SAVED.session);
|
|
210
|
+
restoreEnv('CLEO_PI_RUNNER_ENABLED', SAVED.flag);
|
|
211
|
+
restoreEnv('CLEO_PROJECT_ROOT', SAVED.projectRoot);
|
|
212
|
+
restoreEnv('CLEO_ROOT', SAVED.cleoRoot);
|
|
213
|
+
restoreEnv('ANTHROPIC_API_KEY', SAVED.anthropic);
|
|
214
|
+
restoreEnv('ANTHROPIC_OAUTH_TOKEN', SAVED.anthropicOauth);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it('routes the in-process skill node THROUGH the Pi adapter (proven by the cleo no-credential failure)', async () => {
|
|
218
|
+
process.env['CLEO_PI_RUNNER_ENABLED'] = '1';
|
|
219
|
+
|
|
220
|
+
const { definition, sourceHash } = parsePlaybook(readFileSync(FIXTURE_PATH, 'utf8'));
|
|
221
|
+
expect(definition.name).toBe('pi-skill-only');
|
|
222
|
+
expect(definition.nodes).toHaveLength(1);
|
|
223
|
+
|
|
224
|
+
const dispatcher = await buildProductionShapedDispatcher(projectRoot);
|
|
225
|
+
|
|
226
|
+
const result = await executePlaybook({
|
|
227
|
+
db,
|
|
228
|
+
playbook: definition,
|
|
229
|
+
playbookHash: sourceHash,
|
|
230
|
+
initialContext: { taskId: 'T11945' },
|
|
231
|
+
dispatcher,
|
|
232
|
+
projectRoot,
|
|
233
|
+
// One attempt → the node fails once (no credential) and terminates.
|
|
234
|
+
maxIterationsDefault: 1,
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
// The Pi loop never throws — the run reaches a clean terminal state with the
|
|
238
|
+
// node failure recorded (not an uncaught exception).
|
|
239
|
+
expect(result.terminalStatus).toBe('exceeded_iteration_cap');
|
|
240
|
+
expect(result.exceededNodeId).toBe('execute');
|
|
241
|
+
|
|
242
|
+
// THE PROOF: the failure carries a cleo-owned Pi marker (streamFn
|
|
243
|
+
// no-credential error OR the wrapPiCall containment prefix). These ONLY
|
|
244
|
+
// appear when the dispatch reached the PiAgentAdapter → Cleo streamFn (the
|
|
245
|
+
// E9 chokepoint). A `defaultSkillRunner` would have SUCCEEDED (it does no LLM
|
|
246
|
+
// work, terminalStatus 'completed'), so an `exceeded_iteration_cap` carrying
|
|
247
|
+
// this marker is dispositive that the Pi route ran.
|
|
248
|
+
const errorContext = result.errorContext ?? '';
|
|
249
|
+
expect(errorContext).toMatch(PI_DISPATCH_MARKERS);
|
|
250
|
+
|
|
251
|
+
const run = getPlaybookRun(db, result.runId);
|
|
252
|
+
expect(run?.status).toBe('failed');
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it('with CLEO_PI_RUNNER_ENABLED unset the node stays on defaultSkillRunner (zero behaviour change)', async () => {
|
|
256
|
+
// Flag deliberately NOT set → maybeCreatePiRunner returns undefined →
|
|
257
|
+
// runSkillNodeOrSpawn uses defaultSkillRunner, which resolves the skill and
|
|
258
|
+
// returns success WITHOUT any LLM work. The run completes; no Pi string.
|
|
259
|
+
delete process.env['CLEO_PI_RUNNER_ENABLED'];
|
|
260
|
+
|
|
261
|
+
const { definition, sourceHash } = parsePlaybook(readFileSync(FIXTURE_PATH, 'utf8'));
|
|
262
|
+
const dispatcher = await buildProductionShapedDispatcher(projectRoot);
|
|
263
|
+
|
|
264
|
+
const result = await executePlaybook({
|
|
265
|
+
db,
|
|
266
|
+
playbook: definition,
|
|
267
|
+
playbookHash: sourceHash,
|
|
268
|
+
initialContext: { taskId: 'T11945' },
|
|
269
|
+
dispatcher,
|
|
270
|
+
projectRoot,
|
|
271
|
+
maxIterationsDefault: 1,
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
expect(result.terminalStatus).toBe('completed');
|
|
275
|
+
// defaultSkillRunner echoes resolution metadata — never a Pi failure marker.
|
|
276
|
+
expect(result.errorContext ?? '').not.toMatch(PI_DISPATCH_MARKERS);
|
|
277
|
+
expect(result.finalContext).toMatchObject({ skillId: 'ct-task-executor', resolved: true });
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
it('maybeCreatePiRunner is default-OFF: undefined unless CLEO_PI_RUNNER_ENABLED=1', async () => {
|
|
281
|
+
delete process.env['CLEO_PI_RUNNER_ENABLED'];
|
|
282
|
+
expect(await maybeCreatePiRunner({ system: 'task-executor', projectRoot })).toBeUndefined();
|
|
283
|
+
|
|
284
|
+
process.env['CLEO_PI_RUNNER_ENABLED'] = '1';
|
|
285
|
+
const runner = await maybeCreatePiRunner({ system: 'task-executor', projectRoot });
|
|
286
|
+
expect(typeof runner).toBe('function');
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
/** Restore an env var to its saved value (delete when it was unset). */
|
|
291
|
+
function restoreEnv(key: string, value: string | undefined): void {
|
|
292
|
+
if (value === undefined) delete process.env[key];
|
|
293
|
+
else process.env[key] = value;
|
|
294
|
+
}
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* T11803 (M4 cantbook done-gate) — prove the vertical slice: `rcasd.cantbook`
|
|
3
|
+
* execution terminates with EXACTLY ONE LAFS `RenderableEnvelope`, and there is
|
|
4
|
+
* NO GenKit import anywhere in the execution call graph.
|
|
5
|
+
*
|
|
6
|
+
* ## What this proves
|
|
7
|
+
*
|
|
8
|
+
* - **AC1** — the REAL starter `rcasd.cantbook` is parsed and driven end-to-end
|
|
9
|
+
* through {@link executePlaybook} (parse → execute → terminal envelope). The
|
|
10
|
+
* five agentic stages are stubbed by a DETERMINISTIC recording dispatcher (no
|
|
11
|
+
* LLM backend authenticates on CI, and the point of this gate is the ENVELOPE
|
|
12
|
+
* SHAPE + the import graph — NOT a live model round-trip).
|
|
13
|
+
* - **AC2** — the terminal result is folded into exactly ONE LAFS envelope (the
|
|
14
|
+
* ADR-077 `RenderableEnvelope<T>` `{ kind: 'single', data }` wrapped by the
|
|
15
|
+
* canonical `createEnvelope` SSoT) and that envelope passes the
|
|
16
|
+
* `@cleocode/lafs` `validateEnvelope` schema validator. Exactly one — the run
|
|
17
|
+
* produces one `ExecutePlaybookResult`, which maps to one envelope.
|
|
18
|
+
* - **AC3** — NO `@genkit-ai` / GenKit import in the execution call graph: a
|
|
19
|
+
* static module-graph walk over every playbooks `src/` file reachable from
|
|
20
|
+
* `runtime.ts` (the executePlaybook entry) via intra-package imports asserts
|
|
21
|
+
* zero GenKit specifiers, and the runtime entry imports none directly.
|
|
22
|
+
* - **AC5** — the test is NOT skipped (no `.skip`, no `it.skipIf`).
|
|
23
|
+
*
|
|
24
|
+
* ## Determinism + isolation
|
|
25
|
+
*
|
|
26
|
+
* - IN-PROCESS only — vitest source, NO subprocess (tsx unresolvable in CI).
|
|
27
|
+
* - daemon-OFF — an in-memory `node:sqlite` DB, no daemon, no live LLM.
|
|
28
|
+
* - The dispatcher emits each stage's `ensures.schema`-satisfying output so the
|
|
29
|
+
* rcasd contract gates pass and the run reaches `completed` — a faithful
|
|
30
|
+
* full-pipeline traversal, deterministically.
|
|
31
|
+
*
|
|
32
|
+
* @epic T11391
|
|
33
|
+
* @task T11803
|
|
34
|
+
* @saga T11387
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
38
|
+
import { createRequire } from 'node:module';
|
|
39
|
+
import { dirname, join, resolve } from 'node:path';
|
|
40
|
+
import type { DatabaseSync as _DatabaseSyncType } from 'node:sqlite';
|
|
41
|
+
import { fileURLToPath } from 'node:url';
|
|
42
|
+
import type { RenderableEnvelope } from '@cleocode/contracts';
|
|
43
|
+
import { createEnvelope, validateEnvelope } from '@cleocode/lafs';
|
|
44
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
45
|
+
import { parsePlaybook } from '../parser.js';
|
|
46
|
+
import {
|
|
47
|
+
type AgentDispatcher,
|
|
48
|
+
type AgentDispatchInput,
|
|
49
|
+
type AgentDispatchResult,
|
|
50
|
+
type ExecutePlaybookResult,
|
|
51
|
+
executePlaybook,
|
|
52
|
+
} from '../runtime.js';
|
|
53
|
+
|
|
54
|
+
const _require = createRequire(import.meta.url);
|
|
55
|
+
type DatabaseSync = _DatabaseSyncType;
|
|
56
|
+
const { DatabaseSync } = _require('node:sqlite') as {
|
|
57
|
+
DatabaseSync: new (...args: ConstructorParameters<typeof _DatabaseSyncType>) => DatabaseSync;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
61
|
+
const PLAYBOOKS_SRC = resolve(__dirname, '..');
|
|
62
|
+
|
|
63
|
+
const MIGRATION_SQL = resolve(
|
|
64
|
+
__dirname,
|
|
65
|
+
'../../../core/migrations/drizzle-tasks/20260417220000_t889-playbook-tables/migration.sql',
|
|
66
|
+
);
|
|
67
|
+
const RCASD_CANTBOOK = resolve(__dirname, '../../starter/rcasd.cantbook');
|
|
68
|
+
|
|
69
|
+
function applyMigration(db: DatabaseSync, sql: string): void {
|
|
70
|
+
const statements = sql
|
|
71
|
+
.split(/--> statement-breakpoint/)
|
|
72
|
+
.map((s) => s.trim())
|
|
73
|
+
.filter((s) => s.length > 0);
|
|
74
|
+
for (const stmt of statements) {
|
|
75
|
+
const lines = stmt.split('\n');
|
|
76
|
+
const hasSql = lines.some((l) => l.trim().length > 0 && !l.trim().startsWith('--'));
|
|
77
|
+
if (hasSql) db.exec(stmt);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Deterministic recording dispatcher that emits each rcasd stage's
|
|
83
|
+
* `ensures.schema`-satisfying output + the `requires`/edge fields the next stage
|
|
84
|
+
* needs, so the full five-stage pipeline reaches `completed` with NO LLM. Keyed
|
|
85
|
+
* by `nodeId` (the rcasd stage names).
|
|
86
|
+
*/
|
|
87
|
+
function makeRcasdDispatcher(): AgentDispatcher & { nodes: string[] } {
|
|
88
|
+
const nodes: string[] = [];
|
|
89
|
+
// Per-stage outputs satisfying rcasd.cantbook ensures.schema + downstream
|
|
90
|
+
// requires/edge contracts. The first four stages' ensures.schema names
|
|
91
|
+
// (research_summary | consensus_decision | architecture_plan |
|
|
92
|
+
// specification_document) are registered passthrough; the terminal
|
|
93
|
+
// `decomposition` stage's `task_tree` is STRICT — it validates
|
|
94
|
+
// `context['task_tree']` as a non-empty array of { title, acceptance[] }
|
|
95
|
+
// entries — so we emit a conformant `task_tree` to clear the contract gate.
|
|
96
|
+
const STAGE_OUTPUT: Record<string, Record<string, unknown>> = {
|
|
97
|
+
research: {
|
|
98
|
+
summary: 'research summary for the epic',
|
|
99
|
+
risks: ['risk-a', 'risk-b'],
|
|
100
|
+
},
|
|
101
|
+
consensus: {
|
|
102
|
+
decision: 'proceed with approach X',
|
|
103
|
+
},
|
|
104
|
+
architecture: {
|
|
105
|
+
patterns: ['pattern-1', 'pattern-2'],
|
|
106
|
+
adrs: ['ADR-001'],
|
|
107
|
+
},
|
|
108
|
+
specification: {
|
|
109
|
+
acceptance: ['AC1', 'AC2'],
|
|
110
|
+
requirements: ['REQ-1'],
|
|
111
|
+
},
|
|
112
|
+
decomposition: {
|
|
113
|
+
// `children` satisfies the specification→decomposition edge.ensures.
|
|
114
|
+
children: ['T1001', 'T1002'],
|
|
115
|
+
acceptance: ['AC1'],
|
|
116
|
+
requirements: ['REQ-1'],
|
|
117
|
+
// `task_tree` satisfies the strict ensures.schema on the terminal stage.
|
|
118
|
+
task_tree: [
|
|
119
|
+
{ id: 'T1001', title: 'Implement feature A', acceptance: ['AC1'] },
|
|
120
|
+
{ id: 'T1002', title: 'Implement feature B', acceptance: ['AC2'] },
|
|
121
|
+
],
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
return {
|
|
125
|
+
nodes,
|
|
126
|
+
async dispatch(input: AgentDispatchInput): Promise<AgentDispatchResult> {
|
|
127
|
+
nodes.push(input.nodeId);
|
|
128
|
+
return { status: 'success', output: STAGE_OUTPUT[input.nodeId] ?? {} };
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Recursively collect every `*.ts` source file under `dir` (excluding tests),
|
|
135
|
+
* so the no-GenKit assertion covers the WHOLE playbooks runtime package surface
|
|
136
|
+
* reachable by the execution path — not just the entry file.
|
|
137
|
+
*/
|
|
138
|
+
function collectSourceFiles(dir: string): string[] {
|
|
139
|
+
const out: string[] = [];
|
|
140
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
141
|
+
const full = join(dir, entry.name);
|
|
142
|
+
if (entry.isDirectory()) {
|
|
143
|
+
if (entry.name === '__tests__') continue;
|
|
144
|
+
out.push(...collectSourceFiles(full));
|
|
145
|
+
} else if (entry.name.endsWith('.ts')) {
|
|
146
|
+
out.push(full);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return out;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
describe('T11803 (M4): rcasd.cantbook → exactly ONE LAFS envelope, no GenKit', () => {
|
|
153
|
+
let db: DatabaseSync;
|
|
154
|
+
|
|
155
|
+
beforeEach(() => {
|
|
156
|
+
db = new DatabaseSync(':memory:');
|
|
157
|
+
db.exec('PRAGMA foreign_keys=ON');
|
|
158
|
+
applyMigration(db, readFileSync(MIGRATION_SQL, 'utf8'));
|
|
159
|
+
});
|
|
160
|
+
afterEach(() => db.close());
|
|
161
|
+
|
|
162
|
+
it('AC1+AC2: parse→execute→envelope yields exactly ONE valid LAFS RenderableEnvelope', async () => {
|
|
163
|
+
const { definition, sourceHash } = parsePlaybook(readFileSync(RCASD_CANTBOOK, 'utf8'));
|
|
164
|
+
expect(definition.name).toBe('rcasd');
|
|
165
|
+
expect(definition.nodes.map((n) => n.id)).toEqual([
|
|
166
|
+
'research',
|
|
167
|
+
'consensus',
|
|
168
|
+
'architecture',
|
|
169
|
+
'specification',
|
|
170
|
+
'decomposition',
|
|
171
|
+
]);
|
|
172
|
+
|
|
173
|
+
const dispatcher = makeRcasdDispatcher();
|
|
174
|
+
const result: ExecutePlaybookResult = await executePlaybook({
|
|
175
|
+
db,
|
|
176
|
+
playbook: definition,
|
|
177
|
+
playbookHash: sourceHash,
|
|
178
|
+
initialContext: { taskId: 'T999', epicId: 'T999', scope: 'global' },
|
|
179
|
+
dispatcher,
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
// The full five-stage pipeline ran deterministically to completion.
|
|
183
|
+
expect(dispatcher.nodes).toEqual([
|
|
184
|
+
'research',
|
|
185
|
+
'consensus',
|
|
186
|
+
'architecture',
|
|
187
|
+
'specification',
|
|
188
|
+
'decomposition',
|
|
189
|
+
]);
|
|
190
|
+
expect(result.terminalStatus).toBe('completed');
|
|
191
|
+
// No silent contract violation hid behind the completion.
|
|
192
|
+
expect(result.finalContext['__ensuresSchemaViolation']).toBeUndefined();
|
|
193
|
+
expect(result.finalContext['__contractViolation']).toBeUndefined();
|
|
194
|
+
|
|
195
|
+
// Fold the single terminal result into ONE ADR-077 RenderableEnvelope, then
|
|
196
|
+
// into ONE canonical LAFS envelope (the SSoT `createEnvelope`).
|
|
197
|
+
const renderable: RenderableEnvelope<ExecutePlaybookResult> = {
|
|
198
|
+
kind: 'single',
|
|
199
|
+
data: result,
|
|
200
|
+
};
|
|
201
|
+
const lafsEnvelope = createEnvelope({
|
|
202
|
+
success: true,
|
|
203
|
+
result: renderable as unknown as Record<string, unknown>,
|
|
204
|
+
meta: { operation: 'playbook.run', requestId: `rcasd-${result.runId}` },
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
// EXACTLY ONE — a single object, not an array of envelopes.
|
|
208
|
+
expect(Array.isArray(lafsEnvelope)).toBe(false);
|
|
209
|
+
expect(lafsEnvelope.success).toBe(true);
|
|
210
|
+
expect(lafsEnvelope.result).toMatchObject({ kind: 'single' });
|
|
211
|
+
|
|
212
|
+
// Validated by the @cleocode/lafs schema validator.
|
|
213
|
+
const validation = validateEnvelope(lafsEnvelope);
|
|
214
|
+
expect(validation.errors).toEqual([]);
|
|
215
|
+
expect(validation.valid).toBe(true);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it('AC3: NO @genkit-ai / GenKit import in the execution call graph', () => {
|
|
219
|
+
// The execution call graph for executePlaybook (runtime → state, approval,
|
|
220
|
+
// parser, schema) lives entirely under packages/playbooks/src. Walk every
|
|
221
|
+
// non-test source file and assert zero GenKit specifiers.
|
|
222
|
+
const files = collectSourceFiles(PLAYBOOKS_SRC);
|
|
223
|
+
expect(files.length).toBeGreaterThan(0);
|
|
224
|
+
|
|
225
|
+
const offenders: string[] = [];
|
|
226
|
+
const GENKIT_IMPORT =
|
|
227
|
+
/(?:import|from|require)\s*(?:\(\s*)?['"]@?genkit(?:-ai)?(?:\/[^'"]*)?['"]/;
|
|
228
|
+
for (const file of files) {
|
|
229
|
+
const src = readFileSync(file, 'utf8');
|
|
230
|
+
if (GENKIT_IMPORT.test(src)) offenders.push(file);
|
|
231
|
+
}
|
|
232
|
+
expect(offenders).toEqual([]);
|
|
233
|
+
|
|
234
|
+
// And the runtime entry — the executePlaybook module itself — imports no
|
|
235
|
+
// GenKit specifier directly (belt-and-suspenders against a future edit).
|
|
236
|
+
const runtimeSrc = readFileSync(resolve(PLAYBOOKS_SRC, 'runtime.ts'), 'utf8');
|
|
237
|
+
expect(runtimeSrc).not.toMatch(/@?genkit/i);
|
|
238
|
+
});
|
|
239
|
+
});
|
package/src/index.ts
CHANGED
|
@@ -79,6 +79,8 @@ export {
|
|
|
79
79
|
executePlaybook,
|
|
80
80
|
type PlaybookTerminalStatus,
|
|
81
81
|
type ResumePlaybookOptions,
|
|
82
|
+
// T1944 (M4): per-step `node.inputs` → dispatch-boundary binding resolver.
|
|
83
|
+
resolveStepBindings,
|
|
82
84
|
resumePlaybook,
|
|
83
85
|
} from './runtime.js';
|
|
84
86
|
// W4-8: state layer CRUD for playbook_runs + playbook_approvals
|
package/src/parser.ts
CHANGED
|
@@ -337,6 +337,19 @@ function parseAgenticNode(
|
|
|
337
337
|
|
|
338
338
|
const context_files = parseStringArray(raw.context_files, `nodes[${index}].context_files`);
|
|
339
339
|
|
|
340
|
+
// T11759 (M4): per-stage LLM pin. Each is an optional non-empty string — the
|
|
341
|
+
// dispatcher resolves them through the E9 chokepoint at dispatch time. Absent
|
|
342
|
+
// fields leave resolution unchanged (additive).
|
|
343
|
+
const profile = parseOptionalNonEmptyString(raw.profile, `nodes[${index}].profile`);
|
|
344
|
+
const model = parseOptionalNonEmptyString(raw.model, `nodes[${index}].model`);
|
|
345
|
+
const provider = parseOptionalNonEmptyString(raw.provider, `nodes[${index}].provider`);
|
|
346
|
+
|
|
347
|
+
// T1947 (M4): per-step skill/tool allowlists. Each is an optional string[] —
|
|
348
|
+
// enforced at spawn time by composeSpawnPayload as an INTERSECTION against the
|
|
349
|
+
// Tier-0/1/2 baseline (restrict-only, never expand). Absent → no restriction.
|
|
350
|
+
const allowed_skills = parseStringArray(raw.allowed_skills, `nodes[${index}].allowed_skills`);
|
|
351
|
+
const allowed_tools = parseStringArray(raw.allowed_tools, `nodes[${index}].allowed_tools`);
|
|
352
|
+
|
|
340
353
|
return {
|
|
341
354
|
...base,
|
|
342
355
|
type: 'agentic',
|
|
@@ -345,6 +358,11 @@ function parseAgenticNode(
|
|
|
345
358
|
role,
|
|
346
359
|
inputs,
|
|
347
360
|
...(context_files !== undefined ? { context_files } : {}),
|
|
361
|
+
...(profile !== undefined ? { profile } : {}),
|
|
362
|
+
...(model !== undefined ? { model } : {}),
|
|
363
|
+
...(provider !== undefined ? { provider } : {}),
|
|
364
|
+
...(allowed_skills !== undefined ? { allowed_skills } : {}),
|
|
365
|
+
...(allowed_tools !== undefined ? { allowed_tools } : {}),
|
|
348
366
|
};
|
|
349
367
|
}
|
|
350
368
|
|
|
@@ -768,6 +786,23 @@ function parseStringArray(raw: unknown, field: string): string[] | undefined {
|
|
|
768
786
|
});
|
|
769
787
|
}
|
|
770
788
|
|
|
789
|
+
/**
|
|
790
|
+
* Parse an optional scalar string field that, when present, MUST be a non-empty
|
|
791
|
+
* string (T11759 — the per-stage LLM pins `profile` / `model` / `provider`).
|
|
792
|
+
*
|
|
793
|
+
* @param raw - Raw YAML value.
|
|
794
|
+
* @param field - Field path for diagnostics.
|
|
795
|
+
* @returns The trimmed-as-authored string, or `undefined` when absent.
|
|
796
|
+
* @throws {PlaybookParseError} When present but not a non-empty string.
|
|
797
|
+
*/
|
|
798
|
+
function parseOptionalNonEmptyString(raw: unknown, field: string): string | undefined {
|
|
799
|
+
if (raw === undefined) return undefined;
|
|
800
|
+
if (typeof raw !== 'string' || raw.length === 0) {
|
|
801
|
+
throw new PlaybookParseError(`${field} must be a non-empty string`, field, raw);
|
|
802
|
+
}
|
|
803
|
+
return raw;
|
|
804
|
+
}
|
|
805
|
+
|
|
771
806
|
/**
|
|
772
807
|
* Detect cycles across the combined edge-set (explicit `edges[]` plus
|
|
773
808
|
* `depends[]` back-references). Uses 3-color DFS.
|