@cleocode/playbooks 2026.6.12 → 2026.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +9 -2
- package/dist/parser.js +114 -1
- package/dist/runtime.d.ts +92 -34
- package/dist/runtime.js +257 -149
- package/package.json +3 -3
- package/src/__tests__/fixtures/conditional-branch.cantbook +50 -0
- package/src/__tests__/fixtures/pi-skill-only.cantbook +26 -0
- package/src/__tests__/guarded-deterministic-dispatch.test.ts +225 -0
- package/src/__tests__/inputs-bindings.test.ts +250 -0
- package/src/__tests__/node-allowlist.test.ts +186 -0
- package/src/__tests__/node-profile.test.ts +142 -0
- package/src/__tests__/parser.test.ts +191 -0
- package/src/__tests__/pi-cantbook-e2e.test.ts +294 -0
- package/src/__tests__/rcasd-one-envelope.test.ts +239 -0
- package/src/__tests__/runtime.test.ts +502 -4
- package/src/index.ts +8 -3
- package/src/parser.ts +144 -1
- package/src/runtime.ts +363 -198
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* T11759 (M4 cantbook done-gate) — `PlaybookAgenticNode.profile` / `.model` /
|
|
3
|
+
* `.provider` reach the dispatch boundary so a cantbook stage can pin its LLM.
|
|
4
|
+
*
|
|
5
|
+
* Two layers:
|
|
6
|
+
* 1. Parser → runtime forwarding: a `.cantbook` node with `profile:` is parsed
|
|
7
|
+
* and the pin reaches {@link AgentDispatchInput} (with `playbookName`) at
|
|
8
|
+
* dispatch time — asserted with a recorder dispatcher (no live backend).
|
|
9
|
+
* 2. Default (un-pinned) nodes carry NO pin — the dispatch path is unchanged.
|
|
10
|
+
*
|
|
11
|
+
* The dispatcher's E9 resolution itself (cantbook system key + profile pin) is
|
|
12
|
+
* covered in-process by `@cleocode/core`'s `cantbook-profile.test.ts`; this
|
|
13
|
+
* suite proves the runtime hands the pin to the dispatcher so that resolution
|
|
14
|
+
* can occur. No `@cleocode/*` module is mocked.
|
|
15
|
+
*
|
|
16
|
+
* @task T11759
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { readFileSync } from 'node:fs';
|
|
20
|
+
import { createRequire } from 'node:module';
|
|
21
|
+
import { dirname, resolve } from 'node:path';
|
|
22
|
+
import type { DatabaseSync as _DatabaseSyncType } from 'node:sqlite';
|
|
23
|
+
import { fileURLToPath } from 'node:url';
|
|
24
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
25
|
+
import { parsePlaybook } from '../parser.js';
|
|
26
|
+
import {
|
|
27
|
+
type AgentDispatcher,
|
|
28
|
+
type AgentDispatchInput,
|
|
29
|
+
type AgentDispatchResult,
|
|
30
|
+
executePlaybook,
|
|
31
|
+
} from '../runtime.js';
|
|
32
|
+
|
|
33
|
+
const _require = createRequire(import.meta.url);
|
|
34
|
+
type DatabaseSync = _DatabaseSyncType;
|
|
35
|
+
const { DatabaseSync } = _require('node:sqlite') as {
|
|
36
|
+
DatabaseSync: new (...args: ConstructorParameters<typeof _DatabaseSyncType>) => DatabaseSync;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
40
|
+
const MIGRATION_SQL = resolve(
|
|
41
|
+
__dirname,
|
|
42
|
+
'../../../core/migrations/drizzle-tasks/20260417220000_t889-playbook-tables/migration.sql',
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
function applyMigration(db: DatabaseSync, sql: string): void {
|
|
46
|
+
const statements = sql
|
|
47
|
+
.split(/--> statement-breakpoint/)
|
|
48
|
+
.map((s) => s.trim())
|
|
49
|
+
.filter((s) => s.length > 0);
|
|
50
|
+
for (const stmt of statements) {
|
|
51
|
+
const lines = stmt.split('\n');
|
|
52
|
+
const hasSql = lines.some((l) => l.trim().length > 0 && !l.trim().startsWith('--'));
|
|
53
|
+
if (hasSql) db.exec(stmt);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Records the full {@link AgentDispatchInput} per call. */
|
|
58
|
+
function makeRecorder(): AgentDispatcher & { calls: AgentDispatchInput[] } {
|
|
59
|
+
const calls: AgentDispatchInput[] = [];
|
|
60
|
+
return {
|
|
61
|
+
calls,
|
|
62
|
+
async dispatch(input: AgentDispatchInput): Promise<AgentDispatchResult> {
|
|
63
|
+
calls.push({ ...input, context: { ...input.context } });
|
|
64
|
+
return { status: 'success', output: {} };
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
describe('T11759: node profile pin reaches the dispatch boundary', () => {
|
|
70
|
+
let db: DatabaseSync;
|
|
71
|
+
|
|
72
|
+
beforeEach(() => {
|
|
73
|
+
db = new DatabaseSync(':memory:');
|
|
74
|
+
db.exec('PRAGMA foreign_keys=ON');
|
|
75
|
+
applyMigration(db, readFileSync(MIGRATION_SQL, 'utf8'));
|
|
76
|
+
});
|
|
77
|
+
afterEach(() => db.close());
|
|
78
|
+
|
|
79
|
+
it('AC3: a `.cantbook` stage with `profile:` forwards the pin + playbookName', async () => {
|
|
80
|
+
const yaml = `
|
|
81
|
+
version: "1.0"
|
|
82
|
+
name: pinned-flow
|
|
83
|
+
nodes:
|
|
84
|
+
- id: review
|
|
85
|
+
type: agentic
|
|
86
|
+
skill: ct-validator
|
|
87
|
+
profile: frontier-review
|
|
88
|
+
model: claude-opus-4-8
|
|
89
|
+
provider: anthropic
|
|
90
|
+
`;
|
|
91
|
+
const { definition } = parsePlaybook(yaml);
|
|
92
|
+
const dispatcher = makeRecorder();
|
|
93
|
+
|
|
94
|
+
const result = await executePlaybook({
|
|
95
|
+
db,
|
|
96
|
+
playbook: definition,
|
|
97
|
+
playbookHash: 'hash-t11759',
|
|
98
|
+
initialContext: { taskId: 'T11759' },
|
|
99
|
+
dispatcher,
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
expect(result.terminalStatus).toBe('completed');
|
|
103
|
+
expect(dispatcher.calls).toHaveLength(1);
|
|
104
|
+
const call = dispatcher.calls[0];
|
|
105
|
+
// The cantbook node identity reaches the dispatcher so it can build the
|
|
106
|
+
// `cantbook:<playbook>#<nodeId>` system key (AC2 resolution input).
|
|
107
|
+
expect(call?.playbookName).toBe('pinned-flow');
|
|
108
|
+
expect(call?.nodeId).toBe('review');
|
|
109
|
+
// The per-stage LLM pin is forwarded verbatim (the dispatcher resolves it
|
|
110
|
+
// through `resolveCantbookNodeProfile` → E9).
|
|
111
|
+
expect(call?.profile).toBe('frontier-review');
|
|
112
|
+
expect(call?.model).toBe('claude-opus-4-8');
|
|
113
|
+
expect(call?.provider).toBe('anthropic');
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('AC3: an un-pinned stage carries NO profile/model/provider (additive)', async () => {
|
|
117
|
+
const yaml = `
|
|
118
|
+
version: "1.0"
|
|
119
|
+
name: plain-flow
|
|
120
|
+
nodes:
|
|
121
|
+
- id: research
|
|
122
|
+
type: agentic
|
|
123
|
+
skill: ct-research-agent
|
|
124
|
+
`;
|
|
125
|
+
const { definition } = parsePlaybook(yaml);
|
|
126
|
+
const dispatcher = makeRecorder();
|
|
127
|
+
|
|
128
|
+
await executePlaybook({
|
|
129
|
+
db,
|
|
130
|
+
playbook: definition,
|
|
131
|
+
playbookHash: 'hash-t11759-b',
|
|
132
|
+
initialContext: { taskId: 'T123' },
|
|
133
|
+
dispatcher,
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const call = dispatcher.calls[0];
|
|
137
|
+
expect(call?.playbookName).toBe('plain-flow');
|
|
138
|
+
expect(call?.profile).toBeUndefined();
|
|
139
|
+
expect(call?.model).toBeUndefined();
|
|
140
|
+
expect(call?.provider).toBeUndefined();
|
|
141
|
+
});
|
|
142
|
+
});
|
|
@@ -340,6 +340,86 @@ nodes:
|
|
|
340
340
|
});
|
|
341
341
|
});
|
|
342
342
|
|
|
343
|
+
describe('T11759: agentic node LLM profile pin', () => {
|
|
344
|
+
it('AC1: parses profile/model/provider on an agentic node', () => {
|
|
345
|
+
const yaml = `
|
|
346
|
+
version: "1.0"
|
|
347
|
+
name: pinned
|
|
348
|
+
nodes:
|
|
349
|
+
- id: review
|
|
350
|
+
type: agentic
|
|
351
|
+
skill: ct-validator
|
|
352
|
+
profile: frontier-review
|
|
353
|
+
model: claude-opus-4-8
|
|
354
|
+
provider: anthropic
|
|
355
|
+
`;
|
|
356
|
+
const { definition } = parsePlaybook(yaml);
|
|
357
|
+
expect(definition.nodes[0]).toMatchObject({
|
|
358
|
+
id: 'review',
|
|
359
|
+
type: 'agentic',
|
|
360
|
+
profile: 'frontier-review',
|
|
361
|
+
model: 'claude-opus-4-8',
|
|
362
|
+
provider: 'anthropic',
|
|
363
|
+
});
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
it('AC1: omits profile/model/provider when not declared (additive)', () => {
|
|
367
|
+
const yaml = `
|
|
368
|
+
version: "1.0"
|
|
369
|
+
name: unpinned
|
|
370
|
+
nodes:
|
|
371
|
+
- id: plain
|
|
372
|
+
type: agentic
|
|
373
|
+
skill: ct-research-agent
|
|
374
|
+
`;
|
|
375
|
+
const node = parsePlaybook(yaml).definition.nodes[0];
|
|
376
|
+
expect(node).not.toHaveProperty('profile');
|
|
377
|
+
expect(node).not.toHaveProperty('model');
|
|
378
|
+
expect(node).not.toHaveProperty('provider');
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
it('AC1: a profile-only pin is valid', () => {
|
|
382
|
+
const yaml = `
|
|
383
|
+
version: "1.0"
|
|
384
|
+
name: profile-only
|
|
385
|
+
nodes:
|
|
386
|
+
- id: stage
|
|
387
|
+
type: agentic
|
|
388
|
+
skill: ct-task-executor
|
|
389
|
+
profile: fast-lane
|
|
390
|
+
`;
|
|
391
|
+
const node = parsePlaybook(yaml).definition.nodes[0];
|
|
392
|
+
expect(node).toMatchObject({ profile: 'fast-lane' });
|
|
393
|
+
expect(node).not.toHaveProperty('model');
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
it('AC1: rejects an empty-string profile', () => {
|
|
397
|
+
const yaml = `
|
|
398
|
+
version: "1.0"
|
|
399
|
+
name: bad-profile
|
|
400
|
+
nodes:
|
|
401
|
+
- id: stage
|
|
402
|
+
type: agentic
|
|
403
|
+
skill: ct-task-executor
|
|
404
|
+
profile: ""
|
|
405
|
+
`;
|
|
406
|
+
expect(() => parsePlaybook(yaml)).toThrow(/profile must be a non-empty string/);
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
it('AC1: rejects a non-string model', () => {
|
|
410
|
+
const yaml = `
|
|
411
|
+
version: "1.0"
|
|
412
|
+
name: bad-model
|
|
413
|
+
nodes:
|
|
414
|
+
- id: stage
|
|
415
|
+
type: agentic
|
|
416
|
+
skill: ct-task-executor
|
|
417
|
+
model: 42
|
|
418
|
+
`;
|
|
419
|
+
expect(() => parsePlaybook(yaml)).toThrow(/model must be a non-empty string/);
|
|
420
|
+
});
|
|
421
|
+
});
|
|
422
|
+
|
|
343
423
|
describe('bounds checks', () => {
|
|
344
424
|
it('rejects max_iterations > 10', () => {
|
|
345
425
|
const yaml = `
|
|
@@ -453,4 +533,115 @@ nodes:
|
|
|
453
533
|
}
|
|
454
534
|
});
|
|
455
535
|
});
|
|
536
|
+
|
|
537
|
+
describe('T11806: edge.when branch guard', () => {
|
|
538
|
+
const base = `
|
|
539
|
+
version: "1.0"
|
|
540
|
+
name: branch
|
|
541
|
+
nodes:
|
|
542
|
+
- id: classify
|
|
543
|
+
type: agentic
|
|
544
|
+
skill: x
|
|
545
|
+
- id: a
|
|
546
|
+
type: agentic
|
|
547
|
+
skill: y
|
|
548
|
+
- id: b
|
|
549
|
+
type: agentic
|
|
550
|
+
skill: z
|
|
551
|
+
`;
|
|
552
|
+
|
|
553
|
+
it('parses an equals/notEquals guarded branch', () => {
|
|
554
|
+
const { definition } = parsePlaybook(`${base}edges:
|
|
555
|
+
- from: classify
|
|
556
|
+
to: a
|
|
557
|
+
when:
|
|
558
|
+
field: needsResearch
|
|
559
|
+
equals: true
|
|
560
|
+
- from: classify
|
|
561
|
+
to: b
|
|
562
|
+
when:
|
|
563
|
+
field: needsResearch
|
|
564
|
+
notEquals: true
|
|
565
|
+
`);
|
|
566
|
+
expect(definition.edges).toHaveLength(2);
|
|
567
|
+
expect(definition.edges[0]?.when).toEqual({ field: 'needsResearch', equals: true });
|
|
568
|
+
expect(definition.edges[1]?.when).toEqual({ field: 'needsResearch', notEquals: true });
|
|
569
|
+
});
|
|
570
|
+
|
|
571
|
+
it('parses exists / truthy / in operators', () => {
|
|
572
|
+
const { definition } = parsePlaybook(`${base}edges:
|
|
573
|
+
- from: classify
|
|
574
|
+
to: a
|
|
575
|
+
when:
|
|
576
|
+
field: route
|
|
577
|
+
in: [x, y, z]
|
|
578
|
+
- from: classify
|
|
579
|
+
to: b
|
|
580
|
+
when:
|
|
581
|
+
field: route
|
|
582
|
+
exists: false
|
|
583
|
+
`);
|
|
584
|
+
expect(definition.edges[0]?.when).toEqual({ field: 'route', in: ['x', 'y', 'z'] });
|
|
585
|
+
expect(definition.edges[1]?.when).toEqual({ field: 'route', exists: false });
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
it('rejects a when guard with no operator', () => {
|
|
589
|
+
expect(() =>
|
|
590
|
+
parsePlaybook(`${base}edges:
|
|
591
|
+
- from: classify
|
|
592
|
+
to: a
|
|
593
|
+
when:
|
|
594
|
+
field: route
|
|
595
|
+
`),
|
|
596
|
+
).toThrow(/must declare exactly one of/);
|
|
597
|
+
});
|
|
598
|
+
|
|
599
|
+
it('rejects a when guard with multiple operators', () => {
|
|
600
|
+
expect(() =>
|
|
601
|
+
parsePlaybook(`${base}edges:
|
|
602
|
+
- from: classify
|
|
603
|
+
to: a
|
|
604
|
+
when:
|
|
605
|
+
field: route
|
|
606
|
+
equals: x
|
|
607
|
+
exists: true
|
|
608
|
+
`),
|
|
609
|
+
).toThrow(/declares multiple operators/);
|
|
610
|
+
});
|
|
611
|
+
|
|
612
|
+
it('rejects a when guard with a missing field', () => {
|
|
613
|
+
expect(() =>
|
|
614
|
+
parsePlaybook(`${base}edges:
|
|
615
|
+
- from: classify
|
|
616
|
+
to: a
|
|
617
|
+
when:
|
|
618
|
+
equals: x
|
|
619
|
+
`),
|
|
620
|
+
).toThrow(/field must be a non-empty string/);
|
|
621
|
+
});
|
|
622
|
+
|
|
623
|
+
it('rejects a non-boolean exists operator', () => {
|
|
624
|
+
expect(() =>
|
|
625
|
+
parsePlaybook(`${base}edges:
|
|
626
|
+
- from: classify
|
|
627
|
+
to: a
|
|
628
|
+
when:
|
|
629
|
+
field: route
|
|
630
|
+
exists: "yes"
|
|
631
|
+
`),
|
|
632
|
+
).toThrow(/exists must be a boolean/);
|
|
633
|
+
});
|
|
634
|
+
|
|
635
|
+
it('rejects an empty in array', () => {
|
|
636
|
+
expect(() =>
|
|
637
|
+
parsePlaybook(`${base}edges:
|
|
638
|
+
- from: classify
|
|
639
|
+
to: a
|
|
640
|
+
when:
|
|
641
|
+
field: route
|
|
642
|
+
in: []
|
|
643
|
+
`),
|
|
644
|
+
).toThrow(/in must be a non-empty array/);
|
|
645
|
+
});
|
|
646
|
+
});
|
|
456
647
|
});
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* T11945 (M4) — deterministic in-process e2e: a skill-only `.cantbook` executes
|
|
3
|
+
* THROUGH the PiAgentAdapter when the Pi runner is wired into the production
|
|
4
|
+
* dispatcher.
|
|
5
|
+
*
|
|
6
|
+
* ## What this proves (the M4 keystone)
|
|
7
|
+
*
|
|
8
|
+
* The two production cantbook dispatchers (`buildDefaultDispatcher` in
|
|
9
|
+
* `packages/cleo/src/dispatch/domains/playbook.ts` and `buildGoDispatcher` in
|
|
10
|
+
* `packages/cleo/src/cli/commands/go-ivtr-runner.ts`) now pass
|
|
11
|
+
* `runner: await maybeCreatePiRunner({ system, projectRoot })` into
|
|
12
|
+
* {@link runSkillNodeOrSpawn}. This test reconstructs that EXACT production
|
|
13
|
+
* dispatcher shape (`createToolGuard` + `runSkillNodeOrSpawn` + the
|
|
14
|
+
* `maybeCreatePiRunner` wiring helper) and drives a single skill-only node
|
|
15
|
+
* through the real {@link executePlaybook} runtime state machine.
|
|
16
|
+
*
|
|
17
|
+
* With `CLEO_PI_RUNNER_ENABLED=1` the in-process skill node routes THROUGH the
|
|
18
|
+
* PiAgentAdapter. There is no working LLM backend on this box, so we do NOT
|
|
19
|
+
* assert a live round-trip — we assert the DISPATCH PATH reached the Pi adapter,
|
|
20
|
+
* proven by the **cleo-owned no-credential failure string**
|
|
21
|
+
* (`no credential resolved for system "task-executor"`) emitted by the Cleo
|
|
22
|
+
* streamFn (`createPiStreamFn`, the E9 chokepoint). A pi-ai-internal failure
|
|
23
|
+
* shape would not carry that exact cleo string. The Pi adapter NEVER throws —
|
|
24
|
+
* it returns a contained `{ status: 'failure' }` (pi-agent-adapter.test.ts:197
|
|
25
|
+
* precedent), so the run terminates cleanly.
|
|
26
|
+
*
|
|
27
|
+
* ## Determinism + isolation contract (AC3 · AC4)
|
|
28
|
+
*
|
|
29
|
+
* - IN-PROCESS only — vitest source, NO subprocess (tsx unresolvable in CI).
|
|
30
|
+
* - daemon-OFF — the runtime is the in-memory `node:sqlite` DB, no daemon.
|
|
31
|
+
* - `CLEO_SESSION_ID` is stamped so the Pi runner's ZERO-authority guard passes
|
|
32
|
+
* (production relies on the daemon stamping it; here the test stamps it).
|
|
33
|
+
* - A real `ct-*` skill is materialised under a temp project root's
|
|
34
|
+
* `.agents/skills/` dir so `findSkill` resolves it and the in-process skill
|
|
35
|
+
* path (not subprocess spawn) is taken. `CLEO_PROJECT_ROOT` pins resolution
|
|
36
|
+
* to that temp dir.
|
|
37
|
+
* - `maxIterationsDefault: 1` so the failing node runs exactly once → the run
|
|
38
|
+
* terminates as `exceeded_iteration_cap` with the Pi error in `errorContext`.
|
|
39
|
+
*
|
|
40
|
+
* @epic T10403
|
|
41
|
+
* @task T11761
|
|
42
|
+
* @task T11945
|
|
43
|
+
*/
|
|
44
|
+
|
|
45
|
+
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
46
|
+
import { createRequire } from 'node:module';
|
|
47
|
+
import { tmpdir } from 'node:os';
|
|
48
|
+
import { dirname, join, resolve } from 'node:path';
|
|
49
|
+
import type { DatabaseSync as _DatabaseSyncType } from 'node:sqlite';
|
|
50
|
+
import { fileURLToPath } from 'node:url';
|
|
51
|
+
import {
|
|
52
|
+
type AgentDispatcher,
|
|
53
|
+
type AgentDispatchInput,
|
|
54
|
+
type AgentDispatchResult,
|
|
55
|
+
createToolGuard,
|
|
56
|
+
maybeCreatePiRunner,
|
|
57
|
+
runSkillNodeOrSpawn,
|
|
58
|
+
} from '@cleocode/core/internal';
|
|
59
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
60
|
+
import { parsePlaybook } from '../parser.js';
|
|
61
|
+
import { executePlaybook } from '../runtime.js';
|
|
62
|
+
import { getPlaybookRun } from '../state.js';
|
|
63
|
+
|
|
64
|
+
const _require = createRequire(import.meta.url);
|
|
65
|
+
type DatabaseSync = _DatabaseSyncType;
|
|
66
|
+
const { DatabaseSync } = _require('node:sqlite') as {
|
|
67
|
+
DatabaseSync: new (...args: ConstructorParameters<typeof _DatabaseSyncType>) => DatabaseSync;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
71
|
+
|
|
72
|
+
/** Absolute path to the T889 playbook-tables migration SQL. */
|
|
73
|
+
const MIGRATION_SQL_PATH = resolve(
|
|
74
|
+
__dirname,
|
|
75
|
+
'../../../core/migrations/drizzle-tasks/20260417220000_t889-playbook-tables/migration.sql',
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
/** The skill-only fixture cantbook materialised for this suite. */
|
|
79
|
+
const FIXTURE_PATH = resolve(__dirname, 'fixtures/pi-skill-only.cantbook');
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Cleo-owned Pi failure markers that ONLY appear when a dispatch reached the
|
|
83
|
+
* PiAgentAdapter → Cleo streamFn (E9 chokepoint). A `defaultSkillRunner` would
|
|
84
|
+
* have SUCCEEDED (it does no LLM work), so seeing any of these proves the Pi
|
|
85
|
+
* route was taken:
|
|
86
|
+
* - `no credential resolved for system` — the Cleo streamFn's no-credential
|
|
87
|
+
* terminal error (`createPiStreamFn`), when NO key is reachable.
|
|
88
|
+
* - `Pi loop failed:` — the Cleo `wrapPiCall` containment prefix
|
|
89
|
+
* (`pi-errors.ts`), when a credential DID resolve and the real LLM transport
|
|
90
|
+
* failed (e.g. an expired `sk-ant-oat` token → 401). Either way the call went
|
|
91
|
+
* through the Pi adapter, never `defaultSkillRunner`.
|
|
92
|
+
*
|
|
93
|
+
* Mirrors the `pi-agent-adapter.test.ts:197` precedent
|
|
94
|
+
* (`/no credential resolved for system|pi loop error|error/i`).
|
|
95
|
+
*/
|
|
96
|
+
const PI_DISPATCH_MARKERS = /no credential resolved for system|Pi loop failed:/i;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Apply a multi-statement Drizzle migration file (split on the
|
|
100
|
+
* `--> statement-breakpoint` token). Mirrors starter.e2e.test.ts.
|
|
101
|
+
*/
|
|
102
|
+
function applyMigration(db: DatabaseSync, sql: string): void {
|
|
103
|
+
const statements = sql
|
|
104
|
+
.split(/--> statement-breakpoint/)
|
|
105
|
+
.map((s) => s.trim())
|
|
106
|
+
.filter((s) => s.length > 0);
|
|
107
|
+
for (const stmt of statements) {
|
|
108
|
+
const lines = stmt.split('\n');
|
|
109
|
+
const hasSql = lines.some((l) => l.trim().length > 0 && !l.trim().startsWith('--'));
|
|
110
|
+
if (hasSql) db.exec(stmt);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Materialise a real `ct-task-executor` skill under `<projectRoot>/.agents/skills/`
|
|
116
|
+
* so {@link runSkillNodeOrSpawn}'s `findSkill(agentId, projectRoot)` resolves it
|
|
117
|
+
* via the `project-custom` search scope, taking the in-process skill path.
|
|
118
|
+
*/
|
|
119
|
+
function writeFixtureSkill(projectRoot: string): void {
|
|
120
|
+
const skillDir = join(projectRoot, '.agents', 'skills', 'ct-task-executor');
|
|
121
|
+
mkdirSync(skillDir, { recursive: true });
|
|
122
|
+
writeFileSync(
|
|
123
|
+
join(skillDir, 'SKILL.md'),
|
|
124
|
+
[
|
|
125
|
+
'---',
|
|
126
|
+
'name: ct-task-executor',
|
|
127
|
+
'description: T11945 e2e fixture skill — drives the in-process Pi dispatch path.',
|
|
128
|
+
'---',
|
|
129
|
+
'',
|
|
130
|
+
'Execute the assigned task.',
|
|
131
|
+
'',
|
|
132
|
+
].join('\n'),
|
|
133
|
+
'utf8',
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Build the EXACT production dispatcher shape from `buildDefaultDispatcher`:
|
|
139
|
+
* in-process `ct-*` skill nodes run over a deny-first guarded tool surface via
|
|
140
|
+
* {@link runSkillNodeOrSpawn} with the Pi runner injected; isolation/agent nodes
|
|
141
|
+
* fall back to the (here-unused) subprocess spawn. The `runner` is obtained from
|
|
142
|
+
* the M4 wiring helper {@link maybeCreatePiRunner}, exactly as production does.
|
|
143
|
+
*/
|
|
144
|
+
async function buildProductionShapedDispatcher(projectRoot: string): Promise<AgentDispatcher> {
|
|
145
|
+
const tools = createToolGuard({ allowedRoots: [projectRoot] });
|
|
146
|
+
// The M4 keystone wiring under test — with CLEO_PI_RUNNER_ENABLED=1 this is
|
|
147
|
+
// the Pi `SkillRunner`; with the flag unset it is `undefined`.
|
|
148
|
+
const runner = await maybeCreatePiRunner({ system: 'task-executor', projectRoot });
|
|
149
|
+
|
|
150
|
+
/** Subprocess fallback — never exercised by the skill-only fixture. */
|
|
151
|
+
const spawn = async (input: AgentDispatchInput): Promise<AgentDispatchResult> => ({
|
|
152
|
+
status: 'failure',
|
|
153
|
+
output: {},
|
|
154
|
+
error: `unexpected subprocess spawn for ${input.agentId}`,
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
async dispatch(input: AgentDispatchInput): Promise<AgentDispatchResult> {
|
|
159
|
+
return runSkillNodeOrSpawn(
|
|
160
|
+
{ nodeId: input.nodeId, agentId: input.agentId, context: input.context },
|
|
161
|
+
{
|
|
162
|
+
tools,
|
|
163
|
+
cwd: projectRoot,
|
|
164
|
+
subprocessSpawn: () => spawn(input),
|
|
165
|
+
...(runner !== undefined ? { runner } : {}),
|
|
166
|
+
},
|
|
167
|
+
);
|
|
168
|
+
},
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ---------------------------------------------------------------------------
|
|
173
|
+
|
|
174
|
+
describe('T11945 (M4): skill-only .cantbook routes through the PiAgentAdapter', () => {
|
|
175
|
+
let db: DatabaseSync;
|
|
176
|
+
let projectRoot: string;
|
|
177
|
+
|
|
178
|
+
const SAVED = {
|
|
179
|
+
session: process.env['CLEO_SESSION_ID'],
|
|
180
|
+
flag: process.env['CLEO_PI_RUNNER_ENABLED'],
|
|
181
|
+
projectRoot: process.env['CLEO_PROJECT_ROOT'],
|
|
182
|
+
cleoRoot: process.env['CLEO_ROOT'],
|
|
183
|
+
anthropic: process.env['ANTHROPIC_API_KEY'],
|
|
184
|
+
anthropicOauth: process.env['ANTHROPIC_OAUTH_TOKEN'],
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
beforeEach(() => {
|
|
188
|
+
db = new DatabaseSync(':memory:');
|
|
189
|
+
db.exec('PRAGMA foreign_keys=ON');
|
|
190
|
+
applyMigration(db, readFileSync(MIGRATION_SQL_PATH, 'utf8'));
|
|
191
|
+
|
|
192
|
+
projectRoot = mkdtempSync(join(tmpdir(), 't11945-pi-e2e-'));
|
|
193
|
+
writeFixtureSkill(projectRoot);
|
|
194
|
+
|
|
195
|
+
// Pin skill + project resolution to the temp root (getProjectRoot honours
|
|
196
|
+
// CLEO_ROOT / CLEO_PROJECT_ROOT before any walk).
|
|
197
|
+
process.env['CLEO_PROJECT_ROOT'] = projectRoot;
|
|
198
|
+
process.env['CLEO_ROOT'] = projectRoot;
|
|
199
|
+
// Daemon-stamped identity — the Pi runner fails closed without it (ZERO authority).
|
|
200
|
+
process.env['CLEO_SESSION_ID'] = 'sess-t11945-e2e';
|
|
201
|
+
// No credential reachable → deterministic cleo-owned no-credential failure.
|
|
202
|
+
delete process.env['ANTHROPIC_API_KEY'];
|
|
203
|
+
delete process.env['ANTHROPIC_OAUTH_TOKEN'];
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
afterEach(() => {
|
|
207
|
+
db.close();
|
|
208
|
+
rmSync(projectRoot, { recursive: true, force: true });
|
|
209
|
+
restoreEnv('CLEO_SESSION_ID', SAVED.session);
|
|
210
|
+
restoreEnv('CLEO_PI_RUNNER_ENABLED', SAVED.flag);
|
|
211
|
+
restoreEnv('CLEO_PROJECT_ROOT', SAVED.projectRoot);
|
|
212
|
+
restoreEnv('CLEO_ROOT', SAVED.cleoRoot);
|
|
213
|
+
restoreEnv('ANTHROPIC_API_KEY', SAVED.anthropic);
|
|
214
|
+
restoreEnv('ANTHROPIC_OAUTH_TOKEN', SAVED.anthropicOauth);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it('routes the in-process skill node THROUGH the Pi adapter (proven by the cleo no-credential failure)', async () => {
|
|
218
|
+
process.env['CLEO_PI_RUNNER_ENABLED'] = '1';
|
|
219
|
+
|
|
220
|
+
const { definition, sourceHash } = parsePlaybook(readFileSync(FIXTURE_PATH, 'utf8'));
|
|
221
|
+
expect(definition.name).toBe('pi-skill-only');
|
|
222
|
+
expect(definition.nodes).toHaveLength(1);
|
|
223
|
+
|
|
224
|
+
const dispatcher = await buildProductionShapedDispatcher(projectRoot);
|
|
225
|
+
|
|
226
|
+
const result = await executePlaybook({
|
|
227
|
+
db,
|
|
228
|
+
playbook: definition,
|
|
229
|
+
playbookHash: sourceHash,
|
|
230
|
+
initialContext: { taskId: 'T11945' },
|
|
231
|
+
dispatcher,
|
|
232
|
+
projectRoot,
|
|
233
|
+
// One attempt → the node fails once (no credential) and terminates.
|
|
234
|
+
maxIterationsDefault: 1,
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
// The Pi loop never throws — the run reaches a clean terminal state with the
|
|
238
|
+
// node failure recorded (not an uncaught exception).
|
|
239
|
+
expect(result.terminalStatus).toBe('exceeded_iteration_cap');
|
|
240
|
+
expect(result.exceededNodeId).toBe('execute');
|
|
241
|
+
|
|
242
|
+
// THE PROOF: the failure carries a cleo-owned Pi marker (streamFn
|
|
243
|
+
// no-credential error OR the wrapPiCall containment prefix). These ONLY
|
|
244
|
+
// appear when the dispatch reached the PiAgentAdapter → Cleo streamFn (the
|
|
245
|
+
// E9 chokepoint). A `defaultSkillRunner` would have SUCCEEDED (it does no LLM
|
|
246
|
+
// work, terminalStatus 'completed'), so an `exceeded_iteration_cap` carrying
|
|
247
|
+
// this marker is dispositive that the Pi route ran.
|
|
248
|
+
const errorContext = result.errorContext ?? '';
|
|
249
|
+
expect(errorContext).toMatch(PI_DISPATCH_MARKERS);
|
|
250
|
+
|
|
251
|
+
const run = getPlaybookRun(db, result.runId);
|
|
252
|
+
expect(run?.status).toBe('failed');
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it('with CLEO_PI_RUNNER_ENABLED unset the node stays on defaultSkillRunner (zero behaviour change)', async () => {
|
|
256
|
+
// Flag deliberately NOT set → maybeCreatePiRunner returns undefined →
|
|
257
|
+
// runSkillNodeOrSpawn uses defaultSkillRunner, which resolves the skill and
|
|
258
|
+
// returns success WITHOUT any LLM work. The run completes; no Pi string.
|
|
259
|
+
delete process.env['CLEO_PI_RUNNER_ENABLED'];
|
|
260
|
+
|
|
261
|
+
const { definition, sourceHash } = parsePlaybook(readFileSync(FIXTURE_PATH, 'utf8'));
|
|
262
|
+
const dispatcher = await buildProductionShapedDispatcher(projectRoot);
|
|
263
|
+
|
|
264
|
+
const result = await executePlaybook({
|
|
265
|
+
db,
|
|
266
|
+
playbook: definition,
|
|
267
|
+
playbookHash: sourceHash,
|
|
268
|
+
initialContext: { taskId: 'T11945' },
|
|
269
|
+
dispatcher,
|
|
270
|
+
projectRoot,
|
|
271
|
+
maxIterationsDefault: 1,
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
expect(result.terminalStatus).toBe('completed');
|
|
275
|
+
// defaultSkillRunner echoes resolution metadata — never a Pi failure marker.
|
|
276
|
+
expect(result.errorContext ?? '').not.toMatch(PI_DISPATCH_MARKERS);
|
|
277
|
+
expect(result.finalContext).toMatchObject({ skillId: 'ct-task-executor', resolved: true });
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
it('maybeCreatePiRunner is default-OFF: undefined unless CLEO_PI_RUNNER_ENABLED=1', async () => {
|
|
281
|
+
delete process.env['CLEO_PI_RUNNER_ENABLED'];
|
|
282
|
+
expect(await maybeCreatePiRunner({ system: 'task-executor', projectRoot })).toBeUndefined();
|
|
283
|
+
|
|
284
|
+
process.env['CLEO_PI_RUNNER_ENABLED'] = '1';
|
|
285
|
+
const runner = await maybeCreatePiRunner({ system: 'task-executor', projectRoot });
|
|
286
|
+
expect(typeof runner).toBe('function');
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
/** Restore an env var to its saved value (delete when it was unset). */
|
|
291
|
+
function restoreEnv(key: string, value: string | undefined): void {
|
|
292
|
+
if (value === undefined) delete process.env[key];
|
|
293
|
+
else process.env[key] = value;
|
|
294
|
+
}
|