@intentius/chant-lexicon-fly 0.46.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/components/capability-plugin.d.ts +6 -0
- package/dist/components/capability-plugin.d.ts.map +1 -0
- package/dist/components/index.d.ts +10 -0
- package/dist/components/index.d.ts.map +1 -0
- package/dist/components/run-agent.d.ts +92 -0
- package/dist/components/run-agent.d.ts.map +1 -0
- package/dist/composites/fly-deploy.d.ts +4 -4
- package/dist/composites/fly-deploy.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/integrity.json +2 -2
- package/dist/manifest.json +1 -1
- package/dist/op/builders.d.ts +73 -0
- package/dist/op/builders.d.ts.map +1 -0
- package/package.json +7 -2
- package/src/components/capability-plugin.test.ts +30 -0
- package/src/components/capability-plugin.ts +36 -0
- package/src/components/index.ts +14 -0
- package/src/components/run-agent-conformance.test.ts +352 -0
- package/src/components/run-agent.test.ts +302 -0
- package/src/components/run-agent.ts +139 -0
- package/src/composites/fly-deploy.ts +4 -4
- package/src/index.ts +16 -5
- package/src/op/builders.test.ts +73 -0
- package/src/op/builders.ts +102 -0
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `run-agent` contract/conformance suite (#1944, epic #1564 phase 4 — the
|
|
3
|
+
* final child of #1564). All offline, against the in-process Sprites fake
|
|
4
|
+
* (`../op/activities/sprites-fake.ts`, S7) and an injected `ProcessRunner`
|
|
5
|
+
* mock (`@intentius/chant/components/verbs/__tests__/mock-process-runner`) —
|
|
6
|
+
* no HTTP/WS to a real endpoint, no `docker`/`cosign`.
|
|
7
|
+
*
|
|
8
|
+
* Distinct from `./run-agent.test.ts` (#1942/#1943's own unit tests, which
|
|
9
|
+
* call the capability directly) in one important way: the tests here run
|
|
10
|
+
* `run-agent` through the actual *component saga* — core's driver
|
|
11
|
+
* (`@intentius/chant/components`'s `runComponentDeploy`, ../capability.ts's
|
|
12
|
+
* `CapabilityRegistry`) — proving the driver-level contract the epic's phase
|
|
13
|
+
* 4 bar requires, not just the capability's own `run()`/`rollback()` pair
|
|
14
|
+
* called back to back.
|
|
15
|
+
*
|
|
16
|
+
* Three things proved here:
|
|
17
|
+
*
|
|
18
|
+
* 1. **Saga-unwind restore.** A composition where `run-agent` succeeds, a
|
|
19
|
+
* LATER step throws, and the saga unwinds — the sprite must be restored
|
|
20
|
+
* to the pre-run checkpoint, through `driver.ts`'s `rollbackExecuted`, not
|
|
21
|
+
* the Op-level `onFailure` path `examples/sprites-agent-task` already
|
|
22
|
+
* demonstrates.
|
|
23
|
+
* 2. **The durable-identity channel (#1944's scope addition, from #1949's
|
|
24
|
+
* review).** On the Temporal durable path, `run()` and `rollback()`
|
|
25
|
+
* execute as separate Activities, each rebuilding `input` fresh — the
|
|
26
|
+
* in-process `WeakMap` `run-agent`'s capability keeps never gets a hit
|
|
27
|
+
* there. `Capability.rollback` grew an optional third `output` parameter
|
|
28
|
+
* for exactly this (`../../../packages/core/src/components/capability.ts`),
|
|
29
|
+
* and `run-agent`'s own rollback prefers it. This file proves the
|
|
30
|
+
* capability's own logic honors that channel — simulating the Activity
|
|
31
|
+
* boundary directly (a completely different `input` object at rollback
|
|
32
|
+
* time, no WeakMap hit possible) — while
|
|
33
|
+
* `lexicons/temporal/src/component-op/runtime.test.ts` proves the
|
|
34
|
+
* generated *codegen* actually threads it end to end through a real
|
|
35
|
+
* Temporal worker.
|
|
36
|
+
* 3. **Attestation conformance, as a composition.** The
|
|
37
|
+
* `run-agent -> sign -> attest-provenance -> verify` chain, run through
|
|
38
|
+
* the driver (not called capability-by-capability, which `./run-agent.
|
|
39
|
+
* test.ts` already covers) — reusing #1951's helpers
|
|
40
|
+
* (`buildRunAgentProvenanceStatement`, `createMockProcessRunner`) rather
|
|
41
|
+
* than duplicating them. A tampered/missing attestation makes `verify`
|
|
42
|
+
* throw `VerificationFailedError`, and the composition fails before any
|
|
43
|
+
* `Apply` phase step runs.
|
|
44
|
+
*/
|
|
45
|
+
import { describe, test, expect, beforeAll, afterAll } from "vitest";
|
|
46
|
+
import { createSpritesFake } from "../op/activities/sprites-fake";
|
|
47
|
+
import { createFlyRunAgentCapability, createFlySpriteActivities } from "./run-agent";
|
|
48
|
+
import type { RunAgentInput, RunAgentOutput } from "@intentius/chant/components/verbs/run-agent";
|
|
49
|
+
import { RUN_AGENT_BUILD_TYPE } from "@intentius/chant/components/verbs/run-agent";
|
|
50
|
+
import { createSignCapability, createAttestProvenanceCapability } from "@intentius/chant/components/verbs/sign";
|
|
51
|
+
import { createVerifyCapability, VerificationFailedError } from "@intentius/chant/components/verbs/verify";
|
|
52
|
+
import { createMockProcessRunner } from "@intentius/chant/components/verbs/__tests__/mock-process-runner";
|
|
53
|
+
import {
|
|
54
|
+
CapabilityRegistry,
|
|
55
|
+
runComponentDeploy,
|
|
56
|
+
type Capability,
|
|
57
|
+
type DeployContext,
|
|
58
|
+
type DriverComponent,
|
|
59
|
+
} from "@intentius/chant/components";
|
|
60
|
+
|
|
61
|
+
const ctx: DeployContext = { env: "dev", component: "agent-turn" };
|
|
62
|
+
|
|
63
|
+
let fake: { url: string; close(): Promise<void> };
|
|
64
|
+
let prevBaseUrl: string | undefined;
|
|
65
|
+
|
|
66
|
+
beforeAll(async () => {
|
|
67
|
+
fake = await createSpritesFake();
|
|
68
|
+
prevBaseUrl = process.env.SPRITES_BASE_URL;
|
|
69
|
+
process.env.SPRITES_BASE_URL = fake.url;
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
afterAll(async () => {
|
|
73
|
+
if (prevBaseUrl === undefined) delete process.env.SPRITES_BASE_URL;
|
|
74
|
+
else process.env.SPRITES_BASE_URL = prevBaseUrl;
|
|
75
|
+
await fake?.close();
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
/** A fake capability that always fails — stands in for "some later step in the composition breaks," triggering saga rollback. */
|
|
79
|
+
function alwaysFailsCapability(kind: string): Capability<Record<string, unknown>, never> {
|
|
80
|
+
return {
|
|
81
|
+
kind,
|
|
82
|
+
async run(): Promise<never> {
|
|
83
|
+
throw new Error(`${kind}: forced failure for saga-unwind conformance`);
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** A fake capability recording every `run()` call — stands in for an apply-style step that must never run once an earlier gate refuses. */
|
|
89
|
+
function markerCapability(kind: string, calls: unknown[]): Capability<Record<string, unknown>, { ok: true }> {
|
|
90
|
+
return {
|
|
91
|
+
kind,
|
|
92
|
+
async run(_c, input): Promise<{ ok: true }> {
|
|
93
|
+
calls.push(input);
|
|
94
|
+
return { ok: true };
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
describe("run-agent — saga-unwind restore through the component driver (#1944)", () => {
|
|
100
|
+
test("run-agent succeeds, a LATER step throws, and the saga unwinds — the sprite is restored to the pre-run checkpoint", async () => {
|
|
101
|
+
// A reused (not freshly created) sprite: run-agent never destroys it on
|
|
102
|
+
// success, so rollback has something real to restore afterward.
|
|
103
|
+
const sprites = createFlySpriteActivities();
|
|
104
|
+
const spriteName = `saga-unwind-${Date.now()}`;
|
|
105
|
+
await sprites.create({ name: spriteName });
|
|
106
|
+
|
|
107
|
+
const registry = new CapabilityRegistry();
|
|
108
|
+
registry.register(createFlyRunAgentCapability());
|
|
109
|
+
registry.register(alwaysFailsCapability("always-fails"));
|
|
110
|
+
|
|
111
|
+
const component: DriverComponent = {
|
|
112
|
+
name: "agent-turn",
|
|
113
|
+
dependsOn: [],
|
|
114
|
+
deploy: [
|
|
115
|
+
{
|
|
116
|
+
phase: "Run",
|
|
117
|
+
steps: [
|
|
118
|
+
{
|
|
119
|
+
kind: "run-agent",
|
|
120
|
+
agent: "echo hi > /work/output",
|
|
121
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
122
|
+
workspace: { spriteName },
|
|
123
|
+
},
|
|
124
|
+
],
|
|
125
|
+
},
|
|
126
|
+
{ phase: "Verify", steps: [{ kind: "always-fails" }] },
|
|
127
|
+
],
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const result = await runComponentDeploy(component, ctx, registry, {});
|
|
131
|
+
|
|
132
|
+
expect(result.ok).toBe(false);
|
|
133
|
+
const runRecord = result.records.find((r) => r.phase === "Run" && r.kind === "run-agent");
|
|
134
|
+
expect(runRecord?.status).toBe("ok");
|
|
135
|
+
const rollbackRecord = result.records.find((r) => r.phase === "Run" && r.kind === "run-agent" && r.status === "rolled-back");
|
|
136
|
+
expect(rollbackRecord).toBeDefined();
|
|
137
|
+
|
|
138
|
+
// Observably restored: /work/output was written during the run, and the
|
|
139
|
+
// pre-run checkpoint predates that write — a direct read now 404s.
|
|
140
|
+
await expect(sprites.readFile({ id: spriteName, path: "/work/output" })).rejects.toThrow();
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("a step with a native rollback is never reported as rollback-opted-out during the same unwind", async () => {
|
|
144
|
+
const sprites = createFlySpriteActivities();
|
|
145
|
+
const spriteName = `saga-unwind-optout-${Date.now()}`;
|
|
146
|
+
await sprites.create({ name: spriteName });
|
|
147
|
+
|
|
148
|
+
const registry = new CapabilityRegistry();
|
|
149
|
+
registry.register(createFlyRunAgentCapability());
|
|
150
|
+
registry.register(alwaysFailsCapability("always-fails"));
|
|
151
|
+
|
|
152
|
+
const component: DriverComponent = {
|
|
153
|
+
name: "agent-turn",
|
|
154
|
+
dependsOn: [],
|
|
155
|
+
deploy: [
|
|
156
|
+
{
|
|
157
|
+
phase: "Run",
|
|
158
|
+
steps: [
|
|
159
|
+
{
|
|
160
|
+
kind: "run-agent",
|
|
161
|
+
agent: "echo hi > /work/output",
|
|
162
|
+
task: { prompt: "irrelevant" },
|
|
163
|
+
workspace: { spriteName },
|
|
164
|
+
},
|
|
165
|
+
],
|
|
166
|
+
},
|
|
167
|
+
{ phase: "Verify", steps: [{ kind: "always-fails" }] },
|
|
168
|
+
],
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
const result = await runComponentDeploy(component, ctx, registry, {});
|
|
172
|
+
const optedOut = result.records.filter((r) => r.kind === "run-agent" && r.status === "rollback-opted-out");
|
|
173
|
+
expect(optedOut).toHaveLength(0);
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
describe("run-agent — durable identity channel (#1944, scope addition from #1949's review)", () => {
|
|
178
|
+
test("rollback restores via output.spriteId/checkpointId even when called with a freshly-rebuilt input object (no WeakMap hit) — the Temporal Activity-boundary shape", async () => {
|
|
179
|
+
// On the Temporal durable path, rollbackCapabilityStep resolves its own
|
|
180
|
+
// fresh `resolvedInput` from JSON every call
|
|
181
|
+
// (lexicons/temporal/src/component-op/activities.ts) — never the same
|
|
182
|
+
// object run() was called with. This test reproduces that exact shape
|
|
183
|
+
// directly against the real capability, without needing a Temporal
|
|
184
|
+
// worker: build input, run(), then rollback() with a DIFFERENT (shallow-
|
|
185
|
+
// cloned) input object, passing run()'s own output as the third
|
|
186
|
+
// parameter — the durable identity channel.
|
|
187
|
+
const capability = createFlyRunAgentCapability();
|
|
188
|
+
const spriteName = `durable-identity-${Date.now()}`;
|
|
189
|
+
const sprites = createFlySpriteActivities();
|
|
190
|
+
await sprites.create({ name: spriteName });
|
|
191
|
+
|
|
192
|
+
const runInput: RunAgentInput = {
|
|
193
|
+
agent: "echo hi > /work/output",
|
|
194
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
195
|
+
workspace: { spriteName },
|
|
196
|
+
};
|
|
197
|
+
const output = await capability.run(ctx, runInput);
|
|
198
|
+
expect(output.turn.status).toBe("completed");
|
|
199
|
+
|
|
200
|
+
// A structurally-equal but reference-DIFFERENT object — resolveStepInput
|
|
201
|
+
// (../../../packages/core/src/components/driver.ts) rebuilds exactly this
|
|
202
|
+
// shape fresh on every Activity call; the WeakMap keyed by runInput's
|
|
203
|
+
// object identity cannot possibly have a hit for this object.
|
|
204
|
+
const rebuiltInput: RunAgentInput = JSON.parse(JSON.stringify(runInput));
|
|
205
|
+
expect(rebuiltInput).not.toBe(runInput);
|
|
206
|
+
|
|
207
|
+
await capability.rollback?.(ctx, rebuiltInput, output);
|
|
208
|
+
|
|
209
|
+
// Restored despite the fresh input object: the pre-run checkpoint predates
|
|
210
|
+
// the write, so a direct read now 404s.
|
|
211
|
+
await expect(sprites.readFile({ id: spriteName, path: "/work/output" })).rejects.toThrow();
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
test("without the output parameter (a caller that never threads it) and no WeakMap hit, rollback degrades to an explicit no-op — the documented pre-#1944 behavior for a caller that doesn't opt in", async () => {
|
|
215
|
+
const capability = createFlyRunAgentCapability();
|
|
216
|
+
const spriteName = `no-identity-${Date.now()}`;
|
|
217
|
+
const sprites = createFlySpriteActivities();
|
|
218
|
+
await sprites.create({ name: spriteName });
|
|
219
|
+
|
|
220
|
+
const runInput: RunAgentInput = {
|
|
221
|
+
agent: "echo hi > /work/output",
|
|
222
|
+
task: { prompt: "irrelevant" },
|
|
223
|
+
workspace: { spriteName },
|
|
224
|
+
};
|
|
225
|
+
await capability.run(ctx, runInput);
|
|
226
|
+
|
|
227
|
+
const rebuiltInput: RunAgentInput = JSON.parse(JSON.stringify(runInput));
|
|
228
|
+
// No third argument at all: neither the WeakMap (different object) nor
|
|
229
|
+
// `output` has anything to restore by for spriteId/checkpointId — but
|
|
230
|
+
// `workspace.spriteName` is still present, so rollback falls back to
|
|
231
|
+
// comment-based restore rather than a bare no-op.
|
|
232
|
+
await capability.rollback?.(ctx, rebuiltInput);
|
|
233
|
+
|
|
234
|
+
// Comment-based restore still finds the "pre-run" checkpoint via
|
|
235
|
+
// workspace.spriteName, so this still restores correctly — the pure
|
|
236
|
+
// no-op only happens with no spriteId at all (see run-agent.ts's
|
|
237
|
+
// rollback()), which needs a freshly-created (not spriteName-reused)
|
|
238
|
+
// sprite to reach; that path stays a no-op by design (nothing to
|
|
239
|
+
// identify) and isn't newly broken by this change.
|
|
240
|
+
await expect(sprites.readFile({ id: spriteName, path: "/work/output" })).rejects.toThrow();
|
|
241
|
+
});
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
// ── #1944 design point 3: attestation conformance as a composition ─────────
|
|
245
|
+
|
|
246
|
+
describe("run-agent -> sign -> attest-provenance -> verify, run as a component composition through the driver (#1944)", () => {
|
|
247
|
+
const POLICY = {
|
|
248
|
+
expectedIssuer: "https://token.actions.githubusercontent.com",
|
|
249
|
+
expectedIdentity: "https://github.com/my-org/my-repo/.github/workflows/release.yml@refs/heads/main",
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
function buildChainComponent(): DriverComponent {
|
|
253
|
+
return {
|
|
254
|
+
name: "agent-turn",
|
|
255
|
+
dependsOn: [],
|
|
256
|
+
deploy: [
|
|
257
|
+
{
|
|
258
|
+
phase: "Run",
|
|
259
|
+
steps: [
|
|
260
|
+
{
|
|
261
|
+
kind: "run-agent",
|
|
262
|
+
agent: "echo hi > /work/output",
|
|
263
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
264
|
+
workspace: {},
|
|
265
|
+
},
|
|
266
|
+
],
|
|
267
|
+
},
|
|
268
|
+
{ phase: "Sign", steps: [{ kind: "sign", imageRef: "@Run.attestationRef" }] },
|
|
269
|
+
{
|
|
270
|
+
phase: "Attest",
|
|
271
|
+
steps: [
|
|
272
|
+
{
|
|
273
|
+
kind: "attest-provenance",
|
|
274
|
+
imageRef: "@Run.attestationRef",
|
|
275
|
+
provenance: "@Run.provenance",
|
|
276
|
+
builderId: "https://github.com/actions/runner",
|
|
277
|
+
buildType: RUN_AGENT_BUILD_TYPE,
|
|
278
|
+
externalParameters: { agent: "echo hi > /work/output" },
|
|
279
|
+
internalParameters: {
|
|
280
|
+
spriteId: "@Run.spriteId",
|
|
281
|
+
checkpointId: "@Run.checkpointId",
|
|
282
|
+
turnStatus: "@Run.turn.status",
|
|
283
|
+
turnExitCode: "@Run.turn.exitCode",
|
|
284
|
+
},
|
|
285
|
+
},
|
|
286
|
+
],
|
|
287
|
+
},
|
|
288
|
+
{ phase: "Verify", steps: [{ kind: "verify", imageRef: "@Run.attestationRef", policy: POLICY }] },
|
|
289
|
+
{ phase: "Apply", steps: [{ kind: "apply-marker" }] },
|
|
290
|
+
],
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
test("the full chain succeeds and the Apply phase runs", async () => {
|
|
295
|
+
const proc = createMockProcessRunner();
|
|
296
|
+
const applyCalls: unknown[] = [];
|
|
297
|
+
const registry = new CapabilityRegistry();
|
|
298
|
+
registry.register(createFlyRunAgentCapability());
|
|
299
|
+
registry.register(createSignCapability(proc.runner));
|
|
300
|
+
registry.register(createAttestProvenanceCapability(proc.runner));
|
|
301
|
+
registry.register(createVerifyCapability(proc.runner));
|
|
302
|
+
registry.register(markerCapability("apply-marker", applyCalls));
|
|
303
|
+
|
|
304
|
+
const result = await runComponentDeploy(buildChainComponent(), ctx, registry, {});
|
|
305
|
+
|
|
306
|
+
expect(result.ok).toBe(true);
|
|
307
|
+
expect(applyCalls).toHaveLength(1);
|
|
308
|
+
|
|
309
|
+
const verifyRecord = result.records.find((r) => r.kind === "verify");
|
|
310
|
+
expect(verifyRecord?.status).toBe("ok");
|
|
311
|
+
expect(verifyRecord?.output).toMatchObject({ verified: true, checked: ["signature", "provenance"] });
|
|
312
|
+
|
|
313
|
+
// The wiring actually resolved a real, digest-qualified attestationRef —
|
|
314
|
+
// not a passthrough literal — proving @Run.* references threaded through
|
|
315
|
+
// sign/attest-provenance/verify exactly as #1943 designed.
|
|
316
|
+
const runOutput = result.records.find((r) => r.kind === "run-agent")?.output as RunAgentOutput;
|
|
317
|
+
expect(runOutput.attestationRef).toMatch(/^agent-turn\/run-agent@sha256:[0-9a-f]{64}$/);
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
test("a tampered/missing attestation makes verify throw VerificationFailedError, and the composition fails before the Apply phase ever runs", async () => {
|
|
321
|
+
const proc = createMockProcessRunner({
|
|
322
|
+
failures: { "cosign verify-attestation": "Error: no matching attestations found for the given subject digest" },
|
|
323
|
+
});
|
|
324
|
+
const applyCalls: unknown[] = [];
|
|
325
|
+
const registry = new CapabilityRegistry();
|
|
326
|
+
registry.register(createFlyRunAgentCapability());
|
|
327
|
+
registry.register(createSignCapability(proc.runner));
|
|
328
|
+
registry.register(createAttestProvenanceCapability(proc.runner));
|
|
329
|
+
registry.register(createVerifyCapability(proc.runner));
|
|
330
|
+
registry.register(markerCapability("apply-marker", applyCalls));
|
|
331
|
+
|
|
332
|
+
const result = await runComponentDeploy(buildChainComponent(), ctx, registry, {});
|
|
333
|
+
|
|
334
|
+
expect(result.ok).toBe(false);
|
|
335
|
+
// Apply never ran: runComponentDeploy stops at the first failing phase —
|
|
336
|
+
// the "Apply" DriverPhase is never entered at all once "Verify" throws.
|
|
337
|
+
expect(applyCalls).toHaveLength(0);
|
|
338
|
+
|
|
339
|
+
const verifyRecord = result.records.find((r) => r.kind === "verify");
|
|
340
|
+
expect(verifyRecord?.status).toBe("fail");
|
|
341
|
+
expect(verifyRecord?.error).toContain("verification failed");
|
|
342
|
+
|
|
343
|
+
// Confirm the underlying capability really throws VerificationFailedError
|
|
344
|
+
// (runComponentDeploy's fail record only carries the stringified message,
|
|
345
|
+
// per driver.ts's runCapabilityStep) — same assertion ./run-agent.test.ts
|
|
346
|
+
// makes when calling verify directly, reused here rather than re-derived.
|
|
347
|
+
const verifyCapability = createVerifyCapability(proc.runner);
|
|
348
|
+
await expect(
|
|
349
|
+
verifyCapability.run(ctx, { imageRef: "agent-turn/run-agent@sha256:" + "0".repeat(64), policy: POLICY }),
|
|
350
|
+
).rejects.toThrow(VerificationFailedError);
|
|
351
|
+
});
|
|
352
|
+
});
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests the fly lexicon's `run-agent` adapter (#1942, epic #1564 phase 2):
|
|
3
|
+
* the real `SpriteActivities` implementation (`./run-agent.ts`) against the
|
|
4
|
+
* offline in-process sprites fake (`../op/activities/sprites-fake.ts`, S7) —
|
|
5
|
+
* no HTTP/WS to a real endpoint, no Docker, so this runs in CI the same way
|
|
6
|
+
* `../op/activities/sprites.integration.test.ts` does. Core's own
|
|
7
|
+
* `run()`/`rollback()` sequencing tests live in
|
|
8
|
+
* `packages/core/src/components/verbs/run-agent.test.ts` against a
|
|
9
|
+
* hand-written fake; this file is specifically about the real wire-level
|
|
10
|
+
* reclassification (the exec-throw finding) and the real activities wired
|
|
11
|
+
* end to end.
|
|
12
|
+
*/
|
|
13
|
+
import { describe, test, expect, beforeAll, afterAll } from "vitest";
|
|
14
|
+
import { createSpritesFake } from "../op/activities/sprites-fake";
|
|
15
|
+
import { createFlyRunAgentCapability, createFlySpriteActivities, parseSpriteExecFailure } from "./run-agent";
|
|
16
|
+
import {
|
|
17
|
+
buildRunAgentProvenanceStatement,
|
|
18
|
+
extractTranscriptDigest,
|
|
19
|
+
RUN_AGENT_BUILD_TYPE,
|
|
20
|
+
type RunAgentInput,
|
|
21
|
+
} from "@intentius/chant/components/verbs/run-agent";
|
|
22
|
+
import {
|
|
23
|
+
createAttestProvenanceCapability,
|
|
24
|
+
createSignCapability,
|
|
25
|
+
SignTargetNotDigestError,
|
|
26
|
+
} from "@intentius/chant/components/verbs/sign";
|
|
27
|
+
import {
|
|
28
|
+
createVerifyCapability,
|
|
29
|
+
VerificationFailedError,
|
|
30
|
+
} from "@intentius/chant/components/verbs/verify";
|
|
31
|
+
import { createMockProcessRunner } from "@intentius/chant/components/verbs/__tests__/mock-process-runner";
|
|
32
|
+
|
|
33
|
+
const ctx = { env: "dev", component: "review-agent" };
|
|
34
|
+
|
|
35
|
+
let fake: { url: string; close(): Promise<void> };
|
|
36
|
+
let prevBaseUrl: string | undefined;
|
|
37
|
+
|
|
38
|
+
beforeAll(async () => {
|
|
39
|
+
fake = await createSpritesFake();
|
|
40
|
+
prevBaseUrl = process.env.SPRITES_BASE_URL;
|
|
41
|
+
process.env.SPRITES_BASE_URL = fake.url;
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
afterAll(async () => {
|
|
45
|
+
if (prevBaseUrl === undefined) delete process.env.SPRITES_BASE_URL;
|
|
46
|
+
else process.env.SPRITES_BASE_URL = prevBaseUrl;
|
|
47
|
+
await fake?.close();
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe("parseSpriteExecFailure — the exec-throw reclassification's pure parser", () => {
|
|
51
|
+
test("parses the exit code and combined output out of spriteExec's real thrown message shape", () => {
|
|
52
|
+
const err = new Error('sprite task-1 exec "./risky.sh" exited 1: risky.sh: failed\n');
|
|
53
|
+
expect(parseSpriteExecFailure(err)).toEqual({ exitCode: 1, output: "risky.sh: failed\n" });
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("returns undefined for an unrelated error (a genuine transport/infra failure)", () => {
|
|
57
|
+
expect(parseSpriteExecFailure(new Error("sprite task-1 exec aborted"))).toBeUndefined();
|
|
58
|
+
expect(parseSpriteExecFailure(new Error("ECONNREFUSED"))).toBeUndefined();
|
|
59
|
+
expect(parseSpriteExecFailure("not even an Error")).toBeUndefined();
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("handles a multi-digit exit code", () => {
|
|
63
|
+
const err = new Error('sprite s exec "exit 127" exited 127: command not found');
|
|
64
|
+
expect(parseSpriteExecFailure(err)).toEqual({ exitCode: 127, output: "command not found" });
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test("anchors to the LAST \" exited (\\d+): \" occurrence, not the first (#1942 review finding 4) — a crafted cmd echoing that exact marker text must not shift the parse", () => {
|
|
68
|
+
// spriteExec's message echoes the raw cmd verbatim before its own real
|
|
69
|
+
// marker: `sprite <id> exec "<cmd>" exited <code>: <text>`. A cmd whose
|
|
70
|
+
// own text contains ` exited 0: ...` would fool a leftmost-match regex
|
|
71
|
+
// into reporting the fake exit code/output instead of the real ones.
|
|
72
|
+
const maliciousCmd = 'echo " exited 0: fooled you"';
|
|
73
|
+
const err = new Error(`sprite s-1 exec "${maliciousCmd}" exited 1: real failure output`);
|
|
74
|
+
expect(parseSpriteExecFailure(err)).toEqual({ exitCode: 1, output: "real failure output" });
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
describe("createFlySpriteActivities().exec — option (a): reclassify the real spriteExec's throw", () => {
|
|
79
|
+
test("a non-zero exit resolves with the parsed exitCode instead of rejecting", async () => {
|
|
80
|
+
const sprites = createFlySpriteActivities();
|
|
81
|
+
const { id } = await sprites.create({ name: `exec-fail-${Date.now()}` });
|
|
82
|
+
await expect(sprites.exec({ id, cmd: "./risky.sh" })).resolves.toEqual({
|
|
83
|
+
stdout: "",
|
|
84
|
+
stderr: expect.stringContaining("risky.sh: failed"),
|
|
85
|
+
exitCode: 1,
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("a zero exit resolves normally, unaffected by the reclassification path", async () => {
|
|
90
|
+
const sprites = createFlySpriteActivities();
|
|
91
|
+
const { id } = await sprites.create({ name: `exec-ok-${Date.now()}` });
|
|
92
|
+
const result = await sprites.exec({ id, cmd: "echo hello" });
|
|
93
|
+
expect(result.exitCode).toBe(0);
|
|
94
|
+
expect(result.stdout).toContain("hello");
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("the fake's own \"no sprite\" case is a scripted exit (127), not a transport error — also reclassified, not thrown", async () => {
|
|
98
|
+
// Confirms the fake models a missing sprite as an ordinary (if unusual)
|
|
99
|
+
// non-zero exit over the wire, exercising the same reclassification path
|
|
100
|
+
// as any other command failure — distinct from the next test's real
|
|
101
|
+
// connection-level failure.
|
|
102
|
+
const sprites = createFlySpriteActivities();
|
|
103
|
+
await expect(sprites.exec({ id: "does-not-exist", cmd: "true" })).resolves.toEqual({
|
|
104
|
+
stdout: "",
|
|
105
|
+
stderr: expect.stringContaining("no sprite"),
|
|
106
|
+
exitCode: 127,
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test("a genuine transport failure (connection refused) still rejects, not reclassified", async () => {
|
|
111
|
+
const prev = process.env.SPRITES_BASE_URL;
|
|
112
|
+
process.env.SPRITES_BASE_URL = "http://127.0.0.1:1"; // nothing listens on port 1
|
|
113
|
+
try {
|
|
114
|
+
const sprites = createFlySpriteActivities();
|
|
115
|
+
await expect(sprites.exec({ id: "s-1", cmd: "true" })).rejects.toThrow();
|
|
116
|
+
} finally {
|
|
117
|
+
process.env.SPRITES_BASE_URL = prev;
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
describe("createFlyRunAgentCapability — end to end against the offline fake (#1942)", () => {
|
|
123
|
+
test("a successful turn: create -> checkpoint -> stage -> exec -> collect -> destroy", async () => {
|
|
124
|
+
const capability = createFlyRunAgentCapability();
|
|
125
|
+
const input: RunAgentInput = {
|
|
126
|
+
agent: "echo hi > /work/output", // unrecognized "runtime" -> passthrough (see core's buildRuntimeCommand)
|
|
127
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
128
|
+
workspace: {},
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
const output = await capability.run(ctx, input);
|
|
132
|
+
|
|
133
|
+
expect(output.turn.status).toBe("completed");
|
|
134
|
+
expect(output.turn.exitCode).toBe(0);
|
|
135
|
+
expect(output.artifacts.files).toEqual([
|
|
136
|
+
{ path: "/work/output", digest: expect.stringMatching(/^sha256:[0-9a-f]{64}$/) },
|
|
137
|
+
]);
|
|
138
|
+
|
|
139
|
+
// Destroyed on success — the sprite is gone, so a direct read 404s.
|
|
140
|
+
const sprites = createFlySpriteActivities();
|
|
141
|
+
await expect(sprites.readFile({ id: output.spriteId, path: "/work/output" })).rejects.toThrow();
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test("a failed turn (real spriteExec throw, reclassified): status \"failed\", no throw, sprite left alive, rollback restores it", async () => {
|
|
145
|
+
const capability = createFlyRunAgentCapability();
|
|
146
|
+
const input: RunAgentInput = {
|
|
147
|
+
agent: "./risky.sh", // the fake's scripted failing job (lexicons/fly/.../sprites-fake.ts): mutates /work/output then exits 1
|
|
148
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
149
|
+
workspace: {},
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const output = await capability.run(ctx, input);
|
|
153
|
+
|
|
154
|
+
expect(output.turn.status).toBe("failed");
|
|
155
|
+
expect(output.turn.exitCode).toBe(1);
|
|
156
|
+
expect(output.artifacts.files).toEqual([
|
|
157
|
+
{ path: "/work/output", digest: expect.stringMatching(/^sha256:[0-9a-f]{64}$/) },
|
|
158
|
+
]);
|
|
159
|
+
|
|
160
|
+
// Left alive: reading it back directly still works (not destroyed).
|
|
161
|
+
const sprites = createFlySpriteActivities();
|
|
162
|
+
const { content } = await sprites.readFile({ id: output.spriteId, path: "/work/output" });
|
|
163
|
+
expect(content).toBe("partial-corrupt");
|
|
164
|
+
|
|
165
|
+
// rollback() restores the pre-run checkpoint — /work/output never existed
|
|
166
|
+
// at that point (the checkpoint predates both staging and exec).
|
|
167
|
+
await capability.rollback?.(ctx, input);
|
|
168
|
+
await expect(sprites.readFile({ id: output.spriteId, path: "/work/output" })).rejects.toThrow();
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("reuses an existing sprite when workspace.spriteName is given — no create, no destroy", async () => {
|
|
172
|
+
const sprites = createFlySpriteActivities();
|
|
173
|
+
const name = `warm-${Date.now()}`;
|
|
174
|
+
await sprites.create({ name });
|
|
175
|
+
|
|
176
|
+
const capability = createFlyRunAgentCapability();
|
|
177
|
+
const input: RunAgentInput = {
|
|
178
|
+
agent: "echo warm > /work/output",
|
|
179
|
+
task: { prompt: "n/a" },
|
|
180
|
+
workspace: { spriteName: name },
|
|
181
|
+
};
|
|
182
|
+
const output = await capability.run(ctx, input);
|
|
183
|
+
|
|
184
|
+
expect(output.spriteId).toBe(name);
|
|
185
|
+
// Still alive post-success (reuse is never destroyed) — a direct read succeeds.
|
|
186
|
+
const { content } = await sprites.readFile({ id: name, path: "/work/output" });
|
|
187
|
+
expect(content).toContain("warm");
|
|
188
|
+
});
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
// ── #1943: provenance + attestation, verify-gate interop over a real turn ───
|
|
192
|
+
|
|
193
|
+
describe("run-agent -> sign -> attest-provenance -> verify (#1943), a real turn against the offline sprites-fake", () => {
|
|
194
|
+
const POLICY = {
|
|
195
|
+
expectedIssuer: "https://token.actions.githubusercontent.com",
|
|
196
|
+
expectedIdentity: "https://github.com/my-org/my-repo/.github/workflows/release.yml@refs/heads/main",
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
test("a valid attestation over a real sprite turn's output passes sign/attest-provenance/verify completely unmodified", async () => {
|
|
200
|
+
const capability = createFlyRunAgentCapability();
|
|
201
|
+
const input: RunAgentInput = {
|
|
202
|
+
agent: "echo hi > /work/output",
|
|
203
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
204
|
+
workspace: {},
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
// A real turn against the offline fake — not a hand-rolled SpriteActivities mock.
|
|
208
|
+
const output = await capability.run(ctx, input);
|
|
209
|
+
expect(output.turn.status).toBe("completed");
|
|
210
|
+
expect(output.attestationRef).toMatch(/^review-agent\/run-agent@sha256:[0-9a-f]{64}$/);
|
|
211
|
+
|
|
212
|
+
const proc = createMockProcessRunner();
|
|
213
|
+
const signCapability = createSignCapability(proc.runner);
|
|
214
|
+
const attestCapability = createAttestProvenanceCapability(proc.runner);
|
|
215
|
+
const verifyCapability = createVerifyCapability(proc.runner);
|
|
216
|
+
|
|
217
|
+
// sign/attest-provenance accept output.attestationRef exactly like any
|
|
218
|
+
// other digest-qualified imageRef — no new code path, no
|
|
219
|
+
// SignTargetNotDigestError.
|
|
220
|
+
const signOutput = await signCapability.run(ctx, { imageRef: output.attestationRef });
|
|
221
|
+
expect(signOutput).toEqual({ imageRef: output.attestationRef, signed: true, method: "keyless" });
|
|
222
|
+
|
|
223
|
+
const statement = buildRunAgentProvenanceStatement(input, output, "https://github.com/actions/runner");
|
|
224
|
+
expect(statement.predicateType).toBe("https://slsa.dev/provenance/v1");
|
|
225
|
+
expect(statement.predicate.buildDefinition.buildType).toBe(RUN_AGENT_BUILD_TYPE);
|
|
226
|
+
|
|
227
|
+
const attestOutput = await attestCapability.run(ctx, {
|
|
228
|
+
imageRef: output.attestationRef,
|
|
229
|
+
provenance: output.provenance,
|
|
230
|
+
builderId: "https://github.com/actions/runner",
|
|
231
|
+
buildType: RUN_AGENT_BUILD_TYPE,
|
|
232
|
+
externalParameters: { agent: input.agent },
|
|
233
|
+
internalParameters: {
|
|
234
|
+
spriteId: output.spriteId,
|
|
235
|
+
checkpointId: output.checkpointId,
|
|
236
|
+
turnStatus: output.turn.status,
|
|
237
|
+
turnExitCode: output.turn.exitCode,
|
|
238
|
+
},
|
|
239
|
+
});
|
|
240
|
+
expect(attestOutput.attested).toBe(true);
|
|
241
|
+
|
|
242
|
+
// The unmodified verify gate (imageRef: "@RunAgent.attestationRef" in a
|
|
243
|
+
// real component's step wiring) accepts it — no requireProvenance
|
|
244
|
+
// override, no predicateType plumbing needed on the verify side (see
|
|
245
|
+
// packages/core/src/components/verbs/run-agent.ts's doc comment,
|
|
246
|
+
// "Predicate-type decision, made").
|
|
247
|
+
const verifyOutput = await verifyCapability.run(ctx, { imageRef: output.attestationRef, policy: POLICY });
|
|
248
|
+
expect(verifyOutput).toEqual({ verified: true, checked: ["signature", "provenance"] });
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
test("the same unmodified verify gate refuses when the attestation doesn't check out", async () => {
|
|
252
|
+
const capability = createFlyRunAgentCapability();
|
|
253
|
+
const input: RunAgentInput = {
|
|
254
|
+
agent: "echo hi > /work/output",
|
|
255
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
256
|
+
workspace: {},
|
|
257
|
+
};
|
|
258
|
+
const output = await capability.run(ctx, input);
|
|
259
|
+
|
|
260
|
+
// Simulates a tampered/missing attestation the same way verify.test.ts's
|
|
261
|
+
// own "FAILS the deploy on missing/invalid provenance" suite does — cosign
|
|
262
|
+
// itself is the one that would refuse a mismatched digest or a stripped
|
|
263
|
+
// attestation in reality; this proves `verify` (composed after
|
|
264
|
+
// `run-agent`, unmodified) propagates that refusal as
|
|
265
|
+
// VerificationFailedError rather than passing the gate silently.
|
|
266
|
+
const proc = createMockProcessRunner({
|
|
267
|
+
failures: { "cosign verify-attestation": "Error: no matching attestations found for the given subject digest" },
|
|
268
|
+
});
|
|
269
|
+
const verifyCapability = createVerifyCapability(proc.runner);
|
|
270
|
+
|
|
271
|
+
await expect(
|
|
272
|
+
verifyCapability.run(ctx, { imageRef: output.attestationRef, policy: POLICY }),
|
|
273
|
+
).rejects.toThrow(VerificationFailedError);
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
test("a turn whose output differs by one byte produces a different attestationRef — verifying the original digest against the tampered turn's evidence would target the wrong subject", async () => {
|
|
277
|
+
const capability = createFlyRunAgentCapability();
|
|
278
|
+
const original = await capability.run(ctx, {
|
|
279
|
+
agent: "echo original > /work/output",
|
|
280
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
281
|
+
workspace: {},
|
|
282
|
+
});
|
|
283
|
+
const tampered = await capability.run(ctx, {
|
|
284
|
+
agent: "echo tampered > /work/output",
|
|
285
|
+
task: { prompt: "irrelevant for this scripted command" },
|
|
286
|
+
workspace: {},
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
expect(original.attestationRef).not.toBe(tampered.attestationRef);
|
|
290
|
+
expect(extractTranscriptDigest(original.provenance.sourceRef)).not.toBe(
|
|
291
|
+
extractTranscriptDigest(tampered.provenance.sourceRef),
|
|
292
|
+
);
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
test("refuses to sign a non-digest-shaped reference the same way it would for any other verb — attestationRef is always digest-qualified by construction", async () => {
|
|
296
|
+
const proc = createMockProcessRunner();
|
|
297
|
+
const signCapability = createSignCapability(proc.runner);
|
|
298
|
+
await expect(signCapability.run(ctx, { imageRef: "review-agent/run-agent:latest" })).rejects.toThrow(
|
|
299
|
+
SignTargetNotDigestError,
|
|
300
|
+
);
|
|
301
|
+
});
|
|
302
|
+
});
|