@agentswf/engine 0.1.0-beta.4 → 0.1.0-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/package.json +4 -4
  2. package/src/agent-cli/cli.ts +57 -17
  3. package/src/attempt-ending.ts +2 -4
  4. package/src/attempt-output.ts +2 -1
  5. package/src/attempt-view.ts +1 -1
  6. package/src/control-plane.ts +37 -5
  7. package/src/coordinator-socket.ts +15 -6
  8. package/src/deadlines.ts +69 -2
  9. package/src/decisions/directory.ts +9 -11
  10. package/src/deprecations.ts +177 -0
  11. package/src/duration.ts +50 -20
  12. package/src/emit-command.ts +55 -41
  13. package/src/emit.ts +124 -15
  14. package/src/help/emit.txt +12 -4
  15. package/src/help/inbox.txt +2 -2
  16. package/src/help/logs.txt +5 -4
  17. package/src/help/reply.txt +5 -5
  18. package/src/help/run.txt +21 -7
  19. package/src/herdr-run-session.ts +2 -2
  20. package/src/holder-command.ts +36 -31
  21. package/src/inbox-command.ts +7 -10
  22. package/src/init-template/hello/workflow.test.ts.template +3 -3
  23. package/src/init-template/hello/workflow.ts.template +2 -2
  24. package/src/keep-alive.ts +28 -13
  25. package/src/logs-command.ts +12 -12
  26. package/src/mailbox.ts +2 -20
  27. package/src/messaging/limits.ts +4 -4
  28. package/src/messaging/messaging.ts +26 -21
  29. package/src/messaging/routes.ts +1 -1
  30. package/src/messaging/text.ts +7 -5
  31. package/src/operator-cli.ts +23 -9
  32. package/src/operator-runtime.ts +5 -5
  33. package/src/pane-layout.ts +125 -32
  34. package/src/progress-view.ts +8 -8
  35. package/src/run-command.ts +25 -4
  36. package/src/run-commands.ts +1 -1
  37. package/src/run-prepare.ts +46 -8
  38. package/src/run-usage.ts +1 -1
  39. package/src/run-view.ts +9 -4
  40. package/src/runs-command.ts +12 -10
  41. package/src/runs.ts +44 -8
  42. package/src/sources.ts +42 -6
  43. package/src/stage-ledger.ts +42 -12
  44. package/src/stage-plan.ts +37 -1
  45. package/src/test-command.ts +7 -3
  46. package/src/testing.ts +7 -8
  47. package/src/tools.ts +46 -18
  48. package/src/workflow-loader.ts +5 -2
  49. package/src/workflow-runner.ts +553 -235
  50. package/src/workflow-testing/coordinator.typecheck.ts +7 -7
  51. package/src/workflow-testing/coordinators.ts +2 -2
  52. package/src/workflow-testing/host.ts +107 -38
  53. package/src/workflow-testing/index.ts +45 -836
  54. package/src/workflow-testing/run-test.ts +1213 -0
  55. package/src/workflow-testing/script.ts +102 -41
  56. package/src/workflow-testing/workflow-testing.typecheck.ts +11 -8
@@ -0,0 +1,1213 @@
1
+ import { mkdtempSync, rmSync } from "node:fs";
2
+ import { mkdir } from "node:fs/promises";
3
+ import { tmpdir } from "node:os";
4
+ import { dirname, isAbsolute, join, relative, resolve } from "node:path";
5
+ import {
6
+ MAILBOX_RECORD_VERSION,
7
+ type MailboxRecord,
8
+ type SandboxRecord,
9
+ STAGE_RECORD_VERSION,
10
+ type StageNeed,
11
+ type StageRecord,
12
+ } from "@agentswf/contract/records";
13
+ import {
14
+ AWF_ID_PREFIX,
15
+ awfEventOf,
16
+ type Duration,
17
+ type Ending,
18
+ type EventSchemas,
19
+ type HarnessAllowance,
20
+ isEventWorkflow,
21
+ type JsonObject,
22
+ type JsonValue,
23
+ type OutputOf,
24
+ type ParticipantReply,
25
+ positiveDurationMilliseconds,
26
+ type RuntimeAliases,
27
+ WORKFLOW_KIND,
28
+ type Workflow,
29
+ } from "@agentswf/contract/workflow";
30
+ import { type Harness, sandboxTokens } from "@agentswf/harness";
31
+ import { createFakeSandboxProvider } from "@agentswf/sandbox/testing/fake";
32
+ import { decideEnding, type Finished } from "../attempt-ending";
33
+ import { continueCommand } from "../attempt-view";
34
+ import { type Deprecated, RunDeprecations } from "../deprecations";
35
+ import { checkEvent, eventIdProblem, replyRecord } from "../emit";
36
+ import { messageOf } from "../errors";
37
+ import { appendMailbox, foldMailbox, RunParked, readMailbox } from "../mailbox";
38
+ import { OPERATOR_ALIASES } from "../operator-aliases";
39
+ import {
40
+ createRun,
41
+ idProblem,
42
+ readEvents,
43
+ readStageRecords,
44
+ readTurns,
45
+ writeStageRecord,
46
+ } from "../runs";
47
+ import { primaryFailure, WorkflowStopped } from "../stopped";
48
+ import { assertJsonValue, type EventWorkflow } from "../workflow-loader";
49
+ import type { RunnableWorkflow } from "../workflow-runner";
50
+ import { runWorkflow, type SettledRun, WorkflowRunError } from "../workflow-runner";
51
+ import { ParticipantScripts } from "./coordinators";
52
+ import { createScriptedDecisions, type DecisionRequest, type DecisionScript } from "./decisions";
53
+ import {
54
+ type AgentSandbox,
55
+ type CompactionRecord,
56
+ createScriptedHost,
57
+ type OpenedAgent,
58
+ type SetRecord,
59
+ type TestTurn,
60
+ } from "./host";
61
+ import {
62
+ type MessageScript,
63
+ MessageScripts,
64
+ type ParticipantScript,
65
+ type Script,
66
+ Scripts,
67
+ } from "./script";
68
+
69
+ /**
70
+ * What a test gives `testWorkflow`, for a workflow taking `Args`, and with `Events` when it declares
71
+ * them. A key not listed here is refused when the test runs. Bare, `TestOptions` is what any test
72
+ * may share, with no args or events: spread into a test's own, it gives them.
73
+ */
74
+ export type TestOptions<
75
+ Args extends JsonValue = never,
76
+ Events extends EventSchemas = never,
77
+ > = CommonTestOptions & ArgsOption<Args, Events> & EventOptions<Events>;
78
+
79
+ /**
80
+ * `args` is required where the workflow has no events and `null` is not among its args. Absent, the
81
+ * run gets what `prepare` makes of no argv, as `awf run` given none, or a run an event starts, has.
82
+ */
83
+ type ArgsOption<Args extends JsonValue, Events extends EventSchemas> = [Args] extends [never]
84
+ ? { args?: undefined }
85
+ : [Events] extends [never]
86
+ ? null extends Args
87
+ ? {
88
+ /** The args `run` is given, as they are: `prepare` isn't called for them. */
89
+ args?: Args;
90
+ }
91
+ : {
92
+ /** The args `run` is given, as they are: `prepare` isn't called for them. */
93
+ args: Args;
94
+ }
95
+ : {
96
+ /**
97
+ * Its args, as `awf run --id {id} -- {args}` would prepare them; absent, what `prepare` makes
98
+ * of no argv, as a run an event starts has.
99
+ */
100
+ args?: Args;
101
+ };
102
+
103
+ /** What only a test of a workflow with events gives: what its run starts with, and is sent. */
104
+ type EventOptions<Events extends EventSchemas> = [Events] extends [never]
105
+ ? unknown
106
+ : {
107
+ /** The event the run starts with, as `awf emit` would send it; absent, it starts with none. */
108
+ event?: {
109
+ [Name in keyof Events & string]: { name: Name; data: OutputOf<Events[Name]> } & EventIds;
110
+ }[keyof Events & string];
111
+ /** Sent at points in the run, each once its point is reached. */
112
+ events?: readonly ScriptedEvent<Events>[];
113
+ };
114
+
115
+ /** What an earlier attempt left: given, the test's attempt is that run's continue, attempt 2. */
116
+ export type EarlierAttempt = {
117
+ /**
118
+ * The stages it recorded, by name: each value, or `undefined` for a stage that returns nothing,
119
+ * reused without calling their work, as `awf run --continue` would.
120
+ */
121
+ stages?: Readonly<Record<string, JsonValue | undefined>>;
122
+ /**
123
+ * What it left in the run's mailbox, in order: its events, timers, and what its waits and turns
124
+ * wrote. A starting `event` is appended after them, as the event that woke it.
125
+ */
126
+ mailbox?: readonly MailboxRecord[];
127
+ };
128
+
129
+ type CommonTestOptions = {
130
+ /** The run's id, as `awf run --id` or a sender names it; `"test"`. */
131
+ id?: string;
132
+ /** Laid out before the run, by path under its directory, which the test removes. */
133
+ files?: Readonly<Record<string, FileTree>>;
134
+ /** Each agent's script, by its key or a pattern such as `"review:*"`. */
135
+ agents?: Readonly<Record<string, Script>>;
136
+ /**
137
+ * Each agent's compactions, apart from its turns: `answer("summary")` or a `noAnswer`, a list one
138
+ * entry per compaction. An agent with none here has every compaction answer `""`.
139
+ */
140
+ compactions?: Readonly<Record<string, Script>>;
141
+ /**
142
+ * Each agent's message operations, apart from its turns, by its key or a pattern: a script for
143
+ * all of them, or a list, one per operation. An agent a message wakes with none here fails the
144
+ * test.
145
+ */
146
+ messages?: Readonly<Record<string, MessageScript | readonly MessageScript[]>>;
147
+ /**
148
+ * Each participant's script, by the key the workflow connects it under. When no turn runs, no
149
+ * script is still working, and everything left waits on a participant with no `until`, the attempt
150
+ * ends: its open questions expire `attempt-ended` and, unless the workflow returned, `run.value`
151
+ * throws. While a question has an `until`, the test waits for the earliest, and `stall` does not
152
+ * count that wait. A route
153
+ * or message reaching a participant with none here fails the test.
154
+ */
155
+ participants?: Readonly<Record<string, ParticipantScript>>;
156
+ /** Each decision's answers, by its key or a pattern. */
157
+ decisions?: Readonly<Record<string, DecisionScript>>;
158
+ /** What `wf.allowance` reads, one per harness; a harness with none here reads `none`. */
159
+ allowance?: readonly HarnessAllowance[];
160
+ /** Runtime aliases beside the ones `awf run` installs, `claude` and `codex`; the same name replaces one. */
161
+ runtimes?: RuntimeAliases;
162
+ /** The run's deadline; none unless given, as `awf run` has none unless `--timeout` gives one. */
163
+ timeout?: Duration;
164
+ /** @deprecated Give `timeout`, a `Duration` such as `"5m"`. */
165
+ timeoutMs?: number;
166
+ /** Real time with no turn or decision starting or ending before the test fails; `"2s"`. */
167
+ stall?: Duration;
168
+ /** @deprecated Give `stall`, a `Duration` such as `"2s"`. */
169
+ stallMs?: number;
170
+ /**
171
+ * The run's clock is virtual: where a run would park with a timer to come, the clock moves to it
172
+ * and the timer fires, so a two-day reminder takes no time. After this many, the run parks
173
+ * instead, as a loop of reminders nobody answers would; 20. Zero parks at the first.
174
+ */
175
+ maxTimers?: number;
176
+ /** Where the agents work; by default a temporary directory, removed afterwards. */
177
+ cwd?: string;
178
+ /**
179
+ * The session the run is started from, by its harness; absent, `agents.caller` and
180
+ * `agents.forkCaller` answer `null`. It is handed over, as `awf run --here` hands one over, and its
181
+ * turns are answered from the script under the key the workflow gives it; with `here: false` it
182
+ * waits on the run, and `agents.caller` answers `null`. A fork of it runs on `model`, the model its
183
+ * files show; without one, `forkCaller` answers `null`, as it does where they name none.
184
+ */
185
+ caller?: { harness: Harness; model?: string; here?: boolean };
186
+ /**
187
+ * More turns started than this fails the test, so a loop that never ends doesn't hang it; 20.
188
+ * Nudges and wakes for a message don't count.
189
+ */
190
+ maxTurns?: number;
191
+ /** What an earlier attempt left; given, the test's attempt is its continue. */
192
+ earlier?: EarlierAttempt;
193
+ /** @deprecated Give `earlier: { stages }`. */
194
+ recorded?: Readonly<Record<string, JsonValue | undefined>>;
195
+ /** @deprecated Give `earlier: { mailbox }`. */
196
+ mailbox?: readonly MailboxRecord[];
197
+ /**
198
+ * The stage the attempt starts at, as `--from-stage`: a continue of `earlier`, or a new run
199
+ * started there. A workflow with events takes it only with `earlier`, as `awf run` takes it only
200
+ * with `--continue`.
201
+ */
202
+ fromStage?: string;
203
+ /**
204
+ * Values of stages before `fromStage` with no record to reuse, as `awf run --values` gives them;
205
+ * a workflow with events takes none.
206
+ */
207
+ values?: Readonly<Record<string, JsonValue>>;
208
+ };
209
+
210
+ /** Every option a test may give, whatever its workflow's args and events: what the run reads. */
211
+ type AnyTestOptions = CommonTestOptions & {
212
+ args?: JsonValue;
213
+ event?: { name: string; data: JsonValue } & EventIds;
214
+ events?: readonly ScriptedEvent<EventSchemas>[];
215
+ };
216
+
217
+ const OPTION_KEYS: readonly (keyof AnyTestOptions)[] = [
218
+ "args",
219
+ "id",
220
+ "files",
221
+ "agents",
222
+ "compactions",
223
+ "messages",
224
+ "participants",
225
+ "decisions",
226
+ "allowance",
227
+ "runtimes",
228
+ "caller",
229
+ "timeout",
230
+ "stall",
231
+ "maxTimers",
232
+ "maxTurns",
233
+ "cwd",
234
+ "earlier",
235
+ "fromStage",
236
+ "values",
237
+ "event",
238
+ "events",
239
+ ];
240
+ /** Taken, logged, until story 039's last task removes them; never suggested. */
241
+ const DEPRECATED_KEYS: readonly (keyof AnyTestOptions)[] = [
242
+ "timeoutMs",
243
+ "stallMs",
244
+ "recorded",
245
+ "mailbox",
246
+ ];
247
+ const EARLIER_KEYS: readonly (keyof EarlierAttempt)[] = ["stages", "mailbox"];
248
+
249
+ /**
250
+ * A test's options as `testWorkflow` was called: `(workflow, options?)`, or the deprecated
251
+ * `(workflow, args, options)`. Three arguments are always the deprecated form. With two, the second
252
+ * is the options when it is an object, and args otherwise (`null`, a string, a number, a boolean or
253
+ * an array), so a workflow's object args are never taken for args there: they are read as options,
254
+ * and refused by their first key.
255
+ */
256
+ export function testCall(
257
+ workflow: unknown,
258
+ given: readonly unknown[],
259
+ ): { options: AnyTestOptions; used: Deprecated[] } {
260
+ const deprecatedForm =
261
+ given.length >= 2 || (given.length === 1 && given[0] !== undefined && !isObject(given[0]));
262
+ if (!deprecatedForm) {
263
+ const options = plainOptions(given[0] ?? {});
264
+ // Object args given second are named by their key, whatever the workflow is.
265
+ const problem = optionsProblem(
266
+ options,
267
+ isEventWorkflow(workflow),
268
+ (workflow as { meta?: { name?: string } } | null)?.meta?.name ?? "the workflow",
269
+ );
270
+ if (problem) throw new Error(problem);
271
+ if (!isWorkflow(workflow))
272
+ throw new Error(
273
+ "testWorkflow(workflow, options?) tests a workflow defineWorkflow made: define it with defineWorkflow",
274
+ );
275
+ return { options, used: [] };
276
+ }
277
+ // The args-second form never existed for a workflow with events: refused, not deprecated.
278
+ if (isEventWorkflow(workflow))
279
+ throw new Error("a workflow with events is tested as testWorkflow(workflow, { id, event? })");
280
+ const [args, optionsGiven = {}] = given;
281
+ const options = plainOptions(optionsGiven);
282
+ if ("args" in options)
283
+ throw new Error("args are given once: testWorkflow(workflow, { args, … }) is the one form");
284
+ return { options: { ...options, args: args as JsonValue }, used: ["test-args"] };
285
+ }
286
+
287
+ function isObject(value: unknown): value is AnyTestOptions {
288
+ return typeof value === "object" && value !== null && !Array.isArray(value);
289
+ }
290
+
291
+ /** `value` as a test's options: an object literal, never a `Date`, an array or a class's instance. */
292
+ function plainOptions(value: unknown): AnyTestOptions {
293
+ const prototype = isObject(value) ? Object.getPrototypeOf(value) : undefined;
294
+ if (prototype === Object.prototype || prototype === null) return value as AnyTestOptions;
295
+ const kind =
296
+ value === null
297
+ ? "null"
298
+ : Array.isArray(value)
299
+ ? "an array"
300
+ : typeof value === "object"
301
+ ? `a ${(prototype as { constructor?: { name?: string } }).constructor?.name || "object"}`
302
+ : `a ${typeof value}`;
303
+ throw new Error(
304
+ `testWorkflow's options are an object literal, not ${kind}: testWorkflow(workflow, { args, … })`,
305
+ );
306
+ }
307
+
308
+ function isWorkflow(workflow: unknown): workflow is Workflow {
309
+ return (
310
+ typeof workflow === "object" &&
311
+ workflow !== null &&
312
+ (workflow as { kind?: unknown }).kind === WORKFLOW_KIND
313
+ );
314
+ }
315
+
316
+ /** Why a test's options aren't ones `testWorkflow` takes; undefined when they are. */
317
+ function optionsProblem(
318
+ options: AnyTestOptions,
319
+ withEvents: boolean,
320
+ name: string,
321
+ ): string | undefined {
322
+ for (const key of Object.keys(options) as (keyof AnyTestOptions)[]) {
323
+ if (!OPTION_KEYS.includes(key) && !DEPRECATED_KEYS.includes(key))
324
+ return `testWorkflow has no option ${JSON.stringify(key)}: its options are ${OPTION_KEYS.join(", ")}${
325
+ "args" in options ? "" : "; a workflow's args are given as { args }"
326
+ }`;
327
+ if (!withEvents && (key === "event" || key === "events"))
328
+ return `${key} is an option for a workflow with events, and ${name} declares none`;
329
+ }
330
+ const { earlier } = options;
331
+ if (earlier === undefined) return undefined;
332
+ if (!isObject(earlier)) return "earlier is an object: { stages?, mailbox? }";
333
+ const unknown = Object.keys(earlier).find(
334
+ (key) => !EARLIER_KEYS.includes(key as keyof EarlierAttempt),
335
+ );
336
+ return unknown === undefined
337
+ ? undefined
338
+ : `earlier has no ${JSON.stringify(unknown)}: it gives ${EARLIER_KEYS.join(" and ")}`;
339
+ }
340
+
341
+ /** What a test of the engine, not of a workflow, may give `runTest`. */
342
+ export type EngineTestOptions = {
343
+ /**
344
+ * Where an agent may `--await-reply`: `listed`, where the engine's list allows it for the
345
+ * agent's harness and placement, as a run would; `anywhere`, on any harness, the fake's included.
346
+ */
347
+ awaitReply?: "listed" | "anywhere";
348
+ /** Deprecated forms the call itself used, logged with the run's own. */
349
+ used?: readonly Deprecated[];
350
+ };
351
+
352
+ /**
353
+ * A test's length of time in milliseconds: `field`, a `Duration`, or `old`, its deprecated form in
354
+ * milliseconds, which is reported; both at once are refused.
355
+ */
356
+ function milliseconds(
357
+ options: AnyTestOptions,
358
+ field: "timeout" | "stall",
359
+ old: "timeoutMs" | "stallMs",
360
+ deprecated: () => void,
361
+ ): number | undefined {
362
+ const given = options[field];
363
+ const ms = options[old];
364
+ if (given != null) {
365
+ if (ms != null) throw new Error(`give ${field} alone: ${old} is its deprecated form`);
366
+ return positiveDurationMilliseconds(given, `a test's ${field}`);
367
+ }
368
+ if (ms == null) return undefined;
369
+ if (!Number.isSafeInteger(ms) || ms <= 0)
370
+ throw new Error(`a test's ${old} is a positive whole number of milliseconds: not ${ms}`);
371
+ deprecated();
372
+ return ms;
373
+ }
374
+
375
+ /** A file's contents, or a directory of them by name. */
376
+ export type FileTree = string | { readonly [name: string]: FileTree };
377
+
378
+ /** A `keepAlive` that gives no `window`, in a test: the hour it would be stalls it. */
379
+ const TEST_WINDOW_MS = 100;
380
+
381
+ const WAITED_OUT = "the test ended the attempt: everything left waited on a participant";
382
+
383
+ /**
384
+ * An event a test sends while the run goes on, at an agent's `n`th turn, `"{agent}#{n}"`: `during`
385
+ * it, or `after` it has ended, once the run waits live in an agent's `keepAlive` window.
386
+ */
387
+ export type ScriptedEvent<Events extends EventSchemas> = (
388
+ | { during: string; after?: undefined }
389
+ | { after: string; during?: undefined }
390
+ ) &
391
+ EventIds &
392
+ (
393
+ | {
394
+ [Name in Exclude<keyof Events & string, `${string}:ended`>]: {
395
+ name: Name;
396
+ data: OutputOf<Events[Name]>;
397
+ };
398
+ }[Exclude<keyof Events & string, `${string}:ended`>]
399
+ /**
400
+ * A reply on a participant routed to the code with `to: "code"`; awf fills its author and id
401
+ * when absent.
402
+ *
403
+ * @deprecated Declare `{key}:reply: ParticipantReply` and script `{ name: "{key}:reply", data }`.
404
+ */
405
+ | { name: "participant:reply"; from: string; data: ParticipantReply }
406
+ );
407
+
408
+ /** An event's id in the run's mailbox, as `awf emit --event-id` gives it. */
409
+ type EventIds = {
410
+ eventId?: string;
411
+ /** @deprecated Give it as `eventId`. */
412
+ id?: string;
413
+ };
414
+
415
+ /** @deprecated Use `TestOptions<Args, Events>`, the one options type. */
416
+ export type EventTestOptions<
417
+ Events extends EventSchemas,
418
+ Args extends JsonValue = JsonValue,
419
+ > = TestOptions<Args, Events>;
420
+
421
+ /** An event the run sent, to the run whose ref is `ref`, which a test doesn't start. */
422
+ export type EmittedEvent = { ref: string; name: string; data: JsonValue; eventId?: string };
423
+
424
+ /** What the run did. Under `parallel`, what started first is scheduling: read by key. */
425
+ export type TestRun<Result> = {
426
+ /**
427
+ * What the workflow returned. When it threw instead, reading this throws, with the workflow's
428
+ * error as the cause: `expect(() => run.value).toThrow(…)`. So does spreading or serialising the
429
+ * run.
430
+ */
431
+ readonly value: Result;
432
+ /** Every turn, nudges included, in the order they started. */
433
+ turns: TestTurn[];
434
+ /** One agent's turns, in its session's order. */
435
+ turnsOf(agent: string): TestTurn[];
436
+ /** Every compaction, in the order they started; not among the turns. */
437
+ compactions: CompactionRecord[];
438
+ /** One agent's compactions, in order. */
439
+ compactionsOf(agent: string): CompactionRecord[];
440
+ /** Every switch of an agent's model or effort its host was asked for, in order; not among the turns. */
441
+ sets: SetRecord[];
442
+ /** One agent's switches, in order. */
443
+ setsOf(agent: string): SetRecord[];
444
+ /** The messages admitted for one agent, in the order sent: who sent each, whether it asked, and how it ended. */
445
+ messagesOf(recipient: string): MessageRecord[];
446
+ /** Every agent opened, in the order opened. */
447
+ agents: OpenedAgent[];
448
+ /** The agent opened under `key`; throws, naming the keys opened, when there is none. */
449
+ agentOf(key: string): OpenedAgent;
450
+ decisions: DecisionRequest[];
451
+ logs: { message: string; fields?: JsonObject }[];
452
+ /**
453
+ * Each stage's record in the order entered, as the run wrote it to disk: sessions, times and
454
+ * reason included, which the run's own summaries leave out.
455
+ */
456
+ stages: StageRecord[];
457
+ /**
458
+ * How the attempt ended, as the workflow's `present` and `report` are given it: `completed` with
459
+ * its value, `parked` at its last wait, or `stopped`, `failed`, `timed-out` or `cancelled` with
460
+ * the stage it ended in and why.
461
+ */
462
+ ending: Ending<Result>;
463
+ /**
464
+ * @deprecated Read `run.ending`, whose kind is `"stopped"`. How the attempt stopped, and the
465
+ * values it stopped for; absent when it didn't.
466
+ */
467
+ readonly stopped?: { reason: string; stage?: string; needs?: readonly StageNeed[] };
468
+ /** Every event the run sent to another workflow, in order. */
469
+ emits: EmittedEvent[];
470
+ /**
471
+ * @deprecated Read `run.ending`, whose kind is `"parked"`. A run that parked, and its last wait;
472
+ * absent when it returned.
473
+ */
474
+ readonly parked?: { label?: string; until?: string };
475
+ /** The run's timers in the order set: when each was due, and whether it fired or was cancelled. */
476
+ timers: { name: string; due: string; state: "fired" | "cancelled" | "pending" }[];
477
+ /**
478
+ * Its mailbox as the run left it: its events, timers, and what its waits and turns wrote, in
479
+ * order; a plain run's holds its timers alone.
480
+ */
481
+ mailbox: MailboxRecord[];
482
+ };
483
+
484
+ /**
485
+ * `testWorkflow`, and what only the engine's own tests give it: a workflow may be its `meta` and
486
+ * `run` alone, which the engine's tests build and a workflow's never does.
487
+ */
488
+ export async function runTest<Result extends JsonValue>(
489
+ // biome-ignore lint/suspicious/noExplicitAny: args are checked by the call that typed them.
490
+ workflow: RunnableWorkflow<any, Result> | Workflow<any, Result, EventSchemas>,
491
+ options: AnyTestOptions = {},
492
+ engine: EngineTestOptions = {},
493
+ ): Promise<TestRun<Result>> {
494
+ const withEvents = isEventWorkflow(workflow) ? (workflow as EventWorkflow) : undefined;
495
+ const problem = optionsProblem(options, withEvents !== undefined, workflow.meta.name);
496
+ if (problem) throw new Error(problem);
497
+ const logs: TestRun<Result>["logs"] = [];
498
+ // A test's own forms, apart from the run's: said in its logs as the run says a warning.
499
+ const deprecations = new RunDeprecations((message) => logs.push({ message }));
500
+ const deprecated = (name: Deprecated) => deprecations.used(name);
501
+ for (const name of engine.used ?? []) deprecated(name);
502
+ const earlier = earlierOf(options, deprecated);
503
+ if (withEvents && options.values !== undefined)
504
+ throw new Error(
505
+ "a run of a workflow with events takes no values: before fromStage, a stage with no record runs",
506
+ );
507
+ if (withEvents && options.fromStage !== undefined && earlier.continues === false)
508
+ throw new Error(
509
+ "a run of a workflow with events takes fromStage only with earlier, as awf run takes --from-stage only with --continue",
510
+ );
511
+ const started = startOf(withEvents, options, deprecated);
512
+ const definition = workflow as RunnableWorkflow<JsonValue, Result>;
513
+ const scripts = new Scripts(options.agents);
514
+ const compactionScripts = new Scripts(options.compactions, {
515
+ noun: "compaction",
516
+ option: "compactions",
517
+ });
518
+ const messageScripts = new MessageScripts(options.messages);
519
+ if (options.values !== undefined && options.fromStage === undefined) {
520
+ throw new Error("values give the stages before fromStage: give fromStage too");
521
+ }
522
+ const stopping = new AbortController();
523
+ const problems: string[] = [];
524
+ const stallMs =
525
+ milliseconds(options, "stall", "stallMs", () => deprecations.used("stall-ms")) ?? 2_000;
526
+ const timeoutMs = milliseconds(options, "timeout", "timeoutMs", () =>
527
+ deprecations.used("timeout-ms", "in testWorkflow's options"),
528
+ );
529
+ let stall: ReturnType<typeof setTimeout> | undefined;
530
+ /** The test ended the attempt: everything left waited on a coordinator's script. */
531
+ let waitedOut = false;
532
+ const arm = (ms: number) => {
533
+ clearTimeout(stall);
534
+ stall = setTimeout(stalled, ms);
535
+ };
536
+ const stalled = () => {
537
+ const open = coordinators.open();
538
+ if (host.inFlight().length === 0 && coordinators.idle && open.length > 0) {
539
+ const closes = open.flatMap((each) => (each.closesAt === undefined ? [] : [each.closesAt]));
540
+ // A question that closes on its own is waited for; that wait is not a stall.
541
+ if (closes.length > 0) return arm(Math.max(0, Math.min(...closes) - Date.now()) + stallMs);
542
+ waitedOut = true;
543
+ stopping.abort(new Error("everything left waits on a participant"));
544
+ return;
545
+ }
546
+ if (host.inFlight().length === 0 && liveUntil !== undefined && liveUntil > now())
547
+ return stopWith(
548
+ `the run stalled: it waits live in an agent's keepAlive window for ${liveUntil - now()} ms more; a test gives the agent a short window, or none`,
549
+ );
550
+ const waiting = host
551
+ .inFlight()
552
+ .map((turn) => `"${turn.agent}" turn ${turn.n}${turn.nudge ? " (nudge)" : ""}`);
553
+ stopWith(
554
+ `the run stalled: nothing started or ended for ${stallMs} ms; ${
555
+ waiting.length > 0 ? `in flight: ${waiting.join(", ")}` : "no turn in flight"
556
+ }`,
557
+ );
558
+ };
559
+ /** How far the virtual clock is ahead of the real one. */
560
+ let ahead = 0;
561
+ const now = () => Date.now() + ahead;
562
+ const maxTimers = options.maxTimers ?? 20;
563
+ if (!Number.isInteger(maxTimers) || maxTimers < 0)
564
+ throw new Error(`maxTimers is a whole number of timers, 0 or more, not ${options.maxTimers}`);
565
+ let skipped = 0;
566
+ /** The clock stopped at maxTimers, and the run parked there. */
567
+ let stoppedSkipping = false;
568
+ const skip = async (until: number) => {
569
+ if (skipped >= maxTimers) {
570
+ stoppedSkipping = true;
571
+ return false;
572
+ }
573
+ skipped += 1;
574
+ ahead += Math.max(0, until - now());
575
+ liveUntil = undefined;
576
+ events.onActivity();
577
+ return true;
578
+ };
579
+ const scripted =
580
+ withEvents && new ScriptedEvents(withEvents, options, earlier.mailbox ?? [], now, deprecated);
581
+ const maxTurns = options.maxTurns ?? 20;
582
+ if (!Number.isInteger(maxTurns) || maxTurns < 1)
583
+ throw new Error(`maxTurns is a whole number of turns, at least 1, not ${options.maxTurns}`);
584
+ let turnsStarted = 0;
585
+ /** The end of the window the run last waited live to. */
586
+ let liveUntil: number | undefined;
587
+ let nudge: (() => void) | undefined;
588
+ const events = {
589
+ onActivity: () => arm(stallMs),
590
+ onScriptError: (message: string) => stopWith(message),
591
+ onDeprecated: (name: Deprecated, subject?: string) => deprecations.used(name, subject),
592
+ onTurnStarted: async (turn: TestTurn) => {
593
+ turnsStarted += 1;
594
+ if (turnsStarted > maxTurns)
595
+ return stopWith(
596
+ `the run started more than ${maxTurns} turns (maxTurns): a loop that never ends, or a run that needs a larger maxTurns?`,
597
+ );
598
+ const due = scripted ? scripted.due(turn) : [];
599
+ if (due.length === 0) return;
600
+ for (const record of due) await appendMailbox(run.dir, record);
601
+ nudge?.();
602
+ },
603
+ };
604
+ const attempt = earlier.continues ? 2 : 1;
605
+ const coordinators = new ParticipantScripts(options.participants, attempt, events);
606
+ const stopWith = (message: string) => {
607
+ problems.push(message);
608
+ stopping.abort(new Error(message));
609
+ };
610
+ const temporary: string[] = [];
611
+ const directory = () => {
612
+ const path = mkdtempSync(join(tmpdir(), "awf-test-"));
613
+ temporary.push(path);
614
+ return path;
615
+ };
616
+ // Laid out under a directory of the test's own, which it removes: never into one it was given.
617
+ if (options.files && options.cwd !== undefined)
618
+ throw new Error("files are laid out in the test's own directory: give files or cwd, not both");
619
+ const cwd = options.cwd ?? directory();
620
+ let args: JsonValue;
621
+ try {
622
+ args = argsOf(workflow, options, cwd);
623
+ } catch (error) {
624
+ for (const path of temporary) rmSync(path, { recursive: true, force: true });
625
+ throw error;
626
+ }
627
+ const host = createScriptedHost(
628
+ scripts,
629
+ compactionScripts,
630
+ messageScripts,
631
+ events,
632
+ options.caller ? { ...options.caller, cwd } : undefined,
633
+ );
634
+ const decisions = createScriptedDecisions(options.decisions ?? {}, events);
635
+ // Sandboxes are opened by a provider that confines nothing and hosts panes.
636
+ const sandboxes = createFakeSandboxProvider({ panes: { prelude: "true", ready: "ready" } });
637
+
638
+ let settled: { value: Result } | { error: unknown };
639
+ /** What the run came to, as `awf run` decides its ending from. */
640
+ let finished: Finished;
641
+ /** What the workflow returned, before the run settled: a test that waits out a question keeps it. */
642
+ let returned: { value: Result } | undefined;
643
+ const watched: RunnableWorkflow<JsonValue, Result> = {
644
+ ...definition,
645
+ run: async (context, given) => {
646
+ const value = await definition.run(context, given);
647
+ if (!waitedOut) returned = { value };
648
+ return value;
649
+ },
650
+ };
651
+ let sandboxOf = new Map<string, AgentSandbox>();
652
+ const emits: EmittedEvent[] = [];
653
+ let mailbox: MailboxRecord[] = [];
654
+ let stages: StageRecord[] = [];
655
+ let messages: (MessageRecord & { to: string })[] = [];
656
+ const runRoot = directory();
657
+ const deadline =
658
+ timeoutMs === undefined ? undefined : { unixMilliseconds: Date.now() + timeoutMs };
659
+ const run = await createRun(runRoot, {
660
+ id: started.id,
661
+ workflow: definition.meta.name,
662
+ argv: [],
663
+ cwd,
664
+ sandbox: null,
665
+ }).catch((error: unknown) => {
666
+ for (const path of temporary) rmSync(path, { recursive: true, force: true });
667
+ throw error;
668
+ });
669
+ if (earlier.stages) await recordEarlierAttempt(run.dir, earlier.stages);
670
+ for (const record of earlier.mailbox ?? []) await appendMailbox(run.dir, record);
671
+ if (started.record) await appendMailbox(run.dir, started.record);
672
+ const recordedStages = async ({ stages: entered = [] }: SettledRun) => {
673
+ const records = await readStageRecords(run.dir);
674
+ // A stage the plan stopped has no record of this attempt's: the one there is an earlier one's.
675
+ return entered.flatMap(({ stage, attempt }) => {
676
+ const record = records.get(stage);
677
+ return record?.attempt === attempt ? [record] : [];
678
+ });
679
+ };
680
+ events.onActivity();
681
+ try {
682
+ // A path it can't lay out is the test's mistake, never the workflow's.
683
+ await layOut(options.files, cwd).catch((error: unknown) => {
684
+ stopWith(messageOf(error));
685
+ throw error;
686
+ });
687
+ const result = await runWorkflow(watched, args, {
688
+ runRoot,
689
+ livenessPolicy: { quietMs: 1, responseMs: 25, deliveryMs: 100, releaseMs: 1000 },
690
+ // A workflow's own default hour would stall its test; one it gives is kept.
691
+ defaultWindowMs: TEST_WINDOW_MS,
692
+ now,
693
+ ...(engine.awaitReply === "anywhere" ? { awaitReply: () => true } : {}),
694
+ run: {
695
+ dir: run.dir,
696
+ id: run.record.id,
697
+ attempt,
698
+ ...(options.fromStage === undefined ? {} : { fromStage: options.fromStage }),
699
+ ...(options.values === undefined
700
+ ? {}
701
+ : { values: new Map(Object.entries(options.values)) }),
702
+ ...(scripted
703
+ ? {
704
+ events: true as const,
705
+ park: async () => undefined,
706
+ skip,
707
+ live: async (until: number) => {
708
+ liveUntil = until;
709
+ const due = scripted.ended(host.inFlight());
710
+ for (const record of due) await appendMailbox(run.dir, record);
711
+ if (due.length > 0) nudge?.();
712
+ },
713
+ listenNudges: (given: () => void) => {
714
+ nudge = given;
715
+ },
716
+ }
717
+ : // A plain run waits for a timer in its process; a test moves its clock, or parks it.
718
+ { park: async () => undefined, skip }),
719
+ },
720
+ cwd,
721
+ ...(deadline === undefined ? {} : { deadline }),
722
+ signal: stopping.signal,
723
+ // A test starts no other workflow's run: what it sends is recorded.
724
+ emit: async (target, event) => {
725
+ emits.push({
726
+ ref: `${target.meta.name}/${event.run}`,
727
+ name: event.name,
728
+ data: event.data,
729
+ ...(event.id === undefined ? {} : { eventId: event.id }),
730
+ });
731
+ return "accepted";
732
+ },
733
+ runtime: { aliases: { ...OPERATOR_ALIASES, ...options.runtimes }, host: host.factory },
734
+ sandboxes: {
735
+ providers: {
736
+ installed: { srt: sandboxes.provider, docker: sandboxes.provider },
737
+ },
738
+ sandboxesDir: directory(),
739
+ machineRoot: directory(),
740
+ // A sandbox here launches nothing, so no harness's login is needed: a stand-in for each.
741
+ environment: {
742
+ ...process.env,
743
+ ...Object.fromEntries(sandboxTokens().map((name) => [name, "workflow-test"])),
744
+ },
745
+ },
746
+ decisions: {
747
+ providers: { scripted: decisions.provider },
748
+ aliases: { jev: { provider: "scripted", model: "jev" } },
749
+ },
750
+ allowance: async (harness) => {
751
+ return (
752
+ options.allowance?.find((given) => given.harness === harness) ?? {
753
+ harness,
754
+ read: "none",
755
+ reason: "the test gave no allowance for it",
756
+ }
757
+ );
758
+ },
759
+ onLog: (message, fields) => logs.push({ message, ...(fields ? { fields } : {}) }),
760
+ coordinators: [coordinators],
761
+ });
762
+ settled = { value: result.value };
763
+ finished = { result };
764
+ sandboxOf = sandboxesOf(result.sandboxes);
765
+ stages = await recordedStages(result);
766
+ } catch (caught) {
767
+ finished = { error: caught };
768
+ settled = { error: caught instanceof WorkflowRunError ? caught.cause : caught };
769
+ if (caught instanceof WorkflowRunError) {
770
+ sandboxOf = sandboxesOf(caught.sandboxes);
771
+ stages = await recordedStages(caught);
772
+ }
773
+ } finally {
774
+ clearTimeout(stall);
775
+ if (waitedOut) {
776
+ settled = returned ?? { error: new Error(WAITED_OUT) };
777
+ }
778
+ host.settle(await readTurns(run.dir));
779
+ messages = await messagesIn(run.dir);
780
+ mailbox = await readMailbox(run.dir);
781
+ for (const path of temporary) rmSync(path, { recursive: true, force: true });
782
+ }
783
+ if (problems.length === 0 && scripted) problems.push(...scripted.unsent(host.inFlight()));
784
+ if (problems.length === 0) {
785
+ const leftovers = [
786
+ ...scripts.leftovers(),
787
+ ...compactionScripts.leftovers(),
788
+ ...messageScripts.leftovers(),
789
+ ];
790
+ // What the workflow threw may be why a list went unfinished.
791
+ if (leftovers.length > 0 && "error" in settled) {
792
+ leftovers.push(`the workflow threw: ${messageOf(settled.error)}`);
793
+ }
794
+ problems.push(...leftovers);
795
+ }
796
+ if (problems.length > 0) throw new Error(problems.join("\n"));
797
+ const agents = host.agents.map((agent) => {
798
+ const sandbox = sandboxOf.get(agent.key);
799
+ return sandbox === undefined ? agent : { ...agent, sandbox };
800
+ });
801
+ const goOn = (stop: WorkflowStopped | undefined, entered: readonly { stage: string }[]) =>
802
+ continueCommand(
803
+ {
804
+ shellCwd: cwd,
805
+ cwd,
806
+ cwdGiven: false,
807
+ runRoot,
808
+ runRootGiven: false,
809
+ workflowFile: withEvents?.file ?? definition.meta.name,
810
+ ...(options.fromStage === undefined ? {} : { fromStage: options.fromStage }),
811
+ },
812
+ run.record.id,
813
+ stop,
814
+ entered,
815
+ );
816
+ const decided = decideEnding(finished, deadline, goOn).ending;
817
+ // What waited on a participant the test cancelled, not an operator; after a return, nothing.
818
+ const ending: Ending<Result> =
819
+ "value" in settled
820
+ ? { kind: "completed", value: settled.value, stages: decided.stages }
821
+ : waitedOut || decided.kind === "completed"
822
+ ? {
823
+ kind: "cancelled",
824
+ ...("stage" in decided && decided.stage !== undefined ? { stage: decided.stage } : {}),
825
+ reason: WAITED_OUT,
826
+ stages: decided.stages,
827
+ continue: goOn(undefined, decided.stages),
828
+ }
829
+ : decided;
830
+ const stopped = stoppedOf(settled);
831
+ const parked = parkedOf(settled);
832
+ const result: TestRun<Result> = {
833
+ get value() {
834
+ if ("value" in settled) return settled.value;
835
+ if (primaryFailure(settled.error) instanceof RunParked)
836
+ throw new Error(
837
+ stoppedSkipping && maxTimers > 0
838
+ ? `the run parked after the clock skipped to ${maxTimers} timers (maxTimers): a loop of timers that nothing answers?`
839
+ : "the run parked, and returned nothing: run.ending says at which wait",
840
+ );
841
+ throw new Error(`the workflow threw: ${messageOf(settled.error)}`, { cause: settled.error });
842
+ },
843
+ turns: host.turns,
844
+ emits,
845
+ mailbox,
846
+ turnsOf: (agent) => host.turns.filter((turn) => turn.agent === agent),
847
+ compactions: host.compactions,
848
+ compactionsOf: (agent) => host.compactions.filter((compaction) => compaction.agent === agent),
849
+ sets: host.sets,
850
+ setsOf: (agent) => host.sets.filter((set) => set.agent === agent),
851
+ messagesOf: (recipient) =>
852
+ messages.flatMap(({ to, ...each }) => (to === recipient ? [each] : [])),
853
+ agents,
854
+ agentOf(key) {
855
+ const agent = agents.find((opened) => opened.key === key);
856
+ if (agent) return agent;
857
+ const opened = agents.map((known) => `"${known.key}"`).join(", ") || "none";
858
+ throw new Error(`no agent "${key}" was opened; opened: ${opened}`);
859
+ },
860
+ decisions: decisions.asked,
861
+ logs,
862
+ stages,
863
+ timers: timersOf(mailbox),
864
+ ending,
865
+ };
866
+ // Not enumerable, so spreading the run reads neither.
867
+ Object.defineProperties(result, {
868
+ stopped: {
869
+ get: () => {
870
+ deprecated("run-stopped");
871
+ return stopped;
872
+ },
873
+ },
874
+ parked: {
875
+ get: () => {
876
+ deprecated("run-parked");
877
+ return parked;
878
+ },
879
+ },
880
+ });
881
+ return result;
882
+ }
883
+
884
+ function timersOf(records: readonly MailboxRecord[]): TestRun<unknown>["timers"] {
885
+ return [...foldMailbox(records).timers.values()].map(({ name, due, fired, cancelled }) => ({
886
+ name,
887
+ due,
888
+ state: fired ? "fired" : cancelled ? "cancelled" : "pending",
889
+ }));
890
+ }
891
+
892
+ /**
893
+ * A message as `messagesOf` gives it. A question ends `replied` or `failed`, `reason` saying why it
894
+ * failed (`timed-out`, `withdrawn`, `released`, `cancelled`, `attempt-ended`, or how its peer
895
+ * failed); any other message ends `delivered` once shown, or `undelivered` with the reason.
896
+ * `pending` until then. A message to a coordinator is shown once recorded.
897
+ */
898
+ export type MessageRecord = {
899
+ from: string;
900
+ awaitReply: boolean;
901
+ end: "replied" | "failed" | "delivered" | "undelivered" | "pending";
902
+ reason?: string;
903
+ };
904
+
905
+ /** The attempt's messages, read from the run's event log. */
906
+ async function messagesIn(dir: string): Promise<(MessageRecord & { to: string })[]> {
907
+ const found = new Map<number, MessageRecord & { to: string }>();
908
+ const outside = new Set<string>();
909
+ for (const event of await readEvents(dir)) {
910
+ if (event.kind === "participant-connected") outside.add(event.participant);
911
+ if (event.kind === "message-sent") {
912
+ const awaitReply = event.awaitsReply !== undefined;
913
+ found.set(event.message, {
914
+ from: event.from,
915
+ to: event.to,
916
+ awaitReply,
917
+ // A coordinator reads the run log: what it is sent is shown once recorded.
918
+ end: outside.has(event.to) && !awaitReply ? "delivered" : "pending",
919
+ });
920
+ continue;
921
+ }
922
+ const message =
923
+ "message" in event && typeof event.message === "number"
924
+ ? found.get(event.message)
925
+ : undefined;
926
+ if (!message) continue;
927
+ if (event.kind === "request-ended") {
928
+ message.end = event.end === "replied" ? "replied" : "failed";
929
+ const reason =
930
+ event.end === "failed" ? event.reason : event.end === "replied" ? undefined : event.end;
931
+ if (reason !== undefined) message.reason = reason;
932
+ } else if (!message.awaitReply && event.kind === "message-delivery") {
933
+ message.end = "delivered";
934
+ } else if (!message.awaitReply && event.kind === "message-undelivered") {
935
+ message.end = "undelivered";
936
+ message.reason = event.reason;
937
+ }
938
+ }
939
+ return [...found.values()];
940
+ }
941
+
942
+ /**
943
+ * Writes `recorded` as attempt 1's succeeded stages into a run's folder, a second apart in the order
944
+ * given: a continue orders what is recorded by when each stage started.
945
+ */
946
+ async function recordEarlierAttempt(
947
+ dir: string,
948
+ recorded: Readonly<Record<string, JsonValue | undefined>>,
949
+ ): Promise<void> {
950
+ let at = Date.now() - Object.keys(recorded).length * 1_000;
951
+ for (const [stage, value] of Object.entries(recorded)) {
952
+ const time = new Date(at).toISOString();
953
+ at += 1_000;
954
+ await writeStageRecord(dir, {
955
+ version: STAGE_RECORD_VERSION,
956
+ stage,
957
+ attempt: 1,
958
+ outcome: "succeeded",
959
+ started: time,
960
+ ended: time,
961
+ sessions: [],
962
+ ...(value === undefined ? {} : { value }),
963
+ });
964
+ }
965
+ }
966
+
967
+ /** An event a test sends while its run goes on, once its turn begins. */
968
+ type PendingEvent = {
969
+ at: string;
970
+ after: boolean;
971
+ name: string;
972
+ record: MailboxRecord;
973
+ /** Its turn, once begun: an `after` event waits for it to end. */
974
+ turn?: TestTurn;
975
+ };
976
+
977
+ /** The events a test sends while its run goes on, each once its turn begins. */
978
+ class ScriptedEvents {
979
+ readonly #pending: PendingEvent[];
980
+
981
+ constructor(
982
+ workflow: EventWorkflow,
983
+ options: AnyTestOptions,
984
+ /** What the earlier attempt left in the mailbox, whose event ids are taken. */
985
+ earlier: readonly MailboxRecord[],
986
+ /** The run's clock, which stamps what is sent. */
987
+ private readonly now: () => number,
988
+ deprecated: (name: Deprecated) => void,
989
+ ) {
990
+ const ids = new Set<string>();
991
+ const idOf = (given: string | undefined, made: string) => {
992
+ const id = given ?? made;
993
+ const problem = given === undefined ? undefined : eventIdProblem(given);
994
+ if (problem) throw new Error(`the eventId ${JSON.stringify(given)}: ${problem}`);
995
+ // A repeated id would be appended and then dropped: the test says so instead.
996
+ if (ids.has(id))
997
+ throw new Error(`the eventId ${id} is given twice; an eventId names one event`);
998
+ ids.add(id);
999
+ return id;
1000
+ };
1001
+ for (const record of earlier) if (record.kind === "event") ids.add(record.id);
1002
+ if (options.event) idOf(eventIdOf(options.event, deprecated), `${AWF_ID_PREFIX}start`);
1003
+ this.#pending = (options.events ?? []).map((event, index) => {
1004
+ if ((event.during === undefined) === (event.after === undefined))
1005
+ throw new Error("a scripted event gives during or after, one of them");
1006
+ const after = event.after !== undefined;
1007
+ const at = event.after ?? event.during;
1008
+ if (typeof at !== "string" || !/^[^#]+#[1-9]\d*$/.test(at))
1009
+ throw new Error(
1010
+ `a scripted event's ${after ? "after" : "during"} is "{agent}#{n}", an agent's nth turn, not ${at}`,
1011
+ );
1012
+ const id = idOf(eventIdOf(event, deprecated), `${AWF_ID_PREFIX}test.${index + 1}`);
1013
+ // A participant's reply is recorded as `awf emit` records one, whatever name it is sent by.
1014
+ const stamped = new Date().toISOString();
1015
+ let sent = {
1016
+ id,
1017
+ name: event.name,
1018
+ data: event.data as JsonValue,
1019
+ from: "operator",
1020
+ at: stamped,
1021
+ };
1022
+ const reply = awfEventOf(event.name);
1023
+ if (event.name === "participant:reply") {
1024
+ deprecated("test-participant-reply");
1025
+ const old = event as { from: string; data: ParticipantReply };
1026
+ sent = replyRecord(old.from, id, { author: "engineer", ...old.data }, stamped);
1027
+ } else if (reply?.kind === "reply") {
1028
+ checkEvent(workflow, event, true);
1029
+ sent = replyRecord(reply.key, id, event.data, stamped);
1030
+ } else checkEvent(workflow, event);
1031
+ return {
1032
+ at,
1033
+ after,
1034
+ name: event.name,
1035
+ record: { version: MAILBOX_RECORD_VERSION, kind: "event", ...sent },
1036
+ };
1037
+ });
1038
+ }
1039
+
1040
+ /** The events due during `turn`, taken off the list, stamped now; those after it wait for it. */
1041
+ due(turn: TestTurn): MailboxRecord[] {
1042
+ const at = `${turn.agent}#${turn.n}`;
1043
+ for (const each of this.#pending) if (each.after && each.at === at) each.turn = turn;
1044
+ return this.#take((each) => !each.after && each.at === at);
1045
+ }
1046
+
1047
+ /** The run waits live: the events after a turn that has ended, taken off the list. */
1048
+ ended(inFlight: readonly TestTurn[]): MailboxRecord[] {
1049
+ return this.#take((each) => each.turn !== undefined && !inFlight.includes(each.turn));
1050
+ }
1051
+
1052
+ #take(which: (each: PendingEvent) => boolean): MailboxRecord[] {
1053
+ const due = this.#pending.filter(which);
1054
+ for (const each of due) this.#pending.splice(this.#pending.indexOf(each), 1);
1055
+ return due.map((each) => ({ ...each.record, at: new Date(this.now()).toISOString() }));
1056
+ }
1057
+
1058
+ /** What was never sent: the test fails. */
1059
+ unsent(inFlight: readonly TestTurn[]): string[] {
1060
+ return this.#pending.map(
1061
+ (each) =>
1062
+ `the scripted ${each.name} ${each.after ? "after" : "during"} ${each.at} was never sent: ${
1063
+ each.turn !== undefined && !inFlight.includes(each.turn)
1064
+ ? "the run never waited live after that turn, in an agent's keepAlive window"
1065
+ : "that turn never ran"
1066
+ }`,
1067
+ );
1068
+ }
1069
+ }
1070
+
1071
+ /** Lays out `files` before the run, each path under `cwd`. */
1072
+ async function layOut(
1073
+ files: Readonly<Record<string, FileTree>> | undefined,
1074
+ cwd: string,
1075
+ ): Promise<void> {
1076
+ const write = async (path: string, tree: FileTree): Promise<void> => {
1077
+ if (typeof tree === "string") {
1078
+ await mkdir(dirname(path), { recursive: true });
1079
+ await Bun.write(path, tree);
1080
+ return;
1081
+ }
1082
+ await mkdir(path, { recursive: true });
1083
+ for (const [name, inner] of Object.entries(tree)) await write(join(path, name), inner);
1084
+ };
1085
+ for (const [path, tree] of Object.entries(files ?? {})) {
1086
+ // Under the test's own directory, which it removes: a path outside would be left behind.
1087
+ const target = resolve(cwd, path);
1088
+ if (isAbsolute(path) || relative(cwd, target).startsWith(".."))
1089
+ throw new Error(`files lays out under the run's directory; ${path} is outside it`);
1090
+ await write(target, tree);
1091
+ }
1092
+ }
1093
+
1094
+ /**
1095
+ * The run a test starts: its id, and the event it starts with, as `awf emit` would append it, or
1096
+ * none, as `awf run --id` starts one.
1097
+ */
1098
+ function startOf(
1099
+ workflow: EventWorkflow | undefined,
1100
+ options: AnyTestOptions,
1101
+ deprecated: (name: Deprecated) => void,
1102
+ ): { id: string; record?: MailboxRecord } {
1103
+ const { event, id = "test" } = options;
1104
+ if (typeof id !== "string") throw new Error(`a run's id is a string, not ${JSON.stringify(id)}`);
1105
+ const problem = idProblem(id);
1106
+ if (problem) throw new Error(problem);
1107
+ if (workflow === undefined || event === undefined) return { id };
1108
+ checkEvent(workflow, { name: event.name, data: event.data });
1109
+ const { starts } = workflow;
1110
+ if (starts && !starts.includes(event.name))
1111
+ throw new Error(
1112
+ `${event.name} doesn't start a run of ${workflow.meta.name}: its starts are ${starts.join(", ")}`,
1113
+ );
1114
+ const record = {
1115
+ version: MAILBOX_RECORD_VERSION,
1116
+ at: new Date().toISOString(),
1117
+ kind: "event" as const,
1118
+ id: eventIdOf(event, deprecated) ?? `${AWF_ID_PREFIX}start`,
1119
+ name: event.name,
1120
+ data: event.data,
1121
+ from: "operator",
1122
+ starts: true as const,
1123
+ };
1124
+ return { id, record };
1125
+ }
1126
+
1127
+ /**
1128
+ * What the earlier attempt left, from `earlier` or its deprecated forms, `recorded` and `mailbox`;
1129
+ * `continues` when it left anything, so the test's attempt is attempt 2.
1130
+ */
1131
+ function earlierOf(
1132
+ options: AnyTestOptions,
1133
+ deprecated: (name: Deprecated) => void,
1134
+ ): EarlierAttempt & { continues: boolean } {
1135
+ const { earlier = {}, recorded, mailbox } = options;
1136
+ if (recorded !== undefined) {
1137
+ if (earlier.stages !== undefined)
1138
+ throw new Error("give earlier.stages alone: recorded is its deprecated form");
1139
+ deprecated("test-recorded");
1140
+ }
1141
+ if (mailbox !== undefined) {
1142
+ if (earlier.mailbox !== undefined)
1143
+ throw new Error("give earlier.mailbox alone: mailbox is its deprecated form");
1144
+ deprecated("test-mailbox");
1145
+ }
1146
+ const stages = earlier.stages ?? recorded;
1147
+ const left = earlier.mailbox ?? mailbox;
1148
+ return {
1149
+ ...(stages === undefined ? {} : { stages }),
1150
+ ...(left === undefined ? {} : { mailbox: left }),
1151
+ continues: options.earlier !== undefined || stages !== undefined || left !== undefined,
1152
+ };
1153
+ }
1154
+
1155
+ /**
1156
+ * The args the run is given: the test's, or what `prepare` makes of no argv, as `awf run` given
1157
+ * none, or a run an event starts, has. A workflow of the engine's own tests, without `prepare`,
1158
+ * gets `null`.
1159
+ */
1160
+ function argsOf(
1161
+ workflow: { readonly meta: { name: string }; prepare?: Workflow["prepare"] },
1162
+ options: AnyTestOptions,
1163
+ cwd: string,
1164
+ ): JsonValue {
1165
+ if (options.args !== undefined) return options.args;
1166
+ if (workflow.prepare === undefined) return null;
1167
+ try {
1168
+ const args: unknown = workflow.prepare({ argv: [], cwd });
1169
+ assertJsonValue(args, `${workflow.meta.name} arguments`);
1170
+ return args;
1171
+ } catch (error) {
1172
+ throw new Error(
1173
+ `${workflow.meta.name} was given no args, and its prepare of none failed: ${messageOf(error)}; give args`,
1174
+ { cause: error },
1175
+ );
1176
+ }
1177
+ }
1178
+
1179
+ /** An event's id, given as `eventId` or, deprecated, `id`. */
1180
+ function eventIdOf(event: EventIds, deprecated: (name: Deprecated) => void): string | undefined {
1181
+ if (event.id === undefined) return event.eventId;
1182
+ deprecated("test-event-id");
1183
+ if (event.eventId !== undefined && event.eventId !== event.id)
1184
+ throw new Error(`an event gives eventId ${event.eventId} and id ${event.id}: give eventId`);
1185
+ return event.id;
1186
+ }
1187
+
1188
+ function sandboxesOf(sandboxes: readonly SandboxRecord[] | undefined): Map<string, AgentSandbox> {
1189
+ return new Map(
1190
+ (sandboxes ?? []).flatMap(({ key, provider, spec, domains, agents }) =>
1191
+ agents.map(({ agent }) => [agent, { key, provider, spec, domains }] as const),
1192
+ ),
1193
+ );
1194
+ }
1195
+
1196
+ /** A run that parked, and its last wait: what the attempt ended with. */
1197
+ function parkedOf(settled: { value: unknown } | { error: unknown }): TestRun<unknown>["parked"] {
1198
+ if (!("error" in settled)) return undefined;
1199
+ const parked = primaryFailure(settled.error);
1200
+ return parked instanceof RunParked ? parked.wait : undefined;
1201
+ }
1202
+
1203
+ /** Decided as `awf run` decides it: a stop is the run's first failure, whatever failed beside it. */
1204
+ function stoppedOf(settled: { value: unknown } | { error: unknown }): TestRun<unknown>["stopped"] {
1205
+ if (!("error" in settled)) return undefined;
1206
+ const stop = primaryFailure(settled.error);
1207
+ if (!(stop instanceof WorkflowStopped)) return undefined;
1208
+ return {
1209
+ reason: stop.reason,
1210
+ ...(stop.stage === undefined ? {} : { stage: stop.stage }),
1211
+ ...(stop.needs === undefined ? {} : { needs: stop.needs }),
1212
+ };
1213
+ }