@niwa-labs/workflow 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +13 -2
  3. package/package.json +55 -4
  4. package/src/actions.ts +408 -0
  5. package/src/agent.ts +211 -0
  6. package/src/ask.ts +84 -0
  7. package/src/bridge.ts +710 -0
  8. package/src/catalogue.ts +74 -0
  9. package/src/check.ts +880 -0
  10. package/src/child.ts +1378 -0
  11. package/src/cockpit/actions.ts +128 -0
  12. package/src/cockpit/catalogue-actions.ts +97 -0
  13. package/src/cockpit/dashboard.tsx +134 -0
  14. package/src/cockpit/groups.ts +45 -0
  15. package/src/cockpit/index.tsx +50 -0
  16. package/src/cockpit/launch.tsx +136 -0
  17. package/src/cockpit/plan.ts +101 -0
  18. package/src/cockpit/run-commands.tsx +102 -0
  19. package/src/cockpit/run-page.tsx +140 -0
  20. package/src/cockpit/run-summary.ts +16 -0
  21. package/src/cockpit/runs.tsx +73 -0
  22. package/src/cockpit/statistics.tsx +176 -0
  23. package/src/cockpit/track.tsx +159 -0
  24. package/src/cockpit/workflows.tsx +383 -0
  25. package/src/describe-cache.ts +122 -0
  26. package/src/describe.ts +174 -0
  27. package/src/guide.ts +213 -0
  28. package/src/index.ts +2813 -0
  29. package/src/inspect.ts +235 -0
  30. package/src/installation.ts +90 -0
  31. package/src/language.ts +217 -0
  32. package/src/records-entry.ts +63 -0
  33. package/src/records.ts +574 -0
  34. package/src/registry.ts +250 -0
  35. package/src/revise.ts +89 -0
  36. package/src/runner.ts +339 -0
  37. package/src/script-types.ts +549 -0
  38. package/src/stats.ts +460 -0
  39. package/src/status.ts +512 -0
  40. package/src/store.ts +259 -0
  41. package/src/subscriptions.ts +102 -0
  42. package/src/waiting.ts +61 -0
  43. package/test/cockpit-actions.test.ts +20 -0
  44. package/test/cockpit-groups.test.ts +42 -0
  45. package/test/cockpit-plan.test.ts +118 -0
  46. package/test/fixtures/branches-host.ts +58 -0
  47. package/test/fixtures/branches-script.js +23 -0
  48. package/test/fixtures/bridge-host.ts +23 -0
  49. package/test/fixtures/bridge-keys-script.js +15 -0
  50. package/test/fixtures/bridge-wait-script.js +8 -0
  51. package/test/fixtures/cancel-host.ts +58 -0
  52. package/test/fixtures/cancel-script.js +12 -0
  53. package/test/fixtures/crash-host.ts +61 -0
  54. package/test/fixtures/crash-script.js +14 -0
  55. package/test/fixtures/ping-records.ts +20 -0
  56. package/test/fixtures/revision-host.ts +57 -0
  57. package/test/fixtures/revision-script.js +11 -0
  58. package/test/fixtures/running-host.ts +61 -0
  59. package/test/fixtures/running-script.js +12 -0
  60. package/test/fixtures/with-data/flow.ts +2 -0
  61. package/test/fixtures/with-data/table.json +1 -0
  62. package/test/git-repo.ts +45 -0
  63. package/test/git-worktree.test.ts +413 -0
  64. package/test/question-format.test.ts +279 -0
  65. package/test/workflow-agent-caps.test.ts +91 -0
  66. package/test/workflow-agent-lifetime.test.ts +93 -0
  67. package/test/workflow-agent-options.test.ts +216 -0
  68. package/test/workflow-agent-schema.test.ts +386 -0
  69. package/test/workflow-agent-typings.test.ts +241 -0
  70. package/test/workflow-ask-resume.test.ts +145 -0
  71. package/test/workflow-branch-keys.test.ts +133 -0
  72. package/test/workflow-bridge-inproc.test.ts +230 -0
  73. package/test/workflow-bridge.test.ts +141 -0
  74. package/test/workflow-builtin-candidate.test.ts +230 -0
  75. package/test/workflow-builtin.test.ts +197 -0
  76. package/test/workflow-cancel-agent.test.ts +144 -0
  77. package/test/workflow-cancel.test.ts +166 -0
  78. package/test/workflow-chain.test.ts +84 -0
  79. package/test/workflow-check-options.test.ts +144 -0
  80. package/test/workflow-check-uses.test.ts +148 -0
  81. package/test/workflow-check.test.ts +282 -0
  82. package/test/workflow-contract.test.ts +295 -0
  83. package/test/workflow-crash-resume.test.ts +141 -0
  84. package/test/workflow-deadline.test.ts +102 -0
  85. package/test/workflow-describe-cache.test.ts +240 -0
  86. package/test/workflow-directory.test.ts +73 -0
  87. package/test/workflow-emits.test.ts +261 -0
  88. package/test/workflow-group-keys.test.ts +72 -0
  89. package/test/workflow-harnesses.test.ts +262 -0
  90. package/test/workflow-inspect.test.ts +152 -0
  91. package/test/workflow-language-types.test.ts +148 -0
  92. package/test/workflow-manifest.test.ts +93 -0
  93. package/test/workflow-manual-pause.test.ts +116 -0
  94. package/test/workflow-owners.test.ts +154 -0
  95. package/test/workflow-pause.test.ts +200 -0
  96. package/test/workflow-phases.test.ts +445 -0
  97. package/test/workflow-plugin-entries.test.ts +59 -0
  98. package/test/workflow-record-refs.test.ts +310 -0
  99. package/test/workflow-replay.test.ts +206 -0
  100. package/test/workflow-resume-once.test.ts +188 -0
  101. package/test/workflow-revision.test.ts +216 -0
  102. package/test/workflow-scope.test.ts +505 -0
  103. package/test/workflow-script-defaults.test.ts +119 -0
  104. package/test/workflow-script-error.test.ts +228 -0
  105. package/test/workflow-script-libraries.test.ts +88 -0
  106. package/test/workflow-script-packages.test.ts +119 -0
  107. package/test/workflow-stdout.test.ts +89 -0
  108. package/test/workflow-stop.test.ts +299 -0
  109. package/test/workflow-store.test.ts +220 -0
  110. package/test/workflow-visible.test.ts +136 -0
  111. package/test/workflow-wait-version.test.ts +54 -0
  112. package/test/workflow-wait.test.ts +224 -0
  113. package/test/workflow-waiting-agent.test.ts +64 -0
  114. package/tsconfig.json +15 -0
@@ -0,0 +1,250 @@
1
+ /**
2
+ * The registered workflows as the journal holds them.
3
+ *
4
+ * A workflow is a `workflow.registered` record: the first one issues its id, a later one under the
5
+ * same id is a new version. A version is the hash of the source closure of the script, the entry
6
+ * file and the local files it imports, so an edit of a helper is a new version and a dependency
7
+ * from `node_modules` is not. A version whose latest check after a change of a workflow it calls
8
+ * failed is broken.
9
+ */
10
+
11
+ import { createHash } from "node:crypto";
12
+ import { existsSync, readFileSync, statSync } from "node:fs";
13
+ import { dirname, relative, resolve } from "node:path";
14
+ import type { z } from "zod";
15
+
16
+ import { stableJson } from "@niwa-labs/core";
17
+ import type { EntityDescription, IssuedId, JournalReader, RecordId } from "@niwa-labs/core";
18
+
19
+ import { contractOf } from "./describe.ts";
20
+ import type { Described } from "./describe.ts";
21
+ import { SCRIPT_TYPES } from "./language.ts";
22
+ import type { WorkflowPhases } from "./language.ts";
23
+ import { BuiltinsWanted, BuiltinPromoted, WorkflowRechecked, WorkflowRegistered, WorkflowVersion } from "./records.ts";
24
+ import type { Builtin, ContractHash, EmittedRecord, FlowCall, JsonSchemaValue, Subscription, WorkflowId } from "./records.ts";
25
+
26
+ /** One version of a registered workflow. */
27
+ export interface RegisteredVersion {
28
+ version: WorkflowVersion;
29
+ revision: RecordId;
30
+ file: string;
31
+ description: EntityDescription;
32
+ input?: JsonSchemaValue;
33
+ output?: JsonSchemaValue;
34
+ contract: ContractHash;
35
+ uses: string[];
36
+ calls: FlowCall[];
37
+ /** The `name` of the script. */
38
+ name?: string;
39
+ /** The records of scripts it waits for or reads by name. */
40
+ reads: string[];
41
+ /** The record types it declares in `emits`. */
42
+ emits?: EmittedRecord[];
43
+ /** The records that start it once it is installed. */
44
+ on?: Subscription[];
45
+ /** The phases it declares, in order. */
46
+ phases?: WorkflowPhases;
47
+ /** Why it does not start: the errors of its latest check, when that check failed. */
48
+ broken?: string;
49
+ /** A candidate version of a gated built-in, not accepted yet. */
50
+ candidate?: true;
51
+ /** A candidate replaced by another proposal before it was accepted. */
52
+ superseded?: true;
53
+ }
54
+
55
+ /** A registered workflow: its id, the record that issued it, and its versions, the latest last. */
56
+ export interface RegisteredWorkflow {
57
+ id: WorkflowId;
58
+ record: RecordId;
59
+ versions: RegisteredVersion[];
60
+ latest: RegisteredVersion;
61
+ /** The built-in it is, when a plugin ships it: registered by the workflow module, never by hand. */
62
+ builtin?: Builtin;
63
+ }
64
+
65
+ /** Every registered workflow, in the order of their first registration. */
66
+ export function workflowsOf(journal: JournalReader): RegisteredWorkflow[] {
67
+ const checks = journal.find({ type: WorkflowRechecked.name }).map((row) => WorkflowRechecked.schema.parse(row.data));
68
+ const byId = new Map<WorkflowId, RegisteredWorkflow>();
69
+ const rows = [...journal.find({ type: WorkflowRegistered.name }), ...journal.find({ type: BuiltinPromoted.name })].sort((a, b) => a.seq - b.seq);
70
+ for (const row of rows) {
71
+ if (row.type === BuiltinPromoted.name) {
72
+ // Promoted candidates are accepted: each moves to the end of its versions and is the latest one.
73
+ for (const { workflow, version } of BuiltinPromoted.schema.parse(row.data).versions) {
74
+ const known = byId.get(workflow);
75
+ const was = known?.versions.find((one) => one.version === version);
76
+ if (known === undefined || was === undefined) continue;
77
+ const { candidate: _, superseded: _superseded, ...accepted } = was;
78
+ known.versions = [...known.versions.filter((one) => one !== was), accepted];
79
+ known.latest = accepted;
80
+ }
81
+ continue;
82
+ }
83
+ const data = WorkflowRegistered.schema.parse(row.data);
84
+ const last = checks.filter((check) => check.workflow === data.workflow && check.version === data.version).at(-1);
85
+ const version: RegisteredVersion = {
86
+ version: data.version,
87
+ revision: row.id,
88
+ file: data.file,
89
+ description: data.description,
90
+ ...(data.input === undefined ? {} : { input: data.input }),
91
+ ...(data.output === undefined ? {} : { output: data.output }),
92
+ contract: data.contract,
93
+ uses: data.uses,
94
+ calls: last?.calls ?? data.calls,
95
+ ...(data.name === undefined ? {} : { name: data.name }),
96
+ reads: data.reads ?? [],
97
+ ...(data.emits === undefined ? {} : { emits: data.emits }),
98
+ ...(data.on === undefined ? {} : { on: data.on }),
99
+ ...(data.phases === undefined ? {} : { phases: data.phases }),
100
+ ...(last === undefined || last.ok ? {} : { broken: last.errors.map((error) => `${error.code} ${error.message}`).join("; ") }),
101
+ };
102
+ const known = byId.get(data.workflow);
103
+ if (known === undefined) {
104
+ const first = data.candidate === true ? { ...version, candidate: true as const } : version;
105
+ byId.set(data.workflow, { id: data.workflow, record: row.id, versions: [first], latest: first, ...(data.builtin === undefined ? {} : { builtin: data.builtin }) });
106
+ } else {
107
+ // A version that was accepted stays accepted; a candidate moves the latest version only while no version is accepted.
108
+ const was = known.versions.find((one) => one.version === data.version);
109
+ const candidate = data.candidate === true && (was === undefined || was.candidate === true);
110
+ const now = candidate ? { ...version, candidate: true as const } : version;
111
+ const older = known.versions.filter((one) => one.version !== data.version).map((one) => {
112
+ if (known.builtin === undefined || one.candidate !== true) return one;
113
+ const { candidate: _, ...superseded } = one;
114
+ return { ...superseded, superseded: true as const };
115
+ });
116
+ known.versions = [...older, now];
117
+ if (!candidate || known.latest.candidate === true) known.latest = now;
118
+ }
119
+ }
120
+ return [...byId.values()];
121
+ }
122
+
123
+ /** The current offer, keeping its generation when the same set is requested again for installation. */
124
+ export function builtinOfferOf(journal: JournalReader): { id: RecordId; data: z.output<typeof BuiltinsWanted.schema> } | undefined {
125
+ let current: { id: RecordId; data: z.output<typeof BuiltinsWanted.schema> } | undefined;
126
+ for (const row of journal.find({ type: BuiltinsWanted.name })) {
127
+ const data = BuiltinsWanted.schema.parse(row.data);
128
+ if (current === undefined || stableJson(current.data.items) !== stableJson(data.items)) current = { id: row.id, data };
129
+ }
130
+ return current;
131
+ }
132
+
133
+ /**
134
+ * What a registered version declared, as its registration holds it: the module need not be read for it.
135
+ * Undefined for a version that was not registered, and for one whose registration does not give back the
136
+ * contract it was made under.
137
+ */
138
+ export function declaredBy(journal: JournalReader, version: WorkflowVersion): Described | undefined {
139
+ const registered = workflowsOf(journal)
140
+ .flatMap((one) => one.versions)
141
+ .find((one) => one.version === version);
142
+ if (registered === undefined) return undefined;
143
+ const emits = registered.emits?.map(({ record, schema }) => ({ name: record, schema }));
144
+ // The name is kept by the registration; one made before it was kept has it between the parts of its records `wf.<name>.<record>`.
145
+ const first = registered.emits?.[0];
146
+ const name =
147
+ registered.name ??
148
+ (first === undefined || !first.type.startsWith(SCRIPT_TYPES) || !first.type.endsWith(`.${first.record}`) ? undefined : first.type.slice(SCRIPT_TYPES.length, -(first.record.length + 1)));
149
+ const described: Described = {
150
+ ...(name === undefined ? {} : { name }),
151
+ description: registered.description,
152
+ ...(registered.input === undefined ? {} : { input: registered.input }),
153
+ ...(registered.output === undefined ? {} : { output: registered.output }),
154
+ ...(emits === undefined ? {} : { emits }),
155
+ ...(registered.phases === undefined ? {} : { phases: registered.phases }),
156
+ problems: [],
157
+ };
158
+ return contractOf(described) === registered.contract ? described : undefined;
159
+ }
160
+
161
+ /** The registered workflow that is this built-in, when it was registered. */
162
+ export const builtinOf = (workflows: readonly RegisteredWorkflow[], builtin: Builtin): RegisteredWorkflow | undefined =>
163
+ workflows.find((one) => one.builtin?.plugin === builtin.plugin && one.builtin.name === builtin.name);
164
+
165
+ /** The workflow ids that were issued, each with the record it was born in. */
166
+ export const workflowIdsOf = (journal: JournalReader): IssuedId[] =>
167
+ workflowsOf(journal).map((one) => ({ id: one.id, seed: one.record }));
168
+
169
+ /** The workflows whose latest version calls this one. */
170
+ export const dependentsOf = (workflows: readonly RegisteredWorkflow[], id: WorkflowId): RegisteredWorkflow[] =>
171
+ workflows.filter((one) => one.id !== id && one.latest.calls.some((call) => call.workflow === id));
172
+
173
+ /** The workflows whose latest version waits for or reads one of these record types, by name. */
174
+ export const readersOf = (workflows: readonly RegisteredWorkflow[], id: WorkflowId, types: ReadonlySet<string>): RegisteredWorkflow[] =>
175
+ workflows.filter((one) => one.id !== id && one.latest.reads.some((type) => types.has(type)));
176
+
177
+ /** The name of the action a registered workflow is. */
178
+ export const flowActionName = (id: WorkflowId): `flow.${WorkflowId}` => `flow.${id}`;
179
+
180
+ const transpiler = new Bun.Transpiler({ loader: "tsx" });
181
+
182
+ /** The local file an import names, as the runtime finds it; undefined for a package, or a path that is not there. */
183
+ function localImport(from: string, specifier: string): string | undefined {
184
+ if (!specifier.startsWith(".") && !specifier.startsWith("/")) return undefined;
185
+ const base = resolve(dirname(from), specifier);
186
+ return [base, `${base}.ts`, `${base}.tsx`, `${base}.js`, `${base}/index.ts`].find((candidate) => existsSync(candidate) && statSync(candidate).isFile());
187
+ }
188
+
189
+ /** The source closure of a script: the entry and every local file it imports, each by its absolute path, with its bytes. */
190
+ export interface SourceClosure {
191
+ entry: string;
192
+ files: Map<string, Buffer>;
193
+ }
194
+
195
+ /** The extensions of the files a script imports as code; any other file it imports is data. */
196
+ const CODE = /\.(?:[cm]?[jt]s|[jt]sx)$/;
197
+
198
+ /** Is this file of a closure code, whose imports are part of the closure, or data, which imports nothing? */
199
+ export const isCode = (path: string): boolean => CODE.test(path);
200
+
201
+ /** The source closure of a script, as it is on disk now. Undefined when the entry cannot be read. */
202
+ export function closureOf(file: string): SourceClosure | undefined {
203
+ if (!existsSync(file)) return undefined;
204
+ const entry = resolve(file);
205
+ const files = new Map<string, Buffer>();
206
+ const pending = [entry];
207
+ for (let next = pending.pop(); next !== undefined; next = pending.pop()) {
208
+ if (files.has(next)) continue;
209
+ const bytes = readFileSync(next);
210
+ files.set(next, bytes);
211
+ // A data file (`./table.json`) is part of the version by its bytes; it is not read for imports.
212
+ if (!isCode(next)) continue;
213
+ for (const { path } of transpiler.scanImports(bytes.toString("utf8"))) {
214
+ const found = localImport(next, path);
215
+ if (found !== undefined) pending.push(found);
216
+ }
217
+ }
218
+ return { entry, files };
219
+ }
220
+
221
+ /** The packages a source closure imports by name: the specifiers of its code files that are not paths. */
222
+ export function packageImportsOf(closure: SourceClosure): string[] {
223
+ const named = new Set<string>();
224
+ for (const [path, bytes] of closure.files) {
225
+ if (!isCode(path)) continue;
226
+ for (const { path: specifier } of transpiler.scanImports(bytes.toString("utf8"))) {
227
+ if (!specifier.startsWith(".") && !specifier.startsWith("/")) named.add(specifier);
228
+ }
229
+ }
230
+ return [...named].sort();
231
+ }
232
+
233
+ /** The version a source closure is: the sha256 of each file by its path relative to the entry and its bytes. */
234
+ export function versionOfClosure(closure: SourceClosure): WorkflowVersion {
235
+ const hash = createHash("sha256");
236
+ const home = dirname(closure.entry);
237
+ for (const [path, bytes] of [...closure.files].sort(([a], [b]) => a.localeCompare(b))) {
238
+ hash.update(`${relative(home, path)}\0`).update(bytes).update("\0");
239
+ }
240
+ return WorkflowVersion.parse(hash.digest("hex"));
241
+ }
242
+
243
+ /**
244
+ * The version of a script: the sha256 of its source closure, the entry and every local file it imports, each
245
+ * by its path relative to the entry and its bytes. Undefined when the entry cannot be read.
246
+ */
247
+ export function versionOf(file: string): WorkflowVersion | undefined {
248
+ const closure = closureOf(file);
249
+ return closure === undefined ? undefined : versionOfClosure(closure);
250
+ }
package/src/revise.ts ADDED
@@ -0,0 +1,89 @@
1
+ /**
2
+ * What a run of a script takes from the earlier starts of the same run.
3
+ *
4
+ * Steps are taken along the longest unchanged prefix. A call of the script as it is now is answered
5
+ * with what the run recorded under its key while it is the same call — the same action and input;
6
+ * for an agent call the same prompt and harness — and nothing before it in its branch changed.
7
+ * The first call that is new or changed runs, and so does every call after it: later in its branch,
8
+ * or after the scope that holds it closed. A call after a change that has an effect and was made
9
+ * before as the same call is taken all the same: an effect is never made twice.
10
+ *
11
+ * Nothing is taken from another run: the calls compared are the run's own.
12
+ */
13
+
14
+ import { stableJson } from "@niwa-labs/core";
15
+ import type { JsonValue, Recovery } from "@niwa-labs/core";
16
+
17
+ /** One call of a run, as the journal holds it. */
18
+ export interface RecordedCall {
19
+ key: string;
20
+ action: string;
21
+ input: JsonValue;
22
+ /** Whether the call ended with an answer. */
23
+ finished: boolean;
24
+ /** The answer, when it finished. */
25
+ outcome?: JsonValue;
26
+ /** The recovery contract it was made under: anything but `repeat` has an effect. */
27
+ recovery?: Recovery;
28
+ /** The start of the run that made it. */
29
+ invocation: string;
30
+ }
31
+
32
+ /** One call of the script as it is now. */
33
+ export interface CurrentCall {
34
+ key: string;
35
+ action: string;
36
+ input: JsonValue;
37
+ /** An agent call: its identity is its prompt and its harness, not the other options of its session. */
38
+ agent?: boolean;
39
+ }
40
+
41
+ /** Why a call runs instead of being taken from an earlier start. */
42
+ export type RerunReason = "new" | "changed" | "unfinished" | "after a change" | "recomputed";
43
+
44
+ /** What becomes of one call. */
45
+ export type Taken = { taken: true; outcome: JsonValue } | { taken: false; reason: RerunReason };
46
+
47
+ /** What a revision found: the calls taken from the earlier starts, and the ones that run. */
48
+ export interface RevisionReport {
49
+ reused: Array<{ key: string; action: string; input: JsonValue; outcome: JsonValue }>;
50
+ rerun: Array<{ key: string; action: string; input: JsonValue; reason: RerunReason }>;
51
+ }
52
+
53
+ /** The input of a call without its label: the label is the key, which is compared on its own. */
54
+ function withoutLabel(input: JsonValue): JsonValue {
55
+ if (input === null || typeof input !== "object" || Array.isArray(input)) return input;
56
+ const { label: _label, ...rest } = input;
57
+ return rest;
58
+ }
59
+
60
+ /** What makes two calls the same call. An agent call is its prompt and its harness, which its action names. */
61
+ function identityOf(call: { action: string; input: JsonValue }, agent: boolean): string {
62
+ if (agent && call.input !== null && typeof call.input === "object" && !Array.isArray(call.input)) {
63
+ return stableJson({ action: call.action, prompt: call.input.prompt ?? null });
64
+ }
65
+ return stableJson({ action: call.action, input: withoutLabel(call.input) });
66
+ }
67
+
68
+ /** Whether a recorded call and a current one are the same call. */
69
+ export function sameCall(recorded: { action: string; input: JsonValue }, current: CurrentCall): boolean {
70
+ return identityOf(recorded, current.agent === true) === identityOf(current, current.agent === true);
71
+ }
72
+
73
+ /** Whether a recorded call has an effect: it was made under a contract other than `repeat`. */
74
+ export const hasEffect = (call: RecordedCall): boolean => (call.recovery ?? "repeat") !== "repeat";
75
+
76
+ /**
77
+ * Decide one call: taken from the earlier start that made it, or run.
78
+ *
79
+ * `after` says a call before it in its branch ran, so it lies after a change; `recompute` says the
80
+ * owner asked for it to run again.
81
+ */
82
+ export function decide(before: RecordedCall | undefined, current: CurrentCall, opts: { after: boolean; recompute: boolean }): Taken {
83
+ if (opts.recompute) return { taken: false, reason: "recomputed" };
84
+ if (before === undefined) return { taken: false, reason: "new" };
85
+ if (!sameCall(before, current)) return { taken: false, reason: "changed" };
86
+ if (!before.finished) return { taken: false, reason: "unfinished" };
87
+ if (opts.after && !hasEffect(before)) return { taken: false, reason: "after a change" };
88
+ return { taken: true, outcome: before.outcome ?? null };
89
+ }
package/src/runner.ts ADDED
@@ -0,0 +1,339 @@
1
+ /**
2
+ * The runner of a workflow script.
3
+ *
4
+ * The script runs in a disposable process (`child.ts`). The parent owns all state: the child asks
5
+ * it to make every call, and the answers come back by the number of the call, so the order of the
6
+ * results does not depend on which branch finishes first.
7
+ */
8
+
9
+ import type { Readable } from "node:stream";
10
+ import { fileURLToPath, pathToFileURL } from "node:url";
11
+ import { dirname, resolve } from "node:path";
12
+
13
+ import { z } from "zod";
14
+
15
+ import { StepCancelled, TOKEN_ENV, isSuspend } from "@niwa-labs/core";
16
+ import { spawnAgent } from "@niwa-labs/sandbox";
17
+ import type { AgentProcess, BoxSandboxName } from "@niwa-labs/sandbox";
18
+
19
+ import { isPausing } from "./bridge.ts";
20
+ import { readableRoots } from "./installation.ts";
21
+ import { PROTOCOL_PREFIX, ScriptRequest } from "./language.ts";
22
+
23
+ const CHILD = fileURLToPath(new URL("./child.ts", import.meta.url));
24
+
25
+ /** The run a script process belongs to: its id, so a process can be told from the host's own. */
26
+ export const RUN_ENV = "NIWA_WORKFLOW_RUN";
27
+
28
+ /** Whether the script process runs in a box: `boxed` or `plain`. */
29
+ export const BOX_ENV = "NIWA_WORKFLOW_BOX";
30
+
31
+ /**
32
+ * The box a script process runs in: the sandbox that holds it. The box is empty: nothing to write
33
+ * outside the process, no network, and an environment of only what the bridge needs.
34
+ */
35
+ export interface ScriptBox {
36
+ readonly sandbox: BoxSandboxName;
37
+ /** The host's keys: a box never reads them, whatever else it may read. */
38
+ readonly closed: readonly string[];
39
+ /** What the box may read besides the script and the packages niwa runs with: the folder of the stored copies, where a copy finds those packages. */
40
+ readonly read?: readonly string[];
41
+ }
42
+
43
+ /** What the script asks the parent to do: one call, wait, sleep or read. */
44
+ export interface CallRequest {
45
+ /** The key the request is recorded under: its label, or its place among the unlabeled calls. */
46
+ key: string;
47
+ /** The declared phase the branch of the call is in, when it is in one: the step belongs to it. */
48
+ phase?: string;
49
+ /** A call before it in its branch ran instead of being taken from an earlier start: it lies after a change. */
50
+ after?: boolean;
51
+ /** What is asked. */
52
+ request: ScriptRequest;
53
+ }
54
+
55
+ /** What the parent does for the child. */
56
+ export interface RunnerDeps {
57
+ /** Do what the script asked and answer it. */
58
+ call(request: CallRequest): Promise<unknown>;
59
+ /** Whether the call under the key ran instead of being taken from an earlier start of the run. */
60
+ fresh?(key: string): boolean;
61
+ /** A branch of the script entered a phase (`""` is the script itself): the `ordinal`-th change in that branch, 0 being the phase it was made in. */
62
+ phase(entered: { branch: string; ordinal: number; phase: string }): void;
63
+ /** A branch that was in a phase ended. */
64
+ left(branch: string): void;
65
+ /** A branch joined a group of branches, before its first step. */
66
+ branched?(joined: { scope: string; key: string }): void;
67
+ /** The script wrote a line to the log. */
68
+ log(text: string): void;
69
+ /** The run is being stopped: the calls of the script that are in flight are no longer wanted. */
70
+ stopped?(): void;
71
+ /**
72
+ * The script process is gone: a call that waits for something only the process could bring
73
+ * about must answer now, or the calls in flight that the runner waits for never end.
74
+ */
75
+ gone?(): void;
76
+ }
77
+
78
+ /** What a run of a script ended with. */
79
+ export interface RunnerResult {
80
+ ok: boolean;
81
+ /** What the script returned: any JSON, until the host checks it against the `output` of the script. */
82
+ result?: unknown;
83
+ error?: string;
84
+ /** The name of the error the script ended with (`AskCancelled`, `TypeError`, ...), when it threw one. */
85
+ errorName?: string;
86
+ /** The run was stopped while it was going: its child was killed. */
87
+ cancelled?: boolean;
88
+ }
89
+
90
+ /**
91
+ * The child process of a run that is going.
92
+ *
93
+ * A run is a process: the handle is what makes it possible to stop it, and a run that is not
94
+ * going has no handle and nothing to stop.
95
+ */
96
+ interface Running {
97
+ readonly child: AgentProcess;
98
+ /** What the run is told when it is stopped. */
99
+ readonly deps: RunnerDeps;
100
+ /** Whether the run was stopped by a cancel. */
101
+ stopped: boolean;
102
+ }
103
+
104
+ const children = new Map<string, Running>();
105
+
106
+ /**
107
+ * How long a child that has a suspended call may stay silent before it is stopped. A child ends by
108
+ * itself once only parked calls are left; this stops one that something of the script keeps alive.
109
+ */
110
+ const QUIET_MS = 30_000;
111
+
112
+ /**
113
+ * Stop the child process of a run.
114
+ *
115
+ * The child is disposable and holds nothing, so it is stopped and not asked. It says whether
116
+ * there was a child to stop.
117
+ */
118
+ export function stopRun(run: string): boolean {
119
+ const running = children.get(run);
120
+ if (running === undefined) return false;
121
+ running.stopped = true;
122
+ running.deps.stopped?.();
123
+ running.child.kill();
124
+ return true;
125
+ }
126
+
127
+ /** What a stream says, read to its end. */
128
+ async function textOf(stream: Readable): Promise<string> {
129
+ const decoder = new TextDecoder();
130
+ let text = "";
131
+ for await (const chunk of stream) text += decoder.decode(chunk as Uint8Array, { stream: true });
132
+ return text + decoder.decode();
133
+ }
134
+
135
+ const Message = z.discriminatedUnion("kind", [
136
+ z.object({
137
+ kind: z.literal("call"),
138
+ ordinal: z.number(),
139
+ key: z.string(),
140
+ phase: z.string().optional(),
141
+ after: z.boolean().optional(),
142
+ request: ScriptRequest,
143
+ }),
144
+ z.object({ kind: z.literal("phase"), branch: z.string(), ordinal: z.number().int().nonnegative(), phase: z.string() }),
145
+ z.object({ kind: z.literal("left"), branch: z.string() }),
146
+ z.object({ kind: z.literal("branched"), scope: z.string(), key: z.string() }),
147
+ z.object({ kind: z.literal("log"), text: z.string() }),
148
+ z.object({ kind: z.literal("done"), result: z.unknown() }),
149
+ z.object({ kind: z.literal("failed"), error: z.string(), name: z.string().optional() }),
150
+ ]);
151
+
152
+ /**
153
+ * Start the disposable process of a script, in a mode of the child (`run` or `describe`).
154
+ *
155
+ * The process is born in the launch funnel. The token of the owner is never handed on, boxed or not; in
156
+ * a box nothing of the host's environment is, and the markers tell the two modes apart. A script that
157
+ * is run in a box is also read in one: its top level is the script's own code.
158
+ */
159
+ export async function spawnScriptChild(mode: "run" | "describe", file: string, extra: readonly string[], run: string, box?: ScriptBox): Promise<AgentProcess> {
160
+ const path = resolve(file);
161
+ return spawnAgent({
162
+ harness: "workflow",
163
+ program: process.execPath,
164
+ args: [CHILD, mode, pathToFileURL(path).href, ...extra],
165
+ unset: [TOKEN_ENV],
166
+ env: { [RUN_ENV]: run, [BOX_ENV]: box === undefined ? "plain" : "boxed" },
167
+ ...(box === undefined
168
+ ? {}
169
+ : {
170
+ cwd: dirname(path),
171
+ box: {
172
+ sandbox: box.sandbox,
173
+ // Reading is all the box gives: the child and its modules, and the script.
174
+ spec: { cwd: dirname(path), read: [...readableRoots(), ...(box.read ?? [])], closed: [...box.closed] },
175
+ mark: run,
176
+ },
177
+ }),
178
+ });
179
+ }
180
+
181
+ /**
182
+ * Run a script in a disposable process and wait for its end. The process never outlives this call.
183
+ *
184
+ * The run may be stopped while it is going: the child is killed, and the call that was in flight
185
+ * is left to the caller — it is a step of the invocation, and the invocation decides what becomes
186
+ * of it. The runner waits for the child and nothing else.
187
+ *
188
+ * A call that waits for a record that has not arrived suspends the invocation it runs in. That
189
+ * is not a failure of the script: the child ends once it has nothing but parked calls left, the
190
+ * calls that are in flight are waited for, and the suspend is passed on, so the invocation ends as "waiting" and the run goes on
191
+ * when the answer arrives.
192
+ */
193
+ export async function runScript(
194
+ deps: RunnerDeps,
195
+ file: string,
196
+ args: unknown,
197
+ run: string,
198
+ box?: ScriptBox,
199
+ ): Promise<RunnerResult> {
200
+ const child = await spawnScriptChild("run", file, [JSON.stringify(args ?? null)], run, box);
201
+ // A script that exits before it reads its answers must not take the host down with a broken pipe.
202
+ child.stdin.on("error", () => undefined);
203
+ const exited = new Promise<number | null>((done) => child.once("close", (code) => done(code)));
204
+ const running: Running = { child, deps, stopped: false };
205
+ children.set(run, running);
206
+ let ended: RunnerResult | undefined;
207
+ let stopped = false;
208
+ let suspended: unknown;
209
+ const calls: Promise<void>[] = [];
210
+ // The calls the host is making for the child now: a suspended call is not one of them, nor a wait for a deadline.
211
+ let busy = 0;
212
+ let quiet: ReturnType<typeof setTimeout> | undefined;
213
+ /**
214
+ * A call suspended: the child is told the call is parked and goes on, so the branches that do not
215
+ * wait move on and their steps are recorded. It ends by itself once only parked calls are left and
216
+ * its code has nothing more to run; one that stays silent past the quiet time is stopped.
217
+ */
218
+ const stopWhenQuiet = (): void => {
219
+ if (suspended === undefined || busy > 0) return;
220
+ if (quiet !== undefined) clearTimeout(quiet);
221
+ quiet = setTimeout(() => {
222
+ if (busy === 0) child.kill();
223
+ }, QUIET_MS);
224
+ };
225
+
226
+ const answer = (ordinal: number, reply: { ok?: boolean; parked?: boolean; value?: unknown; error?: string; name?: string; pauses?: boolean; fresh?: boolean }): void => {
227
+ if (stopped) return;
228
+ child.stdin.write(`${JSON.stringify({ ordinal, ...reply })}\n`);
229
+ };
230
+
231
+ const handle = (line: string): void => {
232
+ // Only a line of the protocol is a message: the script shares the stdout with the child,
233
+ // and a line the script wrote is its own output, which is passed through as a log.
234
+ if (!line.startsWith(PROTOCOL_PREFIX)) {
235
+ deps.log(line);
236
+ return;
237
+ }
238
+ const message = Message.parse(JSON.parse(line.slice(PROTOCOL_PREFIX.length)));
239
+ switch (message.kind) {
240
+ case "call": {
241
+ // The call is made, and its answer goes back to the child. A call that is still in flight
242
+ // when the child is stopped is not waited for here: the invocation it belongs to owns it.
243
+ const counted = message.request.op !== "scope-deadline";
244
+ if (counted) busy += 1;
245
+ if (quiet !== undefined) clearTimeout(quiet);
246
+ calls.push(
247
+ deps.call({
248
+ key: message.key,
249
+ ...(message.phase === undefined ? {} : { phase: message.phase }),
250
+ ...(message.after === true ? { after: true } : {}),
251
+ request: message.request,
252
+ }).then(
253
+ // The child is told when the call ran instead of being taken: what comes after it in its branch runs too.
254
+ (value) => answer(message.ordinal, { ok: true, value: value ?? null, ...(deps.fresh?.(message.key) === true ? { fresh: true } : {}) }),
255
+ (error: unknown) => {
256
+ // A call that suspends is not a failure: the invocation waits, and the call is made
257
+ // again when the answer arrives. The child is told, ends once only parked calls are left, and the suspend is passed on.
258
+ if (isSuspend(error)) {
259
+ suspended ??= error;
260
+ answer(message.ordinal, { parked: true });
261
+ return;
262
+ }
263
+ answer(message.ordinal, {
264
+ ok: false,
265
+ // A cancelled step says why it was cancelled: the script sees the reason, as for a cancel of its own.
266
+ error: error instanceof StepCancelled ? error.reason : error instanceof Error ? error.message : String(error),
267
+ // The name goes with the message, so a script can tell one error from another.
268
+ ...(error instanceof Error ? { name: error.name } : {}),
269
+ // An error that pauses the run is not the outcome of a branch: the script ends on it.
270
+ ...(isPausing(error) ? { pauses: true } : {}),
271
+ ...(deps.fresh?.(message.key) === true ? { fresh: true } : {}),
272
+ });
273
+ },
274
+ ).finally(() => {
275
+ if (counted) busy -= 1;
276
+ stopWhenQuiet();
277
+ }),
278
+ );
279
+ return;
280
+ }
281
+ case "phase":
282
+ deps.phase({ branch: message.branch, ordinal: message.ordinal, phase: message.phase });
283
+ return;
284
+ case "left":
285
+ deps.left(message.branch);
286
+ return;
287
+ case "branched":
288
+ deps.branched?.({ scope: message.scope, key: message.key });
289
+ return;
290
+ case "log":
291
+ deps.log(message.text);
292
+ return;
293
+ case "done":
294
+ ended = { ok: true, result: message.result };
295
+ return;
296
+ case "failed":
297
+ ended = { ok: false, error: message.error, ...(message.name === undefined ? {} : { errorName: message.name }) };
298
+ return;
299
+ }
300
+ };
301
+
302
+ const stderr = textOf(child.stderr);
303
+ try {
304
+ const decoder = new TextDecoder();
305
+ let buffer = "";
306
+ for await (const chunk of child.stdout) {
307
+ buffer += decoder.decode(chunk, { stream: true });
308
+ for (let at = buffer.indexOf("\n"); at >= 0; at = buffer.indexOf("\n")) {
309
+ const line = buffer.slice(0, at).trim();
310
+ buffer = buffer.slice(at + 1);
311
+ if (line !== "") handle(line);
312
+ }
313
+ }
314
+ const code = await exited;
315
+ // A call suspended: the child has ended, the calls that are in flight are waited for, and
316
+ // the suspend is passed on. The invocation ends as "waiting", and the run goes on when the
317
+ // answer arrives.
318
+ if (suspended !== undefined) {
319
+ deps.gone?.();
320
+ await Promise.allSettled([...calls]);
321
+ throw suspended;
322
+ }
323
+ if (ended !== undefined) return ended;
324
+ // The run was stopped: its child was killed, so it ended there and not by the script.
325
+ if (running.stopped) return { ok: false, cancelled: true, error: "the run was cancelled" };
326
+ const reason = (await stderr).trim();
327
+ return { ok: false, error: reason !== "" ? reason : `the script exited with ${String(code)} without a result` };
328
+ } finally {
329
+ stopped = true;
330
+ if (quiet !== undefined) clearTimeout(quiet);
331
+ deps.gone?.();
332
+ child.kill();
333
+ // The handle of the run goes with it: a run that is not going has nothing to stop.
334
+ if (children.get(run) === running) children.delete(run);
335
+ // The calls that are in flight are not waited for: a call that was cut off is the
336
+ // invocation's, and it decides what becomes of it.
337
+ for (const call of calls) void call.catch(() => undefined);
338
+ }
339
+ }