@sanoma-ai/workflows 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +105 -0
- package/README.md +493 -2
- package/dist/approvals.d.ts +46 -0
- package/dist/approvals.js +111 -0
- package/dist/approvals.js.map +1 -0
- package/dist/ast.d.ts +57 -0
- package/dist/ast.js +81 -0
- package/dist/ast.js.map +1 -0
- package/dist/call.d.ts +33 -0
- package/dist/call.js +402 -0
- package/dist/call.js.map +1 -0
- package/dist/client.d.ts +167 -0
- package/dist/client.js +400 -0
- package/dist/client.js.map +1 -0
- package/dist/config.d.ts +126 -0
- package/dist/config.js +247 -0
- package/dist/config.js.map +1 -0
- package/dist/credentials.d.ts +30 -0
- package/dist/credentials.js +193 -0
- package/dist/credentials.js.map +1 -0
- package/dist/cucumber.d.ts +64 -0
- package/dist/cucumber.js +158 -0
- package/dist/cucumber.js.map +1 -0
- package/dist/datafile.d.ts +62 -0
- package/dist/datafile.js +326 -0
- package/dist/datafile.js.map +1 -0
- package/dist/db.d.ts +58 -0
- package/dist/db.js +179 -0
- package/dist/db.js.map +1 -0
- package/dist/define.d.ts +205 -0
- package/dist/define.js +103 -0
- package/dist/define.js.map +1 -0
- package/dist/describe.d.ts +135 -0
- package/dist/describe.js +127 -0
- package/dist/describe.js.map +1 -0
- package/dist/drift.d.ts +75 -0
- package/dist/drift.js +90 -0
- package/dist/drift.js.map +1 -0
- package/dist/env.d.ts +49 -0
- package/dist/env.js +83 -0
- package/dist/env.js.map +1 -0
- package/dist/errors.d.ts +88 -0
- package/dist/errors.js +148 -0
- package/dist/errors.js.map +1 -0
- package/dist/expectations.d.ts +26 -0
- package/dist/expectations.js +137 -0
- package/dist/expectations.js.map +1 -0
- package/dist/fake.d.ts +55 -0
- package/dist/fake.js +87 -0
- package/dist/fake.js.map +1 -0
- package/dist/fill.d.ts +9 -0
- package/dist/fill.js +17 -0
- package/dist/fill.js.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -0
- package/dist/ledger.d.ts +217 -0
- package/dist/ledger.js +251 -0
- package/dist/ledger.js.map +1 -0
- package/dist/lint.d.ts +32 -0
- package/dist/lint.js +168 -0
- package/dist/lint.js.map +1 -0
- package/dist/log.d.ts +24 -0
- package/dist/log.js +83 -0
- package/dist/log.js.map +1 -0
- package/dist/op.d.ts +257 -0
- package/dist/op.js +188 -0
- package/dist/op.js.map +1 -0
- package/dist/outline.d.ts +179 -0
- package/dist/outline.js +506 -0
- package/dist/outline.js.map +1 -0
- package/dist/paths.d.ts +11 -0
- package/dist/paths.js +30 -0
- package/dist/paths.js.map +1 -0
- package/dist/plugin.d.ts +28 -0
- package/dist/plugin.js +25 -0
- package/dist/plugin.js.map +1 -0
- package/dist/policy.d.ts +114 -0
- package/dist/policy.js +60 -0
- package/dist/policy.js.map +1 -0
- package/dist/resource.d.ts +212 -0
- package/dist/resource.js +285 -0
- package/dist/resource.js.map +1 -0
- package/dist/resources.d.ts +62 -0
- package/dist/resources.js +197 -0
- package/dist/resources.js.map +1 -0
- package/dist/run.d.ts +87 -0
- package/dist/run.js +2 -0
- package/dist/run.js.map +1 -0
- package/dist/sandbox.d.ts +33 -0
- package/dist/sandbox.js +123 -0
- package/dist/sandbox.js.map +1 -0
- package/dist/scenario.d.ts +212 -0
- package/dist/scenario.js +1079 -0
- package/dist/scenario.js.map +1 -0
- package/dist/scenarios.d.ts +55 -0
- package/dist/scenarios.js +71 -0
- package/dist/scenarios.js.map +1 -0
- package/dist/shared.d.ts +61 -0
- package/dist/shared.js +77 -0
- package/dist/shared.js.map +1 -0
- package/dist/steps.d.ts +76 -0
- package/dist/steps.js +23 -0
- package/dist/steps.js.map +1 -0
- package/dist/version.d.ts +29 -0
- package/dist/version.js +76 -0
- package/dist/version.js.map +1 -0
- package/dist/worker.d.ts +34 -0
- package/dist/worker.js +219 -0
- package/dist/worker.js.map +1 -0
- package/oxlint.json +128 -0
- package/package.json +91 -3
- package/src/approvals.ts +157 -0
- package/src/ast.ts +115 -0
- package/src/call.ts +468 -0
- package/src/client.ts +534 -0
- package/src/config.ts +381 -0
- package/src/credentials.ts +211 -0
- package/src/cucumber.ts +230 -0
- package/src/datafile.ts +409 -0
- package/src/db.ts +197 -0
- package/src/define.ts +285 -0
- package/src/describe.ts +262 -0
- package/src/drift.ts +149 -0
- package/src/env.ts +122 -0
- package/src/errors.ts +202 -0
- package/src/expectations.ts +165 -0
- package/src/fake.ts +156 -0
- package/src/fill.ts +24 -0
- package/src/index.ts +120 -0
- package/src/ledger.ts +419 -0
- package/src/lint.ts +208 -0
- package/src/log.ts +82 -0
- package/src/op.ts +399 -0
- package/src/outline.ts +620 -0
- package/src/paths.ts +32 -0
- package/src/plugin.ts +37 -0
- package/src/policy.ts +139 -0
- package/src/resource.ts +447 -0
- package/src/resources.ts +256 -0
- package/src/run.ts +91 -0
- package/src/sandbox.ts +157 -0
- package/src/scenario.ts +1328 -0
- package/src/scenarios.ts +111 -0
- package/src/shared.ts +116 -0
- package/src/steps.ts +90 -0
- package/src/version.ts +86 -0
- package/src/worker.ts +253 -0
package/src/ledger.ts
ADDED
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { appendFile, mkdir, readFile, truncate } from "node:fs/promises";
|
|
3
|
+
import { join, resolve } from "node:path";
|
|
4
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
5
|
+
import { isDeepStrictEqual } from "node:util";
|
|
6
|
+
import { LEDGER_TABLE as TABLE, lazyPool, pgCode } from "./db.ts";
|
|
7
|
+
import type { Approver, Principal } from "./define.ts";
|
|
8
|
+
import { type ErrorInfo, isFinal, keepCode } from "./errors.ts";
|
|
9
|
+
import { shown, warn } from "./log.ts";
|
|
10
|
+
import type { Effect } from "./op.ts";
|
|
11
|
+
import type { RecordedDecision } from "./policy.ts";
|
|
12
|
+
import type { Run } from "./run.ts";
|
|
13
|
+
import { errorMessage } from "./shared.ts";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The audit record of a run: who started it, every operation call with the policy's
|
|
17
|
+
* decision, every approval, and how it ended. Records are append-only.
|
|
18
|
+
*
|
|
19
|
+
* `id` is deterministic (`<runId>:<type>:<seq>`, `<runId>:<type>:<approvalId>` for an
|
|
20
|
+
* approval requested or decided, `<runId>:approval.refused:<approvalId>:refused:<n>` for
|
|
21
|
+
* the nth message an approval ignored) so that when DBOS replays a run after a restart,
|
|
22
|
+
* the records it writes again have the same ids and the store skips them. `seq` orders a
|
|
23
|
+
* run's records. `v` is the record format's version; `app` is the config's `appName`.
|
|
24
|
+
*/
|
|
25
|
+
export type LedgerRecord = {
|
|
26
|
+
v: 1;
|
|
27
|
+
app: string;
|
|
28
|
+
id: string;
|
|
29
|
+
runId: string;
|
|
30
|
+
seq: number;
|
|
31
|
+
at: number;
|
|
32
|
+
actor: Principal;
|
|
33
|
+
workflow: string;
|
|
34
|
+
/** Set on every record written while a `ctx.all` member runs; absent outside one. */
|
|
35
|
+
group?: LedgerGroup;
|
|
36
|
+
} & (
|
|
37
|
+
| { type: "run.started"; input: unknown }
|
|
38
|
+
| {
|
|
39
|
+
type: "op.called";
|
|
40
|
+
op: string;
|
|
41
|
+
/**
|
|
42
|
+
* The outline node of the call that made it (`OutlineNode.path`): where the step is in the
|
|
43
|
+
* code. Absent in records written before the worker placed calls (ledgers from 0.1).
|
|
44
|
+
*/
|
|
45
|
+
node?: string;
|
|
46
|
+
/**
|
|
47
|
+
* The step's name: the one the call gave (`{ name }`), or the operation's id. Absent in
|
|
48
|
+
* records written before steps had names.
|
|
49
|
+
*/
|
|
50
|
+
name?: string;
|
|
51
|
+
effect: Effect;
|
|
52
|
+
input: unknown;
|
|
53
|
+
/** The policy's decision, with its `reasons` and the policy's `policyVersion` when there are any. */
|
|
54
|
+
decision: RecordedDecision;
|
|
55
|
+
output?: unknown;
|
|
56
|
+
/**
|
|
57
|
+
* Why the call failed: `{ code?, name, message }`, and the vendor's `status`, `vendorCode`
|
|
58
|
+
* and `retryable` from a `DriverError`, or the `data` of one of the runtime's errors.
|
|
59
|
+
*/
|
|
60
|
+
error?: ErrorInfo;
|
|
61
|
+
durationMs: number;
|
|
62
|
+
/** Which try produced the output or the final error, from 1. Idempotent operations are retried. */
|
|
63
|
+
attempt?: number;
|
|
64
|
+
/** The approval that held the call, when its rejection stopped it. */
|
|
65
|
+
approval?: string;
|
|
66
|
+
}
|
|
67
|
+
| {
|
|
68
|
+
type: "approval.requested";
|
|
69
|
+
approval: string;
|
|
70
|
+
/** The outline node of the `ctx.approval` call, or of the held operation's call for a policy request; as for `op.called`. */
|
|
71
|
+
node?: string;
|
|
72
|
+
/** The step's name: the title for a `ctx.approval`, the held call's name for a policy request; as for `op.called`. */
|
|
73
|
+
name?: string;
|
|
74
|
+
title: string;
|
|
75
|
+
approver: Approver;
|
|
76
|
+
requestedBy: "workflow" | "policy";
|
|
77
|
+
/** The operations the approval stands for, by id. */
|
|
78
|
+
covers: string[];
|
|
79
|
+
/** The operation call held, for a policy request. */
|
|
80
|
+
op?: string;
|
|
81
|
+
/** The held call's `seq`, which its `op.called` record has, for a policy request. */
|
|
82
|
+
opSeq?: number;
|
|
83
|
+
}
|
|
84
|
+
/** A message the approval ignored: from someone other than the approver, or not a decision. */
|
|
85
|
+
| { type: "approval.refused"; approval: string; by?: string; reason: string }
|
|
86
|
+
| { type: "approval.decided"; approval: string; decision: "approve" | "reject"; by: string; note?: string }
|
|
87
|
+
/**
|
|
88
|
+
* A `ctx.sleep` began; `until` is when it ends, in ms since the epoch: the time asked for, or
|
|
89
|
+
* for a duration the time the sleep started plus the duration, as the runtime saw it. Written
|
|
90
|
+
* for a time already past too, which waits not at all.
|
|
91
|
+
*/
|
|
92
|
+
| {
|
|
93
|
+
type: "sleep.started";
|
|
94
|
+
node?: string;
|
|
95
|
+
/** The step's name: the one the call gave (`{ name }`), or `Sleep`; as for `op.called`. */
|
|
96
|
+
name?: string;
|
|
97
|
+
until: number;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* A sandbox run seeded its fakes from the scenario named: each `Given` operation called
|
|
101
|
+
* through its fake, with what the fake returned, and each `Given` resource (by its type's id
|
|
102
|
+
* and its import id) put with its fields or removed (`gone`). Follows `run.started`.
|
|
103
|
+
*/
|
|
104
|
+
| {
|
|
105
|
+
type: "scenario.seeded";
|
|
106
|
+
scenario: string;
|
|
107
|
+
seeds: (
|
|
108
|
+
| { op: string; input: unknown; output: unknown }
|
|
109
|
+
| { resource: string; name: string; fields: Record<string, unknown> }
|
|
110
|
+
| { resource: string; name: string; gone: true }
|
|
111
|
+
)[];
|
|
112
|
+
}
|
|
113
|
+
| { type: "run.finished"; output: unknown }
|
|
114
|
+
/** `error` as for `op.called`. */
|
|
115
|
+
| { type: "run.failed"; error: ErrorInfo }
|
|
116
|
+
);
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The `ctx.all` member a record was written in. `id` is `all:<seq>`, the run's next `seq` when
|
|
120
|
+
* the `ctx.all` began, so a replay names it the same; `index` is the member's position, `size`
|
|
121
|
+
* the member count and `node` the `ctx.all` call's outline node.
|
|
122
|
+
*/
|
|
123
|
+
export interface LedgerGroup {
|
|
124
|
+
id: string;
|
|
125
|
+
index: number;
|
|
126
|
+
size: number;
|
|
127
|
+
/** The outline node of the `ctx.all` call; as for `op.called`'s. */
|
|
128
|
+
node?: string;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* The `ctx.all` member a call was made in. Each member runs inside `currentGroup.run`, and a
|
|
133
|
+
* queued call keeps the context it was queued in, so a call made outside the member meanwhile
|
|
134
|
+
* (one the workflow did not await) is not tagged as the member's.
|
|
135
|
+
*/
|
|
136
|
+
export const currentGroup = new AsyncLocalStorage<LedgerGroup>();
|
|
137
|
+
|
|
138
|
+
export interface LedgerStore {
|
|
139
|
+
/**
|
|
140
|
+
* Adds a record, unless one with the same id is already stored. A failed append is tried
|
|
141
|
+
* again, up to three times. A store throws an error with `retryable: false` for a failure
|
|
142
|
+
* that must not be retried because it would only happen again, such as a corrupt file or a
|
|
143
|
+
* record that can't be written as JSON.
|
|
144
|
+
*/
|
|
145
|
+
append(record: LedgerRecord): Promise<void>;
|
|
146
|
+
/** A run's records in `seq` order. */
|
|
147
|
+
read(runId: string): Promise<LedgerRecord[]>;
|
|
148
|
+
/**
|
|
149
|
+
* Releases what the store holds (a connection pool), for good. Called by the one owner of the
|
|
150
|
+
* store: the worker's `stop` and `SanomaClient.close` close the Postgres store each made for a
|
|
151
|
+
* config without a `ledger` (`ownedLedger`); a store a config gives is its own to close.
|
|
152
|
+
*/
|
|
153
|
+
close?(): Promise<void>;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** The store a worker or client runs on, and `close`, which ends it only when it was made for them. */
|
|
157
|
+
export interface OwnedLedger {
|
|
158
|
+
store: LedgerStore;
|
|
159
|
+
close(): Promise<void>;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The config's `ledger`, which stays the config's to close, or `postgresLedger` on its database,
|
|
164
|
+
* made here for the caller, and so the caller's to close: the worker and the client each make
|
|
165
|
+
* their own, and close it with themselves. Two of them on one database share the table, not a pool.
|
|
166
|
+
*/
|
|
167
|
+
export function ownedLedger(config: { ledger?: LedgerStore; databaseUrl: string }): OwnedLedger {
|
|
168
|
+
if (config.ledger) return { store: config.ledger, close: async () => undefined };
|
|
169
|
+
const store = postgresLedger(config.databaseUrl);
|
|
170
|
+
return { store, close: () => store.close() };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const bySeq = (a: LedgerRecord, b: LedgerRecord) => a.seq - b.seq;
|
|
174
|
+
|
|
175
|
+
/** An error that trying again would only repeat. */
|
|
176
|
+
const final = (message: string, cause: unknown) => Object.assign(new Error(message, { cause }), { retryable: false });
|
|
177
|
+
|
|
178
|
+
/** JSON for a record, or an error that names the record when it can't be written as JSON (a BigInt, a cycle). */
|
|
179
|
+
function serialize(record: LedgerRecord): string {
|
|
180
|
+
try {
|
|
181
|
+
return JSON.stringify(record);
|
|
182
|
+
} catch (cause) {
|
|
183
|
+
const op = record.type === "op.called" ? ` (${record.op})` : "";
|
|
184
|
+
throw final(`Ledger record ${record.id}${op} can't be written as JSON: ${errorMessage(cause)}`, cause);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** The line `jsonlLedger` writes for a record: its JSON and a newline. */
|
|
189
|
+
const lineOf = (record: LedgerRecord) => `${serialize(record)}\n`;
|
|
190
|
+
|
|
191
|
+
const COMPARED = ["type", "input", "decision", "error", "output"] as const;
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* The first record with an id stands. Says so when a later one with that id differs where it
|
|
195
|
+
* matters, showing both, so a replay that went another way can be told from a retry. Compared
|
|
196
|
+
* as values, not text: a store may give the kept record's keys back in another order (`jsonb`).
|
|
197
|
+
*/
|
|
198
|
+
function warnIfDifferent(stored: LedgerRecord, line: string) {
|
|
199
|
+
const later = JSON.parse(line) as Record<string, unknown>;
|
|
200
|
+
const differs = COMPARED.flatMap((k) => {
|
|
201
|
+
const kept = (stored as Record<string, unknown>)[k];
|
|
202
|
+
const sent = later[k];
|
|
203
|
+
return isDeepStrictEqual(kept, sent) ? [] : [`${k} (kept ${shown(kept, 200)}, later ${shown(sent, 200)})`];
|
|
204
|
+
});
|
|
205
|
+
if (differs.length) {
|
|
206
|
+
warn(`ledger: kept the first record ${stored.id}; a later write differed in ${differs.join("; ")}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** The file's complete records, and the length of its complete lines when a cut-off line follows them. */
|
|
211
|
+
async function load(path: string): Promise<{ records: LedgerRecord[]; tornAt?: number }> {
|
|
212
|
+
let text: string;
|
|
213
|
+
try {
|
|
214
|
+
text = await readFile(path, "utf8");
|
|
215
|
+
} catch (err) {
|
|
216
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return { records: [] };
|
|
217
|
+
throw err;
|
|
218
|
+
}
|
|
219
|
+
const complete = text.slice(0, text.lastIndexOf("\n") + 1);
|
|
220
|
+
const records: LedgerRecord[] = [];
|
|
221
|
+
complete.split("\n").forEach((line, i) => {
|
|
222
|
+
if (!line.trim()) return;
|
|
223
|
+
try {
|
|
224
|
+
records.push(JSON.parse(line) as LedgerRecord);
|
|
225
|
+
} catch (cause) {
|
|
226
|
+
throw final(`${path}:${i + 1}: corrupt ledger line`, cause);
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
return complete.length < text.length ? { records, tornAt: Buffer.byteLength(complete) } : { records };
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Keeps each run's records in `<dir>/<runId>.jsonl`, one JSON object per line. A run with no
|
|
234
|
+
* file, or a directory not yet created, reads as no records.
|
|
235
|
+
*
|
|
236
|
+
* A line with no newline at the end of a file is a write that was cut off (the process
|
|
237
|
+
* died mid-append): reads ignore it, and the next append removes it before writing.
|
|
238
|
+
* A line that isn't JSON anywhere else is corruption, and reads throw.
|
|
239
|
+
*/
|
|
240
|
+
export function jsonlLedger(dir: string): LedgerStore {
|
|
241
|
+
const root = resolve(dir);
|
|
242
|
+
const file = (runId: string) => join(root, `${encodeURIComponent(runId)}.jsonl`);
|
|
243
|
+
// Appends to one file are serialized, so the id check and the write can't interleave.
|
|
244
|
+
const queues = new Map<string, Promise<unknown>>();
|
|
245
|
+
|
|
246
|
+
return {
|
|
247
|
+
append(record) {
|
|
248
|
+
const path = file(record.runId);
|
|
249
|
+
const next = (queues.get(path) ?? Promise.resolve())
|
|
250
|
+
.catch(() => {})
|
|
251
|
+
.then(async () => {
|
|
252
|
+
const line = lineOf(record);
|
|
253
|
+
const { records, tornAt } = await load(path);
|
|
254
|
+
const stored = records.find((r) => r.id === record.id);
|
|
255
|
+
if (stored) return warnIfDifferent(stored, line);
|
|
256
|
+
await mkdir(root, { recursive: true });
|
|
257
|
+
if (tornAt !== undefined) {
|
|
258
|
+
warn(`ledger: ${path} ended in a cut-off line; removed it before appending ${record.id}`);
|
|
259
|
+
await truncate(path, tornAt);
|
|
260
|
+
}
|
|
261
|
+
await appendFile(path, line);
|
|
262
|
+
});
|
|
263
|
+
queues.set(path, next);
|
|
264
|
+
const forget = () => {
|
|
265
|
+
if (queues.get(path) === next) queues.delete(path);
|
|
266
|
+
};
|
|
267
|
+
next.then(forget, forget);
|
|
268
|
+
return next;
|
|
269
|
+
},
|
|
270
|
+
async read(runId) {
|
|
271
|
+
const path = file(runId);
|
|
272
|
+
await queues.get(path)?.catch(() => {});
|
|
273
|
+
// A missing directory, like a missing file, means nothing has been recorded there yet:
|
|
274
|
+
// `load` reads either as no records. Any other failure is thrown.
|
|
275
|
+
// A case-insensitive file system can map two run ids to one file.
|
|
276
|
+
return (await load(path)).records.filter((r) => r.runId === runId).toSorted(bySeq);
|
|
277
|
+
},
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Postgres's class for a value it cannot store (a string with a NUL in `jsonb`): trying again would only repeat it. */
|
|
282
|
+
const DATA_EXCEPTION = /^22/;
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Keeps records in the `sanoma_ledger` table of the database at `databaseUrl`, beside DBOS's
|
|
286
|
+
* tables: the default store, which the worker, the app and the client share through Postgres.
|
|
287
|
+
* The database and the table are made on first use when they are not there (`prepareDatabase`).
|
|
288
|
+
* The pool is opened on first use and ended by `close()`, after which the store refuses use.
|
|
289
|
+
*/
|
|
290
|
+
export function postgresLedger(databaseUrl: string): LedgerStore & { close(): Promise<void> } {
|
|
291
|
+
const pool = lazyPool(databaseUrl, "ledger");
|
|
292
|
+
|
|
293
|
+
return {
|
|
294
|
+
async append(record) {
|
|
295
|
+
const line = lineOf(record);
|
|
296
|
+
const conn = await pool.db();
|
|
297
|
+
const inserted = await conn
|
|
298
|
+
.query(
|
|
299
|
+
`INSERT INTO ${TABLE} (id, run_id, seq, record) VALUES ($1, $2, $3, $4::jsonb) ON CONFLICT (id) DO NOTHING RETURNING n`,
|
|
300
|
+
[record.id, record.runId, record.seq, line],
|
|
301
|
+
)
|
|
302
|
+
.catch((err: unknown) => {
|
|
303
|
+
if (DATA_EXCEPTION.test(pgCode(err))) {
|
|
304
|
+
throw final(`Ledger record ${record.id} can't be stored: ${errorMessage(err)}`, err);
|
|
305
|
+
}
|
|
306
|
+
throw err;
|
|
307
|
+
});
|
|
308
|
+
if (inserted.rows.length) return;
|
|
309
|
+
// Another write of the id was first: the record it stored stands.
|
|
310
|
+
const { rows } = await conn.query<{ record: LedgerRecord }>(`SELECT record FROM ${TABLE} WHERE id = $1`, [
|
|
311
|
+
record.id,
|
|
312
|
+
]);
|
|
313
|
+
if (rows[0]) warnIfDifferent(rows[0].record, line);
|
|
314
|
+
},
|
|
315
|
+
async read(runId) {
|
|
316
|
+
const { rows } = await (
|
|
317
|
+
await pool.db()
|
|
318
|
+
).query<{ record: LedgerRecord }>(`SELECT record FROM ${TABLE} WHERE run_id = $1 ORDER BY seq, n`, [runId]);
|
|
319
|
+
// In seq order, as the other stores read; two records of one seq in write order.
|
|
320
|
+
return rows.map((r) => r.record);
|
|
321
|
+
},
|
|
322
|
+
close: pool.close,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/** Keeps records in memory, for tests. */
|
|
327
|
+
export function memoryLedger(): LedgerStore {
|
|
328
|
+
const runs = new Map<string, Map<string, LedgerRecord>>();
|
|
329
|
+
return {
|
|
330
|
+
async append(record) {
|
|
331
|
+
const line = serialize(record);
|
|
332
|
+
let run = runs.get(record.runId);
|
|
333
|
+
if (!run) runs.set(record.runId, (run = new Map()));
|
|
334
|
+
const stored = run.get(record.id);
|
|
335
|
+
if (stored) warnIfDifferent(stored, line);
|
|
336
|
+
else run.set(record.id, JSON.parse(line));
|
|
337
|
+
},
|
|
338
|
+
async read(runId) {
|
|
339
|
+
return [...(runs.get(runId)?.values() ?? [])].map((r) => structuredClone(r)).toSorted(bySeq);
|
|
340
|
+
},
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
type Common = "v" | "app" | "id" | "runId" | "seq" | "at" | "actor" | "workflow" | "group";
|
|
345
|
+
|
|
346
|
+
/** What one kind of record says, without what every record carries: `type` and its own fields. */
|
|
347
|
+
export type LedgerBody = LedgerRecord extends infer R ? (R extends LedgerRecord ? Omit<R, Common> : never) : never;
|
|
348
|
+
|
|
349
|
+
/** A record of the run, with the next `seq` unless one is given, tagged with the `ctx.all` member it is written in. */
|
|
350
|
+
export function entry(
|
|
351
|
+
run: Run,
|
|
352
|
+
body: LedgerBody,
|
|
353
|
+
opts: { seq?: number; key?: string; at?: number } = {},
|
|
354
|
+
): LedgerRecord {
|
|
355
|
+
const seq = opts.seq ?? run.seq++;
|
|
356
|
+
const record = {
|
|
357
|
+
v: 1,
|
|
358
|
+
app: run.state.app,
|
|
359
|
+
id: `${run.id}:${body.type}:${opts.key ?? seq}`,
|
|
360
|
+
runId: run.id,
|
|
361
|
+
seq,
|
|
362
|
+
at: opts.at ?? Date.now(),
|
|
363
|
+
actor: run.actor,
|
|
364
|
+
workflow: run.workflow,
|
|
365
|
+
...body,
|
|
366
|
+
} as LedgerRecord;
|
|
367
|
+
const group = currentGroup.getStore();
|
|
368
|
+
if (group) record.group = group;
|
|
369
|
+
return record;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Waits between tries of a failed append. Safe to repeat: append is idempotent by id.
|
|
373
|
+
const RETRY_DELAYS_MS = [50, 200, 800];
|
|
374
|
+
|
|
375
|
+
async function append(run: Run, record: LedgerRecord) {
|
|
376
|
+
for (let i = 0; ; i++) {
|
|
377
|
+
try {
|
|
378
|
+
return await run.state.ledger.append(record);
|
|
379
|
+
} catch (err) {
|
|
380
|
+
const wait = RETRY_DELAYS_MS[i];
|
|
381
|
+
if (wait === undefined || isFinal(err)) throw err;
|
|
382
|
+
warn(`ledger: appending ${record.id} failed (${errorMessage(err)}); trying again in ${wait} ms`);
|
|
383
|
+
await delay(wait);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/*
|
|
389
|
+
* Every ledger write happens outside DBOS steps. When DBOS replays a run after a restart,
|
|
390
|
+
* it re-runs the workflow function and these writes happen again with the same ids, so
|
|
391
|
+
* the store keeps one of each and fills in any the interrupted run never got to write.
|
|
392
|
+
*/
|
|
393
|
+
export async function write(run: Run, record: LedgerRecord) {
|
|
394
|
+
await append(run, record);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* Records a failure. If the ledger fails too, throws both, so neither is lost, with the
|
|
399
|
+
* original's code: the run still fails the way it did.
|
|
400
|
+
*/
|
|
401
|
+
export async function writeFailure(run: Run, record: LedgerRecord, original: unknown) {
|
|
402
|
+
try {
|
|
403
|
+
await append(run, record);
|
|
404
|
+
} catch (ledgerError) {
|
|
405
|
+
throw keepCode(
|
|
406
|
+
new AggregateError(
|
|
407
|
+
[original, ledgerError],
|
|
408
|
+
`${errorMessage(original)} (and the ledger could not record ${record.id}: ${errorMessage(ledgerError)})`,
|
|
409
|
+
{ cause: ledgerError },
|
|
410
|
+
),
|
|
411
|
+
original,
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/** Says that a record was deliberately not written, and why. */
|
|
417
|
+
export function skipped(record: LedgerRecord, why: string) {
|
|
418
|
+
warn(`did not write ${record.type} ${record.id}: ${why}`);
|
|
419
|
+
}
|
package/src/lint.ts
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { basename, dirname, join } from "node:path";
|
|
2
|
+
import { byPosition, childrenOf, type LintProblem, type Node, parsed, type SourceFile } from "./ast.ts";
|
|
3
|
+
import { readDataFile } from "./datafile.ts";
|
|
4
|
+
import { constsOf, outlineBody, workflowLiterals } from "./outline.ts";
|
|
5
|
+
import { inside, nearestDir, RELATIVE } from "./paths.ts";
|
|
6
|
+
// From src/ and from dist/ alike, the package's own oxlint.json: the one list of allowed names.
|
|
7
|
+
import oxlint from "../oxlint.json" with { type: "json" };
|
|
8
|
+
|
|
9
|
+
export type { LintProblem } from "./ast.ts";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* A workflow must call vendors through `ctx`, so the policy sees every call. These are the
|
|
13
|
+
* import rules oxlint's built-in rules cannot express; the clock, randomness, the network
|
|
14
|
+
* and the environment are oxlint's (the package's `oxlint.json`).
|
|
15
|
+
*/
|
|
16
|
+
const THROUGH_CTX = "a workflow must call vendors through ctx, so the policy sees every call";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The packages the author surface is imported from, each allowing the same names: `sanoma`
|
|
20
|
+
* re-exports what `@sanoma-ai/workflows` allows (and `z`). Adding one here accepts it.
|
|
21
|
+
*/
|
|
22
|
+
const AUTHOR_PACKAGES: readonly string[] = ["@sanoma-ai/workflows", "sanoma"];
|
|
23
|
+
|
|
24
|
+
/** The packages a workflow may import from at all: the author packages, connectors and zod. */
|
|
25
|
+
const ALLOWED_PACKAGE = new RegExp(`^(${[...AUTHOR_PACKAGES, "@sanoma-ai/connector-[a-z0-9-]+", "zod"].join("|")})$`);
|
|
26
|
+
|
|
27
|
+
/** Packages refused by name, with the reason. */
|
|
28
|
+
const REFUSED_PACKAGES: [RegExp, string][] = [
|
|
29
|
+
[/^@sanoma-ai\/testing(\/|$)/, `fakes and test helpers belong in tests; ${THROUGH_CTX}`],
|
|
30
|
+
[/^@sanoma-ai\/app(\/|$)/, "the app starts runs and decides approvals, so a workflow could approve itself"],
|
|
31
|
+
[/^@sanoma-ai\/connector-[^/]+\/(fake|driver)(\/|$)/, `${THROUGH_CTX}; import the connector itself`],
|
|
32
|
+
[/^sanoma\//, "`sanoma` has no entry points for a workflow: import its names from `sanoma` itself"],
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* What a workflow or policy may import from an author package, besides types: the one list,
|
|
37
|
+
* `allowImportNames` of oxlint.json's `@sanoma-ai/workflows` entry (`sanoma`'s generated
|
|
38
|
+
* `oxlint.json` derives its own entry from it). Everything else is
|
|
39
|
+
* refused: the client, the worker and the app could start runs or approve the run's own
|
|
40
|
+
* approvals, a ledger store could forge the audit record (a store keeps the first record per
|
|
41
|
+
* id), and config helpers can read credentials.
|
|
42
|
+
*/
|
|
43
|
+
const WORKFLOW_IMPORTS: readonly string[] =
|
|
44
|
+
restrictedImports(oxlint).find((p) => p.name === "@sanoma-ai/workflows")?.allowImportNames ?? [];
|
|
45
|
+
|
|
46
|
+
const ALLOWED_NAMES = new Set(WORKFLOW_IMPORTS);
|
|
47
|
+
|
|
48
|
+
const NOT_ALLOWED =
|
|
49
|
+
`a workflow or policy imports only ${WORKFLOW_IMPORTS.join(", ")} and types from ` +
|
|
50
|
+
`${AUTHOR_PACKAGES.join(" or ")}: the rest could start runs, approve its own approvals, forge the ledger or read credentials`;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Error classes a run must not test with `instanceof`: on a replay DBOS rethrows a serialized
|
|
54
|
+
* copy, which is no instance of them, so a branch on it goes another way than the first time
|
|
55
|
+
* and the run's steps fall out of step. Read the code with `errorCode(err)` instead.
|
|
56
|
+
*/
|
|
57
|
+
const ERROR_CLASSES = new Set(["DriverError", "SanomaError", "PolicyDeniedError", "RejectedError"]);
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Checks a workflow or policy file. Imports allowed: the names in `WORKFLOW_IMPORTS` and any
|
|
61
|
+
* type from `@sanoma-ai/workflows` or `sanoma`, `@sanoma-ai/connector-<vendor>`, `zod`, and relative files. With
|
|
62
|
+
* `filename`, a relative import must stay inside the file's nearest `workflows/` or `policies/`
|
|
63
|
+
* directory, or its own directory when it has neither. No `instanceof` against the runtime's
|
|
64
|
+
* error classes: read `errorCode(err)`.
|
|
65
|
+
*
|
|
66
|
+
* In each workflow's `run` (the literal `defineWorkflow` is called with), it reports what the
|
|
67
|
+
* outline cannot draw, as the worker refuses to start with (`outlineBody`): `ctx` used other
|
|
68
|
+
* than called directly (passed on, aliased, destructured), a function defined in `run` that uses
|
|
69
|
+
* `ctx` where the outline does not read it, `ctx` in a `finally`, a step name that is no literal
|
|
70
|
+
* and two calls with one step name, and in `run` and the callbacks it reads, a `return` before
|
|
71
|
+
* the end, a `throw` outside a `catch`, a `break` out of a loop and `continue`. Each message
|
|
72
|
+
* names what to write instead.
|
|
73
|
+
*
|
|
74
|
+
* It guards against accidental non-determinism and accidental ways around the policy; it is
|
|
75
|
+
* not a sandbox, and code written to get around it can.
|
|
76
|
+
*/
|
|
77
|
+
export function lintWorkflow(source: string, filename?: string): LintProblem[];
|
|
78
|
+
/** The same, over a file `readSource` read: the reading `describeConfig` shares with the outline. */
|
|
79
|
+
export function lintWorkflow(file: SourceFile): LintProblem[];
|
|
80
|
+
export function lintWorkflow(input: string | SourceFile, path?: string): LintProblem[] {
|
|
81
|
+
const filename = typeof input === "string" ? path : input.file;
|
|
82
|
+
const read = typeof input === "string" ? parsed(input, filename ?? "workflow.ts") : input;
|
|
83
|
+
const { program, at } = read;
|
|
84
|
+
const problems = [...read.problems];
|
|
85
|
+
const root = filename === undefined ? undefined : nearestDir(filename, ["workflows", "policies"]);
|
|
86
|
+
|
|
87
|
+
const checkSource = (node: Node) => {
|
|
88
|
+
const spec: string = node.source.value;
|
|
89
|
+
const refusal = whyRefused(spec, filename, root);
|
|
90
|
+
if (refusal) {
|
|
91
|
+
problems.push({ ...at(node.source.start), message: `import "${spec}" is not allowed in a workflow: ${refusal}` });
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
if (!AUTHOR_PACKAGES.includes(spec)) return;
|
|
95
|
+
if (node.type === "ExportAllDeclaration") {
|
|
96
|
+
problems.push({
|
|
97
|
+
...at(node.start),
|
|
98
|
+
message: `export * from "${spec}" is not allowed in a workflow: it would re-export what a workflow may not import`,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
for (const s of node.specifiers ?? []) {
|
|
102
|
+
if (s.type === "ImportNamespaceSpecifier") {
|
|
103
|
+
problems.push({
|
|
104
|
+
...at(s.start),
|
|
105
|
+
message: `import * from "${spec}" is not allowed in a workflow: import the names you use, so the rest stays out`,
|
|
106
|
+
});
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
// `import type { X }`, `import { type X }` and `export type { X }` bring in no values.
|
|
110
|
+
if (
|
|
111
|
+
node.importKind === "type" ||
|
|
112
|
+
node.exportKind === "type" ||
|
|
113
|
+
s.importKind === "type" ||
|
|
114
|
+
s.exportKind === "type"
|
|
115
|
+
) {
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
// What the module exports under: `imported` for an import, `local` for a re-export.
|
|
119
|
+
const named = s.imported ?? s.local;
|
|
120
|
+
const imported: string = s.type === "ImportDefaultSpecifier" ? "default" : (named?.name ?? named?.value);
|
|
121
|
+
if (!ALLOWED_NAMES.has(imported)) {
|
|
122
|
+
problems.push({ ...at(s.start), message: `${imported} is not allowed in a workflow: ${NOT_ALLOWED}` });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
const visit = (node: Node) => {
|
|
128
|
+
if (!node) return;
|
|
129
|
+
if (
|
|
130
|
+
(node.type === "ImportDeclaration" ||
|
|
131
|
+
node.type === "ExportNamedDeclaration" ||
|
|
132
|
+
node.type === "ExportAllDeclaration") &&
|
|
133
|
+
node.source
|
|
134
|
+
) {
|
|
135
|
+
checkSource(node);
|
|
136
|
+
} else if (node.type === "ImportExpression") {
|
|
137
|
+
problems.push({ ...at(node.start), message: "dynamic import is not allowed in a workflow" });
|
|
138
|
+
} else if (node.type === "BinaryExpression" && node.operator === "instanceof") {
|
|
139
|
+
checkInstanceof(node);
|
|
140
|
+
}
|
|
141
|
+
for (const child of childrenOf(node)) visit(child);
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
// A workflow imports none of these classes, so they are refused by the name they have.
|
|
145
|
+
const checkInstanceof = (node: Node) => {
|
|
146
|
+
const right = node.right;
|
|
147
|
+
const name: string | undefined =
|
|
148
|
+
right?.type === "Identifier"
|
|
149
|
+
? right.name
|
|
150
|
+
: right?.type === "MemberExpression" && !right.computed
|
|
151
|
+
? right.property?.name
|
|
152
|
+
: undefined;
|
|
153
|
+
if (name !== undefined && ERROR_CLASSES.has(name)) {
|
|
154
|
+
problems.push({
|
|
155
|
+
...at(node.start),
|
|
156
|
+
message:
|
|
157
|
+
`instanceof ${name} is not allowed in a workflow: on a replay DBOS rethrows a copy of the error, ` +
|
|
158
|
+
"which is no instance of it, so the run would take another branch; read `errorCode(err)`",
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
visit(program);
|
|
164
|
+
// Each workflow's `run` as the worker reads it when it starts: what the outline cannot draw,
|
|
165
|
+
// the worker refuses to start with. One reading, so the lint and the worker agree.
|
|
166
|
+
const outlined = { consts: constsOf(program), source: typeof input === "string" ? input : input.source };
|
|
167
|
+
for (const { run } of workflowLiterals(program))
|
|
168
|
+
for (const { start, message } of outlineBody(run, outlined).problems) problems.push({ ...at(start), message });
|
|
169
|
+
return byPosition(problems);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Checks a data file under `resources/`: only named imports (a connector's constructors from its
|
|
174
|
+
* `…/resources` entry, and resources from other data files inside the same `resources/`
|
|
175
|
+
* directory), `export const <name> = <vendor>.<type>({ … })` with literal fields and names of
|
|
176
|
+
* declared resources, and `export default [ … ]`. Each message says what to write instead.
|
|
177
|
+
* It is the subset `readResources` reads (`readDataFile`), without the config: whether the
|
|
178
|
+
* connector, the type and the values are known is the reader's to say.
|
|
179
|
+
*/
|
|
180
|
+
export function lintResources(source: string, filename: string): LintProblem[] {
|
|
181
|
+
const { program, problems, at } = parsed(source, filename);
|
|
182
|
+
for (const p of readDataFile(program, filename).problems) problems.push({ ...at(p.start), message: p.message });
|
|
183
|
+
return byPosition(problems);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** The `paths` of oxlint.json's `no-restricted-imports` rule. */
|
|
187
|
+
function restrictedImports(config: typeof oxlint): RestrictedPath[] {
|
|
188
|
+
// JSON types the rule's [severity, options] as an array of either.
|
|
189
|
+
return config.overrides.flatMap((o) =>
|
|
190
|
+
"no-restricted-imports" in o.rules
|
|
191
|
+
? (o.rules["no-restricted-imports"] as [string, { paths: RestrictedPath[] }])[1].paths
|
|
192
|
+
: [],
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
type RestrictedPath = { name: string; allowImportNames?: string[] };
|
|
197
|
+
|
|
198
|
+
function whyRefused(spec: string, filename: string | undefined, root: string | undefined): string | undefined {
|
|
199
|
+
if (RELATIVE.test(spec)) {
|
|
200
|
+
if (filename === undefined || root === undefined || inside(root, join(dirname(filename), spec))) return undefined;
|
|
201
|
+
const where =
|
|
202
|
+
basename(root) === "workflows" || basename(root) === "policies" ? `${basename(root)}/` : "this directory";
|
|
203
|
+
return `it reaches outside ${where}, toward the config and its drivers; ${THROUGH_CTX}`;
|
|
204
|
+
}
|
|
205
|
+
for (const [pattern, reason] of REFUSED_PACKAGES) if (pattern.test(spec)) return reason;
|
|
206
|
+
if (ALLOWED_PACKAGE.test(spec)) return undefined;
|
|
207
|
+
return "import only from `sanoma` (or `@sanoma-ai/workflows`), a `@sanoma-ai/connector-*` package, zod, or a file beside this one";
|
|
208
|
+
}
|