@anteros/core 0.0.41 → 0.0.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/database/audit.ts +9 -59
- package/database/collection.ts +3 -1
- package/database/deleteLog.ts +67 -3
- package/database/file.ts +4 -3
- package/database/mongodbadapter.ts +103 -42
- package/database/replication.ts +110 -32
- package/database/ttl.ts +68 -0
- package/database/workflow.ts +466 -240
- package/index.ts +6 -0
- package/lib/agent.ts +1327 -0
- package/lib/agentMemory.ts +586 -0
- package/lib/agents.ts +157 -0
- package/lib/ai.ts +0 -0
- package/lib/api.ts +246 -0
- package/lib/attachment.ts +351 -0
- package/lib/auditResult.ts +13 -0
- package/lib/define.ts +43 -1
- package/lib/endpoints.ts +80 -0
- package/lib/files.ts +2 -1
- package/lib/jsonSchema.ts +183 -0
- package/lib/lifecycle.ts +7 -1
- package/lib/load.ts +17 -0
- package/lib/mcp.ts +3 -2
- package/lib/messages.ts +299 -0
- package/lib/providers.ts +867 -0
- package/lib/routes.ts +14 -1
- package/lib/scriptWorker.ts +4 -1
- package/lib/scripts.ts +6 -1
- package/lib/sockets.ts +3 -1
- package/lib/workflow.ts +209 -37
- package/lib/zip.ts +109 -0
- package/package.json +1 -1
- package/server/access.ts +96 -0
- package/server/agents.ts +451 -0
- package/server/api.ts +14 -72
- package/server/boot.ts +11 -0
- package/server/hono.ts +8 -3
- package/server/mcp.ts +17 -63
- package/server/routes.ts +19 -24
- package/tests/agent-memory.test.ts +617 -0
- package/tests/agent-working-memory.test.ts +402 -0
- package/tests/agent.test.ts +1239 -0
- package/tests/agents-api.test.ts +593 -0
- package/tests/api.test.ts +51 -0
- package/tests/attachment.test.ts +241 -0
- package/tests/audit-operations.test.ts +1 -1
- package/tests/audit-redact.test.ts +29 -3
- package/tests/audit-run-action.test.ts +1 -1
- package/tests/endpoints.test.ts +80 -0
- package/tests/fixtures/agent-tenant/agents/support.agent.ts +9 -0
- package/tests/fixtures/agent-tenant/agents/weather.agent.ts +21 -0
- package/tests/fixtures/agents-api/agents/assistant.agent.ts +46 -0
- package/tests/fixtures/agents-api/agents/plain.agent.ts +23 -0
- package/tests/fixtures/agents-api/agents/private.agent.ts +15 -0
- package/tests/fixtures/agents-api/collections/notes.model.ts +26 -0
- package/tests/fixtures/agents-api/routes/probe.route.ts +21 -0
- package/tests/fixtures/agents-api/services/probe.service.ts +19 -0
- package/tests/fixtures/attachment/data.csv +2 -0
- package/tests/fixtures/fake-provider.ts +128 -0
- package/tests/fixtures/replication-meta/agents/chat.agent.ts +16 -0
- package/tests/fixtures/replication-meta/collections/counters.model.ts +13 -0
- package/tests/fixtures/replication-meta/collections/orders.model.ts +9 -0
- package/tests/fixtures/replication-meta/workflows/run.workflow.ts +8 -0
- package/tests/fixtures/vars-tenant/vars/private.var.ts +4 -1
- package/tests/fixtures/workflow-engine/workflows/failing.workflow.ts +34 -0
- package/tests/fixtures/workflow-engine/workflows/flaky.workflow.ts +23 -0
- package/tests/fixtures/workflow-engine/workflows/hang.workflow.ts +17 -0
- package/tests/fixtures/workflow-engine/workflows/simple.workflow.ts +19 -0
- package/tests/fixtures/workflow-engine/workflows/slow.workflow.ts +23 -0
- package/tests/fixtures/workflow-engine/workflows/versioned.workflow.ts +19 -0
- package/tests/fixtures/workflow-tenant/workflows/broken.workflow.ts +3 -0
- package/tests/fixtures/workflow-tenant/workflows/disabled.workflow.ts +8 -0
- package/tests/fixtures/workflow-tenant/workflows/invoice.workflow.ts +21 -0
- package/tests/fixtures/workflow-tenant/workflows/no-id.workflow.ts +7 -0
- package/tests/fixtures/workflow-tenant/workflows/order-duplicate.workflow.ts +13 -0
- package/tests/fixtures/workflow-tenant/workflows/order.workflow.ts +17 -0
- package/tests/hono.test.ts +6 -3
- package/tests/locks.test.ts +120 -0
- package/tests/mcp.test.ts +11 -11
- package/tests/messages.test.ts +258 -0
- package/tests/replication-control.test.ts +2 -2
- package/tests/replication-meta.test.ts +243 -0
- package/tests/upload.test.ts +44 -12
- package/tests/vars-api.test.ts +1 -1
- package/tests/vars-replication.test.ts +3 -2
- package/tests/workflow-engine.test.ts +340 -0
- package/tests/workflow-retention.test.ts +103 -0
- package/tests/workflow.test.ts +186 -0
- package/types/agent.d.ts +837 -0
- package/types/config.d.ts +16 -0
- package/types/file.d.ts +7 -7
- package/types/hook.d.ts +6 -0
- package/types/lifecycle.d.ts +6 -0
- package/types/mcp.d.ts +11 -1
- package/types/replication.d.ts +29 -0
- package/types/route.d.ts +7 -1
- package/types/scripts.d.ts +6 -0
- package/types/service.d.ts +7 -1
- package/types/tenant.d.ts +13 -0
- package/types/vars.d.ts +2 -2
- package/types/websocket.d.ts +6 -0
- package/types/workflow.d.ts +50 -3
package/tests/vars-api.test.ts
CHANGED
|
@@ -34,7 +34,7 @@ afterAll(async () => {
|
|
|
34
34
|
});
|
|
35
35
|
|
|
36
36
|
const post = (action: string, body: any) =>
|
|
37
|
-
fetch(`${url}/
|
|
37
|
+
fetch(`${url}/api/${TENANT}/vars/${action}`, {
|
|
38
38
|
method: "POST",
|
|
39
39
|
headers: { "Content-Type": "application/json" },
|
|
40
40
|
body: JSON.stringify(body),
|
|
@@ -85,8 +85,9 @@ describe("vars replication", () => {
|
|
|
85
85
|
expect(dest?.value).toBe("SCOPED-1");
|
|
86
86
|
});
|
|
87
87
|
|
|
88
|
-
it("does NOT replicate namespaces that
|
|
89
|
-
// `private.var.ts`
|
|
88
|
+
it("does NOT replicate namespaces that opt out (`enabled: false`)", async () => {
|
|
89
|
+
// `private.var.ts` declares `replication: { enabled: false }` — it must
|
|
90
|
+
// never reach the destination.
|
|
90
91
|
await rest.vars.set("private", "secret", "top-secret");
|
|
91
92
|
await replicateTenant(TEST_TENANT);
|
|
92
93
|
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll, afterAll, beforeEach } from "bun:test";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import { formatConfig } from "../server/config";
|
|
4
|
+
import { syncTenants } from "../database/tenant";
|
|
5
|
+
import { syncWorkflows, ensureWorkflowIndexes, getWorkflow } from "../lib/workflow";
|
|
6
|
+
import { useRest } from "../database/rest";
|
|
7
|
+
import type { WorkflowRun } from "../types/workflow";
|
|
8
|
+
|
|
9
|
+
const TENANT = "wf-engine";
|
|
10
|
+
const DIR = "packages/core/tests/fixtures/workflow-engine";
|
|
11
|
+
|
|
12
|
+
let rest: InstanceType<typeof useRest>;
|
|
13
|
+
/** Shared, in-process state written by the fixture steps */
|
|
14
|
+
const state: { log: string[]; attempts: Record<string, number> } = ((globalThis as any).__wfEngine ??= {
|
|
15
|
+
log: [],
|
|
16
|
+
attempts: {},
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
async function runsOf(workflowId: string): Promise<WorkflowRun[]> {
|
|
20
|
+
const col = rest.db.collection("_workflows_");
|
|
21
|
+
return (await col.find({ workflowId }).sort({ createdAt: -1 }).toArray()) as unknown as WorkflowRun[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** The run currently being executed (its `_id` is only known from the database) */
|
|
25
|
+
async function currentRun(workflowId: string): Promise<any> {
|
|
26
|
+
for (let i = 0; i < 80; i++) {
|
|
27
|
+
const [run] = await runsOf(workflowId);
|
|
28
|
+
if (run && run.status === "running") return run;
|
|
29
|
+
await Bun.sleep(25);
|
|
30
|
+
}
|
|
31
|
+
throw new Error(`no running '${workflowId}' run found`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Insert a run document directly, to set up a state the engine must recover from */
|
|
35
|
+
async function insertRun(doc: Partial<WorkflowRun> & { _id: string; workflowId: string }): Promise<void> {
|
|
36
|
+
await rest.db.collection("_workflows_").insertOne({
|
|
37
|
+
tenant_id: TENANT,
|
|
38
|
+
status: "failed",
|
|
39
|
+
progress: 0,
|
|
40
|
+
data: {},
|
|
41
|
+
currentStep: 0,
|
|
42
|
+
totalExecuted: 0,
|
|
43
|
+
totalSkipped: 0,
|
|
44
|
+
steps: [],
|
|
45
|
+
compensations: [],
|
|
46
|
+
createdAt: new Date(),
|
|
47
|
+
updatedAt: new Date(),
|
|
48
|
+
...doc,
|
|
49
|
+
} as any);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
beforeAll(async () => {
|
|
53
|
+
formatConfig({
|
|
54
|
+
server: { port: 4000 },
|
|
55
|
+
tenants: [{ id: TENANT, dir: DIR, database: { uri: "mongodb://localhost:27017/_WF_ENGINE" } }],
|
|
56
|
+
});
|
|
57
|
+
await syncTenants();
|
|
58
|
+
await syncWorkflows();
|
|
59
|
+
await ensureWorkflowIndexes();
|
|
60
|
+
rest = new useRest({ tenant_id: TENANT, internal: true });
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
beforeEach(() => {
|
|
64
|
+
state.log.length = 0;
|
|
65
|
+
for (const key of Object.keys(state.attempts)) delete state.attempts[key];
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
afterAll(async () => {
|
|
69
|
+
try { await rest.db.collection("_workflows_").drop(); } catch (_) {}
|
|
70
|
+
try { await rest.db.collection("_locks_").drop(); } catch (_) {}
|
|
71
|
+
delete (globalThis as any).__wfEngine;
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
describe("workflow locks", () => {
|
|
75
|
+
it("runs a step once when two nodes resume the same run", async () => {
|
|
76
|
+
await insertRun({
|
|
77
|
+
_id: "lock-1",
|
|
78
|
+
workflowId: "simple",
|
|
79
|
+
status: "paused",
|
|
80
|
+
steps: [{ stepId: "one", status: "pending", startedAt: new Date() } as any],
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const [a, b] = await Promise.allSettled([
|
|
84
|
+
rest.workflow.resume("lock-1"),
|
|
85
|
+
rest.workflow.resume("lock-1"),
|
|
86
|
+
]);
|
|
87
|
+
const outcomes = [a, b].map((r) => r.status);
|
|
88
|
+
const errors = [a, b].map((r: any) => r.reason?.code).filter(Boolean) as string[];
|
|
89
|
+
|
|
90
|
+
// One node wins; the other is either locked out or finds the run finished
|
|
91
|
+
expect(outcomes).toContain("fulfilled");
|
|
92
|
+
for (const code of errors) expect(["RUN_LOCKED", "RUN_TERMINAL"]).toContain(code);
|
|
93
|
+
|
|
94
|
+
// …and above all: the step was NOT executed twice
|
|
95
|
+
expect(state.log.filter((l) => l === "simple:one").length).toBe(1);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("releases the lock when the run finishes (a later resume is not locked)", async () => {
|
|
99
|
+
const run = await rest.workflow.run("simple", { echo: "release" });
|
|
100
|
+
const locks = await rest.db.collection("_locks_").countDocuments({ name: `workflow:${run._id}` });
|
|
101
|
+
expect(locks).toBe(0);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it("refuses a resume while another node holds the run", async () => {
|
|
105
|
+
await insertRun({
|
|
106
|
+
_id: "lock-2",
|
|
107
|
+
workflowId: "simple",
|
|
108
|
+
status: "paused",
|
|
109
|
+
steps: [{ stepId: "one", status: "pending", startedAt: new Date() } as any],
|
|
110
|
+
});
|
|
111
|
+
// Simulate another node holding the lock (not expired)
|
|
112
|
+
await rest.db.collection("_locks_").insertOne({
|
|
113
|
+
_id: `${TENANT}:workflow:lock-2` as any,
|
|
114
|
+
tid: TENANT,
|
|
115
|
+
name: "workflow:lock-2",
|
|
116
|
+
acquiredAt: Date.now(),
|
|
117
|
+
expiresAt: Date.now() + 60_000,
|
|
118
|
+
} as any);
|
|
119
|
+
|
|
120
|
+
await expect(rest.workflow.resume("lock-2")).rejects.toThrow(/another node/);
|
|
121
|
+
await rest.db.collection("_locks_").deleteOne({ name: "workflow:lock-2" });
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
describe("workflow pause & cancel", () => {
|
|
126
|
+
it("pauses an in-flight run at the next step boundary", async () => {
|
|
127
|
+
const pending = rest.workflow.run("slow", { ms: 600 });
|
|
128
|
+
const running = await currentRun("slow");
|
|
129
|
+
|
|
130
|
+
expect(await rest.workflow.pause(running._id)).toBe(true);
|
|
131
|
+
const run: any = await pending;
|
|
132
|
+
|
|
133
|
+
expect(run.status).toBe("paused");
|
|
134
|
+
expect(state.log).toEqual(["slow:fast", "slow:slow:start", "slow:slow:end"]);
|
|
135
|
+
expect(run.steps[2].status).toBe("pending"); // the last step never ran
|
|
136
|
+
}, 20_000);
|
|
137
|
+
|
|
138
|
+
it("resumes a paused run from the step it stopped at", async () => {
|
|
139
|
+
const [paused] = await runsOf("slow");
|
|
140
|
+
state.log.length = 0;
|
|
141
|
+
|
|
142
|
+
const run = await rest.workflow.resume(paused!._id, { ms: 10 });
|
|
143
|
+
expect(run.status).toBe("completed");
|
|
144
|
+
expect(state.log).toEqual(["slow:last"]);
|
|
145
|
+
}, 20_000);
|
|
146
|
+
|
|
147
|
+
it("cancels an in-flight run permanently", async () => {
|
|
148
|
+
const pending = rest.workflow.run("slow", { ms: 600 });
|
|
149
|
+
const running = await currentRun("slow");
|
|
150
|
+
|
|
151
|
+
expect(await rest.workflow.cancel(running._id)).toBe(true);
|
|
152
|
+
const run: any = await pending;
|
|
153
|
+
|
|
154
|
+
expect(run.status).toBe("cancelled");
|
|
155
|
+
expect(state.log).not.toContain("slow:last");
|
|
156
|
+
await expect(rest.workflow.resume(run._id)).rejects.toThrow(/cancelled|not applicable/);
|
|
157
|
+
}, 20_000);
|
|
158
|
+
|
|
159
|
+
it("does not cancel a run that already completed", async () => {
|
|
160
|
+
const done = await rest.workflow.run("simple", { echo: "terminal" });
|
|
161
|
+
expect(await rest.workflow.cancel(done._id)).toBe(false);
|
|
162
|
+
|
|
163
|
+
const stored: any = await rest.workflow.getRun(done._id);
|
|
164
|
+
expect(stored.status).toBe("completed");
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
describe("workflow step retries & timeout", () => {
|
|
169
|
+
it("retries a transient failure instead of rolling the saga back", async () => {
|
|
170
|
+
const run = await rest.workflow.run("flaky", { failUntil: 2 });
|
|
171
|
+
expect(run.status).toBe("completed");
|
|
172
|
+
expect(state.log).toEqual(["flaky:attempt:1", "flaky:attempt:2", "flaky:attempt:3"]);
|
|
173
|
+
expect(run.steps[0]!.output).toEqual({ attempts: 3 });
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it("gives up after `retries` and fails the run", async () => {
|
|
177
|
+
await expect(rest.workflow.run("flaky", { failUntil: 5 })).rejects.toThrow(/transient failure/);
|
|
178
|
+
const [run] = await runsOf("flaky");
|
|
179
|
+
expect(run!.status).toBe("failed");
|
|
180
|
+
expect(state.attempts.unstable).toBe(3); // 1 attempt + 2 retries
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("aborts a step that exceeds its timeout", async () => {
|
|
184
|
+
await expect(rest.workflow.run("hang", { ms: 1000 })).rejects.toThrow(/timed out/);
|
|
185
|
+
const [run] = await runsOf("hang");
|
|
186
|
+
expect(run!.status).toBe("failed");
|
|
187
|
+
expect(run!.error?.code).toBe("WORKFLOW_STEP_TIMEOUT");
|
|
188
|
+
}, 15_000);
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
describe("workflow compensations", () => {
|
|
192
|
+
it("compensates the completed steps and marks the run `compensated`", async () => {
|
|
193
|
+
await expect(rest.workflow.run("failing", {})).rejects.toThrow(/boom-b/);
|
|
194
|
+
const [run] = await runsOf("failing");
|
|
195
|
+
|
|
196
|
+
expect(run!.status).toBe("compensated");
|
|
197
|
+
expect(run!.steps[2]!.status).toBe("pending"); // never reached
|
|
198
|
+
expect(state.log).toEqual([
|
|
199
|
+
"failing:a:rest=client",
|
|
200
|
+
"simple:one", // the sub-workflow run from the step
|
|
201
|
+
"failing:a:sub-workflow-done",
|
|
202
|
+
"failing:b",
|
|
203
|
+
"failing:undo-a:1",
|
|
204
|
+
]);
|
|
205
|
+
}, 15_000);
|
|
206
|
+
|
|
207
|
+
it("refuses to resume a compensated run (never replays the steps)", async () => {
|
|
208
|
+
const [run] = await runsOf("failing");
|
|
209
|
+
state.log.length = 0;
|
|
210
|
+
|
|
211
|
+
await expect(rest.workflow.resume(run!._id)).rejects.toThrow(/compensated/);
|
|
212
|
+
expect(state.log).toEqual([]);
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
it("never replays a compensation through resumeAll", async () => {
|
|
216
|
+
const [before] = await runsOf("failing");
|
|
217
|
+
state.log.length = 0;
|
|
218
|
+
|
|
219
|
+
await rest.workflow.resumeAll();
|
|
220
|
+
|
|
221
|
+
// The compensated run is left alone and its compensation is not replayed
|
|
222
|
+
const [after] = await runsOf("failing");
|
|
223
|
+
expect(after!.status).toBe("compensated");
|
|
224
|
+
expect(state.log.filter((l) => l.startsWith("failing:undo-a"))).toEqual([]);
|
|
225
|
+
expect(before!._id).toBe(after!._id);
|
|
226
|
+
}, 15_000);
|
|
227
|
+
|
|
228
|
+
it("finishes an interrupted rollback without replaying what already succeeded", async () => {
|
|
229
|
+
await insertRun({
|
|
230
|
+
_id: "rollback-1",
|
|
231
|
+
workflowId: "failing",
|
|
232
|
+
status: "failed",
|
|
233
|
+
steps: [
|
|
234
|
+
{ stepId: "a", status: "completed", output: { a: 1 }, startedAt: new Date() } as any,
|
|
235
|
+
{ stepId: "b", status: "failed", error: { message: "boom-b" }, startedAt: new Date() } as any,
|
|
236
|
+
{ stepId: "c", status: "pending", startedAt: new Date() } as any,
|
|
237
|
+
],
|
|
238
|
+
// `undo-a` failed the first time: the rollback must be finished
|
|
239
|
+
compensations: [{ stepId: "undo-a", status: "failed", error: { message: "network" }, startedAt: new Date() } as any],
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
const run: any = await rest.workflow.resume("rollback-1");
|
|
243
|
+
expect(run.status).toBe("compensated");
|
|
244
|
+
expect(run.steps[1].status).toBe("failed"); // never continued forward
|
|
245
|
+
expect(state.log.length).toBe(1);
|
|
246
|
+
expect(state.log[0]).toStartWith("failing:undo-a");
|
|
247
|
+
expect(run.compensations.length).toBe(2); // the failed one + the successful retry
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
it("replays nothing when the rollback already succeeded", async () => {
|
|
251
|
+
await insertRun({
|
|
252
|
+
_id: "rollback-2",
|
|
253
|
+
workflowId: "failing",
|
|
254
|
+
status: "failed",
|
|
255
|
+
steps: [
|
|
256
|
+
{ stepId: "a", status: "completed", output: { a: 1 }, startedAt: new Date() } as any,
|
|
257
|
+
{ stepId: "b", status: "failed", error: { message: "boom-b" }, startedAt: new Date() } as any,
|
|
258
|
+
],
|
|
259
|
+
compensations: [{ stepId: "undo-a", status: "completed", startedAt: new Date() } as any],
|
|
260
|
+
});
|
|
261
|
+
state.log.length = 0;
|
|
262
|
+
|
|
263
|
+
const run: any = await rest.workflow.resume("rollback-2");
|
|
264
|
+
expect(run.status).toBe("compensated");
|
|
265
|
+
expect(state.log).toEqual([]); // the successful compensation was not replayed
|
|
266
|
+
});
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
describe("workflow run ownership", () => {
|
|
270
|
+
it("stops when another node takes the run over", async () => {
|
|
271
|
+
const pending = rest.workflow.run("slow", { ms: 500 });
|
|
272
|
+
const running = await currentRun("slow");
|
|
273
|
+
|
|
274
|
+
// Simulate a takeover: another process owns the run lock now
|
|
275
|
+
await rest.db.collection("_locks_").updateOne(
|
|
276
|
+
{ _id: `${TENANT}:workflow:${running._id}` as any },
|
|
277
|
+
{ $set: { pid: process.pid + 1, hostname: os.hostname(), expiresAt: Date.now() + 60_000 } },
|
|
278
|
+
);
|
|
279
|
+
|
|
280
|
+
const run: any = await pending;
|
|
281
|
+
expect(run.steps[2].status).toBe("pending"); // the last step never ran
|
|
282
|
+
expect(state.log).not.toContain("slow:last");
|
|
283
|
+
|
|
284
|
+
await rest.db.collection("_locks_").deleteOne({ _id: `${TENANT}:workflow:${running._id}` as any });
|
|
285
|
+
}, 20_000);
|
|
286
|
+
|
|
287
|
+
it("records the executing pid on the run", async () => {
|
|
288
|
+
const run = await rest.workflow.run("simple", { echo: "who" });
|
|
289
|
+
expect(run.pid).toBe(process.pid);
|
|
290
|
+
expect(run.hostname).toBe(os.hostname());
|
|
291
|
+
});
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
describe("workflow resume guards", () => {
|
|
295
|
+
it("refuses to resume a run started by another version", async () => {
|
|
296
|
+
await insertRun({
|
|
297
|
+
_id: "version-1",
|
|
298
|
+
workflowId: "versioned",
|
|
299
|
+
workflowVersion: 1,
|
|
300
|
+
status: "paused",
|
|
301
|
+
steps: [{ stepId: "v", status: "pending", startedAt: new Date() } as any],
|
|
302
|
+
});
|
|
303
|
+
// The workflow has been deployed in version 2 since the run started
|
|
304
|
+
getWorkflow("versioned", TENANT)!.version = 2;
|
|
305
|
+
|
|
306
|
+
await expect(rest.workflow.resume("version-1")).rejects.toThrow(/version/);
|
|
307
|
+
|
|
308
|
+
// …unless the operator forces it
|
|
309
|
+
state.log.length = 0;
|
|
310
|
+
const run: any = await rest.workflow.resume("version-1", undefined, { force: true });
|
|
311
|
+
expect(run.status).toBe("completed");
|
|
312
|
+
expect(state.log).toEqual(["versioned:v"]);
|
|
313
|
+
|
|
314
|
+
getWorkflow("versioned", TENANT)!.version = 1;
|
|
315
|
+
});
|
|
316
|
+
});
|
|
317
|
+
|
|
318
|
+
describe("workflow resumeAll", () => {
|
|
319
|
+
it("resumes every paused run, batch by batch", async () => {
|
|
320
|
+
for (let i = 0; i < 5; i++) {
|
|
321
|
+
await insertRun({
|
|
322
|
+
_id: `bulk-${i}`,
|
|
323
|
+
workflowId: "simple",
|
|
324
|
+
status: "paused",
|
|
325
|
+
steps: [{ stepId: "one", status: "pending", startedAt: new Date() } as any],
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
const { resumed } = await rest.workflow.resumeAll(undefined, { batchSize: 2 });
|
|
330
|
+
expect(resumed).toBeGreaterThanOrEqual(5);
|
|
331
|
+
|
|
332
|
+
const left = await rest.db.collection("_workflows_")
|
|
333
|
+
.countDocuments({ _id: /^bulk-/, status: { $in: ["paused", "failed"] } });
|
|
334
|
+
expect(left).toBe(0);
|
|
335
|
+
|
|
336
|
+
const done = await rest.db.collection("_workflows_")
|
|
337
|
+
.countDocuments({ _id: /^bulk-/, status: "completed" });
|
|
338
|
+
expect(done).toBe(5);
|
|
339
|
+
}, 20_000);
|
|
340
|
+
});
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll, afterAll } from "bun:test";
|
|
2
|
+
import { formatConfig, cfg } from "../server/config";
|
|
3
|
+
import { syncTenants, getTenant } from "../database/tenant";
|
|
4
|
+
import { syncWorkflows, ensureWorkflowIndexes, resolveWorkflowsRetention } from "../lib/workflow";
|
|
5
|
+
|
|
6
|
+
const SERVER_RETENTION = "30d";
|
|
7
|
+
|
|
8
|
+
const INHERIT = "wfr-inherit";
|
|
9
|
+
const OVERRIDE = "wfr-override";
|
|
10
|
+
const OFF = "wfr-off";
|
|
11
|
+
|
|
12
|
+
const DBS = {
|
|
13
|
+
[INHERIT]: "mongodb://localhost:27017/_WF_RET_INHERIT",
|
|
14
|
+
[OVERRIDE]: "mongodb://localhost:27017/_WF_RET_OVERRIDE",
|
|
15
|
+
[OFF]: "mongodb://localhost:27017/_WF_RET_OFF",
|
|
16
|
+
} as const;
|
|
17
|
+
|
|
18
|
+
const TTL_INDEX = "_workflow_runs_ttl_";
|
|
19
|
+
|
|
20
|
+
/** TTL index of a tenant's `_workflows_` collection, or undefined */
|
|
21
|
+
async function ttlIndex(tenantId: string): Promise<any> {
|
|
22
|
+
const db = getTenant(tenantId)!.database!.db!;
|
|
23
|
+
return (await db.collection("_workflows_").listIndexes().toArray())
|
|
24
|
+
.find((i: any) => i.name === TTL_INDEX);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
beforeAll(async () => {
|
|
28
|
+
formatConfig({
|
|
29
|
+
server: {
|
|
30
|
+
port: 4000,
|
|
31
|
+
// Server-wide default — inherited unless the tenant declares its own
|
|
32
|
+
workflows: { retention: SERVER_RETENTION },
|
|
33
|
+
},
|
|
34
|
+
tenants: [
|
|
35
|
+
{ id: INHERIT, dir: "packages/core/tests/fixtures/workflow-engine", database: { uri: DBS[INHERIT] } },
|
|
36
|
+
{
|
|
37
|
+
id: OVERRIDE,
|
|
38
|
+
dir: "packages/core/tests/fixtures/workflow-engine",
|
|
39
|
+
database: { uri: DBS[OVERRIDE] },
|
|
40
|
+
workflows: { retention: "1h" },
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
id: OFF,
|
|
44
|
+
dir: "packages/core/tests/fixtures/workflow-engine",
|
|
45
|
+
database: { uri: DBS[OFF] },
|
|
46
|
+
workflows: { retention: false },
|
|
47
|
+
},
|
|
48
|
+
],
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
await syncTenants();
|
|
52
|
+
await syncWorkflows();
|
|
53
|
+
await ensureWorkflowIndexes();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
afterAll(async () => {
|
|
57
|
+
for (const id of Object.keys(DBS)) {
|
|
58
|
+
try { await getTenant(id)?.database?.db?.collection("_workflows_").drop(); } catch (_) {}
|
|
59
|
+
}
|
|
60
|
+
delete (cfg.server as any).workflows;
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
describe("workflow run retention", () => {
|
|
64
|
+
it("resolves the tenant first, the server default as fallback", () => {
|
|
65
|
+
expect(resolveWorkflowsRetention({ id: INHERIT, dir: "x", database: { uri: "" } } as any)).toBe(SERVER_RETENTION);
|
|
66
|
+
expect(resolveWorkflowsRetention({ id: OVERRIDE, dir: "x", database: { uri: "" }, workflows: { retention: "1h" } } as any)).toBe("1h");
|
|
67
|
+
// `false` is a value, not an absence — it must override the server default
|
|
68
|
+
expect(resolveWorkflowsRetention({ id: OFF, dir: "x", database: { uri: "" }, workflows: { retention: false } } as any)).toBe(false);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("applies the server default to a tenant without its own config", async () => {
|
|
72
|
+
const ttl = await ttlIndex(INHERIT);
|
|
73
|
+
expect(ttl).toBeDefined();
|
|
74
|
+
expect(ttl.key).toEqual({ completedAt: 1 });
|
|
75
|
+
expect(ttl.expireAfterSeconds).toBe(30 * 86400);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("lets the tenant override the server default", async () => {
|
|
79
|
+
const ttl = await ttlIndex(OVERRIDE);
|
|
80
|
+
expect(ttl).toBeDefined();
|
|
81
|
+
expect(ttl.expireAfterSeconds).toBe(3600);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it("lets a tenant opt out with `false`", async () => {
|
|
85
|
+
expect(await ttlIndex(OFF)).toBeUndefined();
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it("only prunes finished runs (`completedAt` is never set on an in-flight one)", async () => {
|
|
89
|
+
const db = getTenant(INHERIT)!.database!.db!;
|
|
90
|
+
const col: any = db.collection("_workflows_");
|
|
91
|
+
await col.insertMany([
|
|
92
|
+
{ _id: "ret-running", workflowId: "simple", status: "running", createdAt: new Date(), updatedAt: new Date() },
|
|
93
|
+
{ _id: "ret-paused", workflowId: "simple", status: "paused", createdAt: new Date(), updatedAt: new Date() },
|
|
94
|
+
{ _id: "ret-done", workflowId: "simple", status: "completed", completedAt: new Date(), createdAt: new Date(), updatedAt: new Date() },
|
|
95
|
+
]);
|
|
96
|
+
|
|
97
|
+
const missing = await col.countDocuments({ _id: { $in: ["ret-running", "ret-paused"] }, completedAt: { $exists: false } });
|
|
98
|
+
const finished = await col.countDocuments({ completedAt: { $exists: true } });
|
|
99
|
+
|
|
100
|
+
expect(missing).toBe(2); // never pruned by a TTL on `completedAt`
|
|
101
|
+
expect(finished).toBe(1); // only the finished run can expire
|
|
102
|
+
});
|
|
103
|
+
});
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll, afterAll } from "bun:test";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import { formatConfig } from "../server/config";
|
|
4
|
+
import { syncTenants } from "../database/tenant";
|
|
5
|
+
import { syncWorkflows, ensureWorkflowIndexes, getWorkflow, workflowStats } from "../lib/workflow";
|
|
6
|
+
import { useRest } from "../database/rest";
|
|
7
|
+
|
|
8
|
+
const TENANT = "wf-test";
|
|
9
|
+
// Second tenant sharing the same code folder — same workflows, other database
|
|
10
|
+
const TENANT_2 = "wf-test-2";
|
|
11
|
+
const DIR = "packages/core/tests/fixtures/workflow-tenant";
|
|
12
|
+
|
|
13
|
+
let rest: InstanceType<typeof useRest>;
|
|
14
|
+
|
|
15
|
+
async function indexNames(tenantId: string): Promise<string[]> {
|
|
16
|
+
const tenantRest = new useRest({ tenant_id: tenantId, internal: true });
|
|
17
|
+
return (await tenantRest.db.collection("_workflows_").listIndexes().toArray()).map((i: any) => i.name);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
beforeAll(async () => {
|
|
21
|
+
formatConfig({
|
|
22
|
+
server: { port: 4000 },
|
|
23
|
+
tenants: [
|
|
24
|
+
{ id: TENANT, dir: DIR, database: { uri: "mongodb://localhost:27017/_WF_TEST" } },
|
|
25
|
+
{ id: TENANT_2, dir: DIR, database: { uri: "mongodb://localhost:27017/_WF_TEST_2" } },
|
|
26
|
+
],
|
|
27
|
+
});
|
|
28
|
+
await syncTenants();
|
|
29
|
+
await syncWorkflows();
|
|
30
|
+
// The loader fires the index pass in the background — await it here so the
|
|
31
|
+
// assertions below are deterministic (the non-blocking behaviour is tested
|
|
32
|
+
// separately, via `syncWorkflows()` alone).
|
|
33
|
+
await ensureWorkflowIndexes();
|
|
34
|
+
|
|
35
|
+
rest = new useRest({ tenant_id: TENANT, internal: true });
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
afterAll(async () => {
|
|
39
|
+
for (const id of [TENANT, TENANT_2]) {
|
|
40
|
+
try { await new useRest({ tenant_id: id, internal: true }).db.collection("_workflows_").drop(); } catch (_) {}
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
describe("workflow loader", () => {
|
|
45
|
+
it("registers the enabled workflows of the tenant", () => {
|
|
46
|
+
const invoice = getWorkflow("invoice", TENANT);
|
|
47
|
+
expect(invoice).toBeDefined();
|
|
48
|
+
expect(invoice!._tenant_).toBe(TENANT);
|
|
49
|
+
expect(invoice!.version).toBe(3);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("keeps exactly one definition for a duplicated id", () => {
|
|
53
|
+
const order = getWorkflow("order", TENANT);
|
|
54
|
+
// Which file wins depends on the filesystem order — what matters is that
|
|
55
|
+
// there is a single, valid definition (never two, never a crash)
|
|
56
|
+
expect(["Orders", "Orders (duplicate definition)"]).toContain(order!.name);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("skips disabled workflows (`enabled: false`)", () => {
|
|
60
|
+
expect(getWorkflow("nightly", TENANT)).toBeUndefined();
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("ignores a workflow without an `id`", () => {
|
|
64
|
+
// A missing id used to be registered under the literal `undefined` key
|
|
65
|
+
expect(getWorkflow("undefined", TENANT)).toBeUndefined();
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it("isolates a file that throws on import", () => {
|
|
69
|
+
expect(getWorkflow("broken", TENANT)).toBeUndefined();
|
|
70
|
+
// …and the other workflows are still there
|
|
71
|
+
expect(getWorkflow("order", TENANT)).toBeDefined();
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it("loads the same folder independently for each tenant", () => {
|
|
75
|
+
expect(getWorkflow("order", TENANT)).toBeDefined();
|
|
76
|
+
expect(getWorkflow("order", TENANT_2)).toBeDefined();
|
|
77
|
+
expect(getWorkflow("order", TENANT_2)!._tenant_).toBe(TENANT_2);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it("reports what was loaded", () => {
|
|
81
|
+
const stats = workflowStats();
|
|
82
|
+
expect(stats.total).toBeGreaterThanOrEqual(2); // one per tenant
|
|
83
|
+
expect(stats.tenants).toContain(TENANT);
|
|
84
|
+
expect(stats.tenants).toContain(TENANT_2);
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
describe("workflow indexes", () => {
|
|
89
|
+
it("creates the indexes the engine queries on", async () => {
|
|
90
|
+
const names = await indexNames(TENANT);
|
|
91
|
+
expect(names).toContain("workflowId_1_createdAt_-1"); // listRuns()
|
|
92
|
+
expect(names).toContain("status_1"); // resumeAll()
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it("creates the indexes declared by the workflow context", async () => {
|
|
96
|
+
const names = await indexNames(TENANT);
|
|
97
|
+
expect(names).toContain("context.customerId_1");
|
|
98
|
+
expect(names).toContain("context.total_-1");
|
|
99
|
+
expect(names).toContain("context.companyId_1");
|
|
100
|
+
// a field without `index` is not indexed
|
|
101
|
+
expect(names).not.toContain("context.note_1");
|
|
102
|
+
expect(names).not.toContain("context.processedAt_1");
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it("creates the explicit `indexes` — compound, unique, sparse, named", async () => {
|
|
106
|
+
const names = await indexNames(TENANT);
|
|
107
|
+
expect(names).toContain("context.companyId_1_status_1"); // compound
|
|
108
|
+
expect(names).toContain("uniq_batch"); // explicit name
|
|
109
|
+
|
|
110
|
+
const uniq: any = (await rest.db.collection("_workflows_").listIndexes().toArray())
|
|
111
|
+
.find((i: any) => i.name === "uniq_batch");
|
|
112
|
+
expect(uniq.unique).toBe(true);
|
|
113
|
+
expect(uniq.sparse).toBe(true);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it("is idempotent — running it again creates nothing", async () => {
|
|
117
|
+
const before = (await indexNames(TENANT)).length;
|
|
118
|
+
await ensureWorkflowIndexes();
|
|
119
|
+
expect((await indexNames(TENANT)).length).toBe(before);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it("serves the engine's own query with an index scan", async () => {
|
|
123
|
+
const col: any = rest.db.collection("_workflows_");
|
|
124
|
+
// A run must exist for the collection to be queryable/explainable
|
|
125
|
+
await rest.workflow.run("invoice", { ref: "A-1" });
|
|
126
|
+
const plan = JSON.stringify(await col.find({ workflowId: "invoice" }).explain("queryPlanner"));
|
|
127
|
+
expect(plan).toContain("IXSCAN"); // COLLSCAN before the loader indexed `workflowId`
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
describe("workflow engine wiring", () => {
|
|
132
|
+
it("runs a workflow loaded from the filesystem", async () => {
|
|
133
|
+
const run = await rest.workflow.run("invoice", { ref: "A-1" }, { companyId: "c-1" });
|
|
134
|
+
expect(run.status).toBe("completed");
|
|
135
|
+
expect(run.workflowVersion).toBe(3);
|
|
136
|
+
expect(run.context).toEqual({ companyId: "c-1" });
|
|
137
|
+
expect(run.steps.length).toBe(2);
|
|
138
|
+
// The run says which process executed it (`reusePort` / forked workers)
|
|
139
|
+
expect(run.pid).toBe(process.pid);
|
|
140
|
+
expect(run.hostname).toBe(os.hostname());
|
|
141
|
+
|
|
142
|
+
const runs = await rest.workflow.listRuns("invoice");
|
|
143
|
+
expect(runs.length).toBeGreaterThanOrEqual(1);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it("refuses to run a disabled workflow", async () => {
|
|
147
|
+
await expect(rest.workflow.run("nightly", {})).rejects.toThrow();
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
describe("workflow context validation", () => {
|
|
152
|
+
it("accepts a context that matches the declaration", async () => {
|
|
153
|
+
const run = await rest.workflow.run("invoice", { ref: "B-1" }, { companyId: "c-2" });
|
|
154
|
+
expect(run.context).toEqual({ companyId: "c-2" });
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("rejects a wrong type with a clear code", async () => {
|
|
158
|
+
await expect(
|
|
159
|
+
rest.workflow.run("invoice", {}, { companyId: 42 as any }),
|
|
160
|
+
).rejects.toThrow(/must be a string/);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it("coerces an ISO string to a real Date (so `context.*` indexes stay typed)", async () => {
|
|
164
|
+
const run = await rest.workflow.run("invoice", {}, {
|
|
165
|
+
companyId: "c-3",
|
|
166
|
+
processedAt: "2026-03-04T05:06:07.000Z" as any,
|
|
167
|
+
});
|
|
168
|
+
expect(run.context.processedAt).toBeInstanceOf(Date);
|
|
169
|
+
expect((run.context.processedAt as Date).toISOString()).toBe("2026-03-04T05:06:07.000Z");
|
|
170
|
+
|
|
171
|
+
// …and it is stored as a Date in MongoDB too
|
|
172
|
+
const stored: any = await rest.workflow.getRun(run._id);
|
|
173
|
+
expect(stored.context.processedAt).toBeInstanceOf(Date);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it("rejects an invalid date", async () => {
|
|
177
|
+
await expect(
|
|
178
|
+
rest.workflow.run("invoice", {}, { processedAt: "not-a-date" as any }),
|
|
179
|
+
).rejects.toThrow(/must be a date/);
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it("keeps undeclared keys as-is", async () => {
|
|
183
|
+
const run = await rest.workflow.run("invoice", {}, { companyId: "c-4", extra: { any: "thing" } });
|
|
184
|
+
expect(run.context.extra).toEqual({ any: "thing" });
|
|
185
|
+
});
|
|
186
|
+
});
|