@niwa-labs/tasks 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +4 -2
- package/package.json +42 -4
- package/src/actions.ts +109 -0
- package/src/cockpit/actions.ts +52 -0
- package/src/cockpit/index.tsx +32 -0
- package/src/cockpit/pages.tsx +103 -0
- package/src/executor.ts +214 -0
- package/src/index.ts +516 -0
- package/src/ledger.ts +88 -0
- package/src/owner.ts +178 -0
- package/src/records-entry.ts +38 -0
- package/src/records.ts +118 -0
- package/src/strategy.ts +60 -0
- package/test/asker.test.ts +67 -0
- package/test/cockpit-actions.test.ts +19 -0
- package/test/executor.test.ts +520 -0
- package/test/ledger.test.ts +172 -0
- package/test/notifications.test.ts +170 -0
- package/test/queries.test.ts +19 -0
- package/test/steps.test.ts +147 -0
- package/test/tasks.test.ts +536 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
// What a task says to the address it reports to: the updates first, the end after them, and an end
|
|
2
|
+
// that is too large for a message is kept as an artifact the message points at.
|
|
3
|
+
|
|
4
|
+
import { expect, test } from "bun:test";
|
|
5
|
+
|
|
6
|
+
import { ARTIFACT_THRESHOLD, RECORD_DATA_LIMIT } from "@niwa-labs/core";
|
|
7
|
+
import type { Address, Caller, RecordTypeName } from "@niwa-labs/core";
|
|
8
|
+
import { MessageSent } from "@niwa-labs/messages/records";
|
|
9
|
+
import { plugin } from "@niwa-labs/plugin";
|
|
10
|
+
import type { CallerBinding } from "@niwa-labs/plugin";
|
|
11
|
+
import { connectAsOwner, createNiwa, messageSend, messages, tempHome } from "@niwa-labs/testing";
|
|
12
|
+
import type { Host } from "@niwa-labs/testing";
|
|
13
|
+
|
|
14
|
+
import { taskCreate, taskFail, taskFinish, taskReport, taskResult, tasks } from "../src/index.ts";
|
|
15
|
+
|
|
16
|
+
const WORKER: Address = { kind: "agent", id: "worker" };
|
|
17
|
+
const OWNER: Address = { kind: "owner" };
|
|
18
|
+
|
|
19
|
+
/** Clients speak as the worker, which is responsible for the tasks; the host stands for nobody. */
|
|
20
|
+
const bind: CallerBinding = (caller: Caller) => (caller.kind === "client" ? WORKER : undefined);
|
|
21
|
+
|
|
22
|
+
const input = (text = "Move it.") => ({
|
|
23
|
+
kind: "change",
|
|
24
|
+
description: "Move argument parsing",
|
|
25
|
+
text,
|
|
26
|
+
askedBy: OWNER,
|
|
27
|
+
responsible: WORKER,
|
|
28
|
+
reportTo: OWNER,
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
/** A host with the tasks module, and a client that speaks as the worker. */
|
|
32
|
+
async function served(
|
|
33
|
+
extra: ReadonlyArray<ReturnType<typeof plugin>>,
|
|
34
|
+
run: (host: Host, worker: Awaited<ReturnType<typeof connectAsOwner>>) => Promise<void>,
|
|
35
|
+
) {
|
|
36
|
+
const home = await tempHome();
|
|
37
|
+
const niwa = createNiwa({ home: home.path });
|
|
38
|
+
niwa.use(messages({ bind }));
|
|
39
|
+
niwa.use(tasks({ messages: { bind } }));
|
|
40
|
+
for (const each of extra) niwa.use(each);
|
|
41
|
+
await niwa.serve();
|
|
42
|
+
const worker = await connectAsOwner(home.path, "worker");
|
|
43
|
+
try {
|
|
44
|
+
await run(niwa, worker);
|
|
45
|
+
} finally {
|
|
46
|
+
await worker.close().catch(() => {});
|
|
47
|
+
await niwa.close().catch(() => {});
|
|
48
|
+
await home.cleanup();
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The messages to the owner, in the order they were written. */
|
|
53
|
+
async function toOwner(niwa: Host) {
|
|
54
|
+
const rows = await niwa.records.read({ types: [MessageSent.name as RecordTypeName] });
|
|
55
|
+
return rows.map((row) => row.data as { to: Address; task?: string; body: { type: string; text: string; data?: unknown } }).filter((message) => message.to.kind === "owner");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function eventually(fn: () => boolean | Promise<boolean>, message: string): Promise<void> {
|
|
59
|
+
const deadline = Date.now() + 15_000;
|
|
60
|
+
for (;;) {
|
|
61
|
+
if (await fn()) return;
|
|
62
|
+
if (Date.now() >= deadline) throw new Error(message);
|
|
63
|
+
await Bun.sleep(20);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
test("the end of a task is reported after its updates, even when the last update is delivered late", async () => {
|
|
68
|
+
// The delivery of an update is held until the test lets it go: the end is written meanwhile.
|
|
69
|
+
let release: () => void = () => {};
|
|
70
|
+
const held = new Promise<void>((resolve) => (release = resolve));
|
|
71
|
+
let entered = 0;
|
|
72
|
+
const slow = plugin("slow-updates", (api) => {
|
|
73
|
+
api.guard(messageSend(), async (message) => {
|
|
74
|
+
if (message.body.type !== "update") return;
|
|
75
|
+
entered += 1;
|
|
76
|
+
await held;
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
await served([slow], async (niwa, worker) => {
|
|
81
|
+
const { id } = await niwa.call(taskCreate, input());
|
|
82
|
+
await worker.call(taskReport, { task: id, text: "half way" });
|
|
83
|
+
await eventually(() => entered >= 1, "the delivery of the update should have started");
|
|
84
|
+
await worker.call(taskFinish, { task: id, result: "done" });
|
|
85
|
+
|
|
86
|
+
// The end is written and the update is still on its way: the end waits for it.
|
|
87
|
+
await Bun.sleep(300);
|
|
88
|
+
expect((await toOwner(niwa)).map((message) => message.body.type)).toEqual([]);
|
|
89
|
+
|
|
90
|
+
release();
|
|
91
|
+
await eventually(async () => (await toOwner(niwa)).length >= 2, "the owner should have been told both");
|
|
92
|
+
expect((await toOwner(niwa)).map((message) => message.body.type)).toEqual(["update", "result"]);
|
|
93
|
+
});
|
|
94
|
+
}, 30_000);
|
|
95
|
+
|
|
96
|
+
test("an end that waits for a late update holds no end of another task", async () => {
|
|
97
|
+
let release: () => void = () => {};
|
|
98
|
+
const held = new Promise<void>((resolve) => (release = resolve));
|
|
99
|
+
let entered = 0;
|
|
100
|
+
const slow = plugin("slow-updates", (api) => {
|
|
101
|
+
api.guard(messageSend(), async (message) => {
|
|
102
|
+
if (message.body.type !== "update") return;
|
|
103
|
+
entered += 1;
|
|
104
|
+
await held;
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
await served([slow], async (niwa, worker) => {
|
|
109
|
+
const slowTask = (await niwa.call(taskCreate, input("slow"))).id;
|
|
110
|
+
await worker.call(taskReport, { task: slowTask, text: "half way" });
|
|
111
|
+
await eventually(() => entered >= 1, "the delivery of the update should have started");
|
|
112
|
+
await worker.call(taskFinish, { task: slowTask, result: "slow done" });
|
|
113
|
+
|
|
114
|
+
const quick = (await niwa.call(taskCreate, input("quick"))).id;
|
|
115
|
+
await worker.call(taskFinish, { task: quick, result: "quick done" });
|
|
116
|
+
await eventually(async () => (await toOwner(niwa)).some((message) => message.body.text === "quick done"), "the other task should have been reported");
|
|
117
|
+
expect((await toOwner(niwa)).map((message) => message.body.text)).toEqual(["quick done"]);
|
|
118
|
+
|
|
119
|
+
release();
|
|
120
|
+
await eventually(async () => (await toOwner(niwa)).length >= 3, "the slow task should be reported too");
|
|
121
|
+
expect((await toOwner(niwa)).filter((message) => message.task === slowTask).map((message) => message.body.type)).toEqual(["update", "result"]);
|
|
122
|
+
});
|
|
123
|
+
}, 30_000);
|
|
124
|
+
|
|
125
|
+
test("a result too large for a message is kept as an artifact, and the end of the next task is still reported", async () => {
|
|
126
|
+
await served([], async (niwa, worker) => {
|
|
127
|
+
const big = "x".repeat(150 * 1024);
|
|
128
|
+
const first = (await niwa.call(taskCreate, input("big"))).id;
|
|
129
|
+
await worker.call(taskFinish, { task: first, result: big });
|
|
130
|
+
const second = (await niwa.call(taskCreate, input("small"))).id;
|
|
131
|
+
await worker.call(taskFinish, { task: second, result: "small done" });
|
|
132
|
+
|
|
133
|
+
await eventually(async () => (await toOwner(niwa)).length >= 2, "both ends should have been reported");
|
|
134
|
+
const messages = await toOwner(niwa);
|
|
135
|
+
const [bigMessage, smallMessage] = messages;
|
|
136
|
+
expect(smallMessage?.body.text).toBe("small done");
|
|
137
|
+
|
|
138
|
+
// The message holds a short text and the reference, not the result twice.
|
|
139
|
+
expect(bigMessage?.body.type).toBe("result");
|
|
140
|
+
expect(Buffer.byteLength(JSON.stringify(bigMessage))).toBeLessThan(ARTIFACT_THRESHOLD);
|
|
141
|
+
const kept = (bigMessage?.body.data as { artifact: { digest: string; size: number } }).artifact;
|
|
142
|
+
expect(Buffer.from(niwa.artifacts.read(kept)).toString()).toBe(JSON.stringify(big));
|
|
143
|
+
|
|
144
|
+
// The whole result is still what the task gives.
|
|
145
|
+
expect(await niwa.call(taskResult, { task: first })).toEqual({ task: first, outcome: { status: "result", result: big } });
|
|
146
|
+
});
|
|
147
|
+
}, 30_000);
|
|
148
|
+
|
|
149
|
+
test("a failure whose reason is too large for a message is kept as an artifact too", async () => {
|
|
150
|
+
await served([], async (niwa, worker) => {
|
|
151
|
+
const reason = "e".repeat(100 * 1024);
|
|
152
|
+
const { id } = await niwa.call(taskCreate, input());
|
|
153
|
+
await worker.call(taskFail, { task: id, reason });
|
|
154
|
+
await eventually(async () => (await toOwner(niwa)).length >= 1, "the failure should have been reported");
|
|
155
|
+
const [message] = await toOwner(niwa);
|
|
156
|
+
expect(message?.body.type).toBe("failure");
|
|
157
|
+
expect(Buffer.byteLength(JSON.stringify(message))).toBeLessThan(ARTIFACT_THRESHOLD);
|
|
158
|
+
const kept = (message?.body.data as { artifact: { digest: string } }).artifact;
|
|
159
|
+
expect(Buffer.from(niwa.artifacts.read(kept)).toString()).toBe(reason);
|
|
160
|
+
});
|
|
161
|
+
}, 30_000);
|
|
162
|
+
|
|
163
|
+
test("an end that no record could hold is refused before anything is written, and the task stays open", async () => {
|
|
164
|
+
await served([], async (niwa, worker) => {
|
|
165
|
+
const { id } = await niwa.call(taskCreate, input());
|
|
166
|
+
await expect(worker.call(taskFinish, { task: id, result: "y".repeat(RECORD_DATA_LIMIT) })).rejects.toThrow("more than a record holds");
|
|
167
|
+
await worker.call(taskFinish, { task: id, result: "fits" });
|
|
168
|
+
expect(await niwa.call(taskResult, { task: id })).toEqual({ task: id, outcome: { status: "result", result: "fits" } });
|
|
169
|
+
});
|
|
170
|
+
}, 30_000);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// The query builders of the tasks records entry: the records of one task, as a script waits for
|
|
2
|
+
// them, a reader reads them and a handler subscribes to them.
|
|
3
|
+
|
|
4
|
+
import { expect, test } from "bun:test";
|
|
5
|
+
|
|
6
|
+
import { resolveQuery } from "@niwa-labs/core";
|
|
7
|
+
|
|
8
|
+
import { TaskDone, TaskUpdated, taskDone, taskUpdates } from "../src/records-entry.ts";
|
|
9
|
+
|
|
10
|
+
test("the end and the updates of one task are a query of their type, filtered on the task", () => {
|
|
11
|
+
const task = "task-3fa9c2";
|
|
12
|
+
expect(taskDone(task)).toEqual({ types: [TaskDone], filter: { task } });
|
|
13
|
+
expect(taskUpdates(task)).toEqual({ types: [TaskUpdated], filter: { task } });
|
|
14
|
+
expect(resolveQuery(taskDone(task))).toEqual({ types: ["task.done"], filter: { task } });
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("a text that is not the id of a task is refused when the query is made", () => {
|
|
18
|
+
expect(() => taskDone("run-3fa9c2")).toThrow();
|
|
19
|
+
});
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// What a handler of another module sees of the tasks module through a step: a refusal it can catch
|
|
2
|
+
// and correct, and a result it waits for as an invocation that is waiting, holding nothing in memory.
|
|
3
|
+
|
|
4
|
+
import { expect, test } from "bun:test";
|
|
5
|
+
|
|
6
|
+
import { CORE_TYPES, record } from "@niwa-labs/core";
|
|
7
|
+
import type { Address, Caller } from "@niwa-labs/core";
|
|
8
|
+
import { plugin } from "@niwa-labs/plugin";
|
|
9
|
+
import type { CallerBinding } from "@niwa-labs/plugin";
|
|
10
|
+
import { connectAsOwner, createNiwa, messages, tempHome } from "@niwa-labs/testing";
|
|
11
|
+
import type { Host } from "@niwa-labs/testing";
|
|
12
|
+
import { z } from "zod";
|
|
13
|
+
|
|
14
|
+
import { TaskId, TaskRefusal, TaskWaiters, taskCreate, taskFinish, taskResult, tasks } from "../src/index.ts";
|
|
15
|
+
|
|
16
|
+
const WORKER: Address = { kind: "agent", id: "worker" };
|
|
17
|
+
const OWNER: Address = { kind: "owner" };
|
|
18
|
+
|
|
19
|
+
const SOMEBODY: Address = { kind: "agent", id: "somebody" };
|
|
20
|
+
|
|
21
|
+
/** A handler of the plugin "worker" speaks as the worker; a client speaks as somebody else. */
|
|
22
|
+
const bind: CallerBinding = (caller: Caller) => {
|
|
23
|
+
if (caller.kind === "handler" && caller.handler.startsWith("worker/")) return WORKER;
|
|
24
|
+
return caller.kind === "client" ? SOMEBODY : undefined;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** The record that wakes the worker: a task for it to do. */
|
|
28
|
+
const Job = record("worker.job", z.object({ task: TaskId }), { open: true });
|
|
29
|
+
/** What the worker saw. */
|
|
30
|
+
const Saw = record("worker.saw", z.object({ task: TaskId, note: z.string() }), { open: true });
|
|
31
|
+
|
|
32
|
+
const shape = { type: "object", properties: { files: { type: "number" } }, required: ["files"] };
|
|
33
|
+
|
|
34
|
+
async function served(worker: ReturnType<typeof plugin>, run: (niwa: Host, home: string) => Promise<void>) {
|
|
35
|
+
const home = await tempHome();
|
|
36
|
+
const niwa = createNiwa({ home: home.path });
|
|
37
|
+
niwa.use(messages({ bind }));
|
|
38
|
+
niwa.use(tasks({ messages: { bind } }));
|
|
39
|
+
niwa.use(worker);
|
|
40
|
+
await niwa.serve();
|
|
41
|
+
try {
|
|
42
|
+
await run(niwa, home.path);
|
|
43
|
+
} finally {
|
|
44
|
+
await niwa.close().catch(() => {});
|
|
45
|
+
await home.cleanup();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function eventually(fn: () => boolean | Promise<boolean>, message: string): Promise<void> {
|
|
50
|
+
const deadline = Date.now() + 15_000;
|
|
51
|
+
for (;;) {
|
|
52
|
+
if (await fn()) return;
|
|
53
|
+
if (Date.now() >= deadline) throw new Error(message);
|
|
54
|
+
await Bun.sleep(20);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const states = async (niwa: Host) =>
|
|
59
|
+
(await niwa.records.read({ types: [CORE_TYPES.invocationState] })).map((row) => (row.data as { state: string }).state);
|
|
60
|
+
|
|
61
|
+
test("a result that does not fit is a refusal the handler catches and corrects, and the handler goes on", async () => {
|
|
62
|
+
const corrector = plugin("worker", (api) => {
|
|
63
|
+
api.declare(Job, Saw);
|
|
64
|
+
api.on(Job, async (job, step) => {
|
|
65
|
+
try {
|
|
66
|
+
await step(taskFinish, { task: job.task, result: { files: "two" } });
|
|
67
|
+
} catch (error) {
|
|
68
|
+
if (!(error instanceof TaskRefusal) || error.code !== "result-invalid") throw error;
|
|
69
|
+
await step.emit(Saw, { task: job.task, note: error.code });
|
|
70
|
+
await step(taskFinish, { task: job.task, result: { files: 2 } });
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
await served(corrector, async (niwa, home) => {
|
|
76
|
+
const { id } = await niwa.call(taskCreate, {
|
|
77
|
+
kind: "change",
|
|
78
|
+
description: "Move argument parsing",
|
|
79
|
+
text: "Move it.",
|
|
80
|
+
askedBy: OWNER,
|
|
81
|
+
responsible: WORKER,
|
|
82
|
+
reportTo: OWNER,
|
|
83
|
+
result: shape,
|
|
84
|
+
});
|
|
85
|
+
const client = await connectAsOwner(home);
|
|
86
|
+
try {
|
|
87
|
+
await client.emit(Job, { task: id });
|
|
88
|
+
await eventually(async () => (await niwa.records.read({ types: ["worker.saw" as never] })).length === 1, "the refusal should have been caught");
|
|
89
|
+
expect(await niwa.call(taskResult, { task: id })).toEqual({ task: id, outcome: { status: "result", result: { files: 2 } } });
|
|
90
|
+
// The refusal paused nothing: the handler ran to its end.
|
|
91
|
+
expect(await states(niwa)).not.toContain("paused");
|
|
92
|
+
} finally {
|
|
93
|
+
await client.close().catch(() => {});
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}, 30_000);
|
|
97
|
+
|
|
98
|
+
test("a handler that asks for the result of an open task waits as an invocation, and goes on when the task ends", async () => {
|
|
99
|
+
const waiter = plugin("worker", (api) => {
|
|
100
|
+
api.declare(Job, Saw);
|
|
101
|
+
api.on(Job, async (job, step) => {
|
|
102
|
+
const got = await step(taskResult, { task: job.task });
|
|
103
|
+
await step.emit(Saw, { task: job.task, note: got.outcome.status });
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
await served(waiter, async (niwa, home) => {
|
|
108
|
+
const { id } = await niwa.call(taskCreate, {
|
|
109
|
+
kind: "change",
|
|
110
|
+
description: "Move argument parsing",
|
|
111
|
+
text: "Move it.",
|
|
112
|
+
askedBy: OWNER,
|
|
113
|
+
responsible: SOMEBODY,
|
|
114
|
+
reportTo: OWNER,
|
|
115
|
+
});
|
|
116
|
+
const client = await connectAsOwner(home);
|
|
117
|
+
try {
|
|
118
|
+
await client.emit(Job, { task: id });
|
|
119
|
+
// The invocation is waiting for the end of the task: it is not a step that runs in memory.
|
|
120
|
+
await eventually(async () => (await states(niwa)).includes("waiting"), "the invocation should be waiting");
|
|
121
|
+
await Bun.sleep(200);
|
|
122
|
+
expect((await states(niwa)).at(-1)).toBe("waiting");
|
|
123
|
+
|
|
124
|
+
await client.call(taskFinish, { task: id, result: "done" });
|
|
125
|
+
await eventually(async () => (await niwa.records.read({ types: ["worker.saw" as never] })).length === 1, "the handler should have gone on");
|
|
126
|
+
expect((await niwa.records.read({ types: ["worker.saw" as never] })).map((row) => (row.data as { note: string }).note)).toEqual(["result"]);
|
|
127
|
+
} finally {
|
|
128
|
+
await client.close().catch(() => {});
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
}, 30_000);
|
|
132
|
+
|
|
133
|
+
test("the callers that wait for a task in memory leave nothing behind, whether the task ended or not", () => {
|
|
134
|
+
const waiters = new TaskWaiters();
|
|
135
|
+
const ended = waiters.watch("task-aaaaaa");
|
|
136
|
+
expect(waiters.size).toBe(1);
|
|
137
|
+
// The caller read the state, found the task done and stopped waiting.
|
|
138
|
+
ended.stop();
|
|
139
|
+
expect(waiters.size).toBe(0);
|
|
140
|
+
|
|
141
|
+
const first = waiters.watch("task-bbbbbb");
|
|
142
|
+
const second = waiters.watch("task-bbbbbb");
|
|
143
|
+
first.stop();
|
|
144
|
+
expect(waiters.size).toBe(1);
|
|
145
|
+
second.stop();
|
|
146
|
+
expect(waiters.size).toBe(0);
|
|
147
|
+
});
|