@repogo/plugins 0.1.34 → 0.1.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +380 -26
- package/dist/cache.d.ts +27 -0
- package/dist/collections.d.ts +11 -1
- package/dist/context.d.ts +117 -1
- package/dist/cron.d.ts +21 -0
- package/dist/define.d.ts +36 -11
- package/dist/generated.d.ts +16 -0
- package/dist/index.d.ts +10 -3
- package/dist/index.js +100 -5
- package/dist/logs.d.ts +55 -0
- package/dist/runtime.d.ts +173 -0
- package/dist/self.d.ts +1 -0
- package/dist/testing.d.ts +53 -0
- package/dist/testing.js +3492 -0
- package/dist/tracking.d.ts +5 -0
- package/dist/uploads.d.ts +24 -0
- package/dist/workflow/attempts.d.ts +25 -0
- package/dist/workflow/availability.d.ts +8 -0
- package/dist/workflow/catalog.d.ts +21 -0
- package/dist/workflow/clock.d.ts +6 -0
- package/dist/workflow/contracts.d.ts +121 -0
- package/dist/workflow/definition.d.ts +35 -0
- package/dist/workflow/errors.d.ts +19 -0
- package/dist/workflow/events.d.ts +132 -0
- package/dist/workflow/interrupted.d.ts +3 -0
- package/dist/workflow/journal.d.ts +28 -0
- package/dist/workflow/lease.d.ts +2 -0
- package/dist/workflow/lifetime.d.ts +27 -0
- package/dist/workflow/policy.d.ts +36 -0
- package/dist/workflow/register.d.ts +11 -0
- package/dist/workflow/retry.d.ts +14 -0
- package/dist/workflow/runner.d.ts +29 -0
- package/dist/workflow/scope.d.ts +4 -0
- package/dist/workflow/service.d.ts +48 -0
- package/dist/workflow/sleep.d.ts +6 -0
- package/dist/workflow/state.d.ts +41 -0
- package/dist/workflow/storage.d.ts +22 -0
- package/dist/workflow/summary.d.ts +80 -0
- package/dist/workflow/testing.d.ts +12 -0
- package/dist/workflow-executor.js +22917 -0
- package/package.json +7 -3
package/dist/testing.js
ADDED
|
@@ -0,0 +1,3492 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
import {
|
|
3
|
+
MethodError,
|
|
4
|
+
attemptScope,
|
|
5
|
+
callFiles,
|
|
6
|
+
exports_src,
|
|
7
|
+
parseDuration,
|
|
8
|
+
toCallError,
|
|
9
|
+
workflowContext,
|
|
10
|
+
workflowScope
|
|
11
|
+
} from "./index.js";
|
|
12
|
+
|
|
13
|
+
// src/testing.ts
|
|
14
|
+
import { mkdir as mkdir6, mkdtemp as mkdtemp2, rm as rm5 } from "fs/promises";
|
|
15
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
16
|
+
import { join as join8 } from "path";
|
|
17
|
+
|
|
18
|
+
// src/logs.ts
|
|
19
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
20
|
+
import { format } from "util";
|
|
21
|
+
var logContext = new AsyncLocalStorage;
|
|
22
|
+
var activeRuns = [];
|
|
23
|
+
async function runWith(context, work) {
|
|
24
|
+
activeRuns.push(context);
|
|
25
|
+
try {
|
|
26
|
+
return await logContext.run(context, work);
|
|
27
|
+
} finally {
|
|
28
|
+
const i = activeRuns.lastIndexOf(context);
|
|
29
|
+
if (i >= 0)
|
|
30
|
+
activeRuns.splice(i, 1);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
var textLimit = 8 * 1024;
|
|
34
|
+
var levels = {
|
|
35
|
+
log: "info",
|
|
36
|
+
info: "info",
|
|
37
|
+
debug: "debug",
|
|
38
|
+
warn: "warn",
|
|
39
|
+
error: "error"
|
|
40
|
+
};
|
|
41
|
+
var original = { log: console.log, info: console.info, debug: console.debug, warn: console.warn, error: console.error };
|
|
42
|
+
var sink = null;
|
|
43
|
+
function captureConsole(to, options = {}) {
|
|
44
|
+
sink = to;
|
|
45
|
+
for (const name of Object.keys(levels)) {
|
|
46
|
+
console[name] = (...args) => {
|
|
47
|
+
if (options.passThrough)
|
|
48
|
+
original[name](...args);
|
|
49
|
+
record(levels[name], args);
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function releaseConsole() {
|
|
54
|
+
sink = null;
|
|
55
|
+
Object.assign(console, original);
|
|
56
|
+
}
|
|
57
|
+
function record(level, args, context) {
|
|
58
|
+
const where = context ?? logContext.getStore() ?? activeRuns.at(-1);
|
|
59
|
+
if (!sink || !where) {
|
|
60
|
+
original[level === "info" ? "log" : level](...args);
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const text = format(...args);
|
|
64
|
+
sink({
|
|
65
|
+
at: Date.now(),
|
|
66
|
+
plugin: where.plugin,
|
|
67
|
+
level,
|
|
68
|
+
method: where.method ?? "",
|
|
69
|
+
caller: where.caller ?? "",
|
|
70
|
+
callId: where.callId ?? "",
|
|
71
|
+
chatId: where.chatId ?? "",
|
|
72
|
+
text: text.length > textLimit ? text.slice(0, textLimit) : text,
|
|
73
|
+
data: args.map(carryable)
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
function carryable(value) {
|
|
77
|
+
if (value instanceof Error) {
|
|
78
|
+
return { name: value.name, message: value.message, stack: value.stack ?? "" };
|
|
79
|
+
}
|
|
80
|
+
if (typeof value === "bigint")
|
|
81
|
+
return value.toString();
|
|
82
|
+
if (typeof value === "function" || typeof value === "symbol" || value === undefined)
|
|
83
|
+
return String(value);
|
|
84
|
+
try {
|
|
85
|
+
JSON.stringify(value);
|
|
86
|
+
return value;
|
|
87
|
+
} catch {
|
|
88
|
+
return format(value);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// src/runtime.ts
|
|
93
|
+
import { AsyncLocalStorage as AsyncLocalStorage3 } from "async_hooks";
|
|
94
|
+
import { mkdir as mkdir3, readdir as readdir3 } from "fs/promises";
|
|
95
|
+
import { join as join3 } from "path";
|
|
96
|
+
import { z as z5 } from "zod";
|
|
97
|
+
|
|
98
|
+
// src/buckets.ts
|
|
99
|
+
import { watch } from "fs";
|
|
100
|
+
import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from "fs/promises";
|
|
101
|
+
import { join } from "path";
|
|
102
|
+
|
|
103
|
+
// src/store.ts
|
|
104
|
+
class Store {
|
|
105
|
+
options;
|
|
106
|
+
kind;
|
|
107
|
+
name;
|
|
108
|
+
tag;
|
|
109
|
+
plugins = new Set;
|
|
110
|
+
constructor(options, kind, name) {
|
|
111
|
+
this.options = options;
|
|
112
|
+
this.kind = kind;
|
|
113
|
+
this.name = name;
|
|
114
|
+
this.tag = `${options.namespace}.${kind}.${name}`;
|
|
115
|
+
}
|
|
116
|
+
close() {}
|
|
117
|
+
get sql() {
|
|
118
|
+
return this.options.database.sql;
|
|
119
|
+
}
|
|
120
|
+
async emit(event, payload) {
|
|
121
|
+
for (const plugin of this.plugins)
|
|
122
|
+
await this.options.events.emit(`${plugin}.${this.name}.${event}`, payload);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function tableName(kind, name) {
|
|
126
|
+
return `${kind}_${name.replace(/[^a-zA-Z0-9_]/g, "_")}`;
|
|
127
|
+
}
|
|
128
|
+
var safeName = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
129
|
+
|
|
130
|
+
// src/tracking.ts
|
|
131
|
+
import { AsyncLocalStorage as AsyncLocalStorage2 } from "async_hooks";
|
|
132
|
+
var current = new AsyncLocalStorage2;
|
|
133
|
+
function currentAccess() {
|
|
134
|
+
return current.getStore();
|
|
135
|
+
}
|
|
136
|
+
async function tracked(readOnly, announce, work) {
|
|
137
|
+
const outer = current.getStore();
|
|
138
|
+
const outermost = !outer;
|
|
139
|
+
const access = outer && (outer.readOnly || !readOnly) ? outer : outer ? { ...outer, readOnly: true } : { reads: new Set, writes: new Set, readOnly, done: { value: false }, announce };
|
|
140
|
+
try {
|
|
141
|
+
const value = access === outer ? await work() : await current.run(access, work);
|
|
142
|
+
return { ok: true, value, access, outermost };
|
|
143
|
+
} catch (error) {
|
|
144
|
+
return { ok: false, error, access, outermost };
|
|
145
|
+
} finally {
|
|
146
|
+
if (outermost)
|
|
147
|
+
access.done.value = true;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function isolated(work) {
|
|
151
|
+
return current.run({ reads: new Set, writes: new Set, readOnly: true, done: { value: false }, announce: () => {} }, work);
|
|
152
|
+
}
|
|
153
|
+
function recordRead(tag) {
|
|
154
|
+
current.getStore()?.reads.add(tag);
|
|
155
|
+
}
|
|
156
|
+
function assertWritable(tag) {
|
|
157
|
+
if (current.getStore()?.readOnly) {
|
|
158
|
+
throw new MethodError("FORBIDDEN", `A query can't change data (it tried to write ${tag}).`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
function recordWrite(tag) {
|
|
162
|
+
const access = current.getStore();
|
|
163
|
+
if (!access)
|
|
164
|
+
return;
|
|
165
|
+
if (access.done.value)
|
|
166
|
+
access.announce([tag]);
|
|
167
|
+
else
|
|
168
|
+
access.writes.add(tag);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// src/buckets.ts
|
|
172
|
+
var metaFile = "meta.json";
|
|
173
|
+
var tmpFolder = ".tmp";
|
|
174
|
+
var watchSettleMs = 200;
|
|
175
|
+
var scanEveryMs = 60000;
|
|
176
|
+
|
|
177
|
+
class Bucket extends Store {
|
|
178
|
+
schema;
|
|
179
|
+
table;
|
|
180
|
+
searchTable;
|
|
181
|
+
folder;
|
|
182
|
+
watcher = null;
|
|
183
|
+
settling = null;
|
|
184
|
+
scanning = Promise.resolve();
|
|
185
|
+
lastScan = 0;
|
|
186
|
+
constructor(options, name, schema) {
|
|
187
|
+
super(options, "buckets", name);
|
|
188
|
+
this.schema = schema;
|
|
189
|
+
if ("folder" in schema.shape)
|
|
190
|
+
throw new Error(`bucket ${name}: "folder" is the file's place, not a meta field`);
|
|
191
|
+
this.table = tableName("buckets", name);
|
|
192
|
+
this.searchTable = `${this.table}_search`;
|
|
193
|
+
this.folder = join(options.dir, "buckets", name);
|
|
194
|
+
}
|
|
195
|
+
get path() {
|
|
196
|
+
return this.folder;
|
|
197
|
+
}
|
|
198
|
+
async setup() {
|
|
199
|
+
await mkdir(join(this.folder, tmpFolder), { recursive: true });
|
|
200
|
+
await this.sql.unsafe(`create table if not exists "${this.table}" (
|
|
201
|
+
id text primary key,
|
|
202
|
+
name text not null,
|
|
203
|
+
folder text not null,
|
|
204
|
+
size integer not null,
|
|
205
|
+
type text not null,
|
|
206
|
+
meta text not null,
|
|
207
|
+
status text not null,
|
|
208
|
+
problem text,
|
|
209
|
+
revision integer not null,
|
|
210
|
+
file_print text not null,
|
|
211
|
+
meta_print text not null,
|
|
212
|
+
created_at text not null,
|
|
213
|
+
updated_at text not null
|
|
214
|
+
)`);
|
|
215
|
+
await this.sql.unsafe(`create virtual table if not exists "${this.searchTable}" using fts5(id unindexed, name, folder, text)`);
|
|
216
|
+
await this.scan();
|
|
217
|
+
this.watcher = watch(this.folder, { recursive: true }, (_, file) => {
|
|
218
|
+
if (String(file ?? "").startsWith(tmpFolder))
|
|
219
|
+
return;
|
|
220
|
+
if (this.settling)
|
|
221
|
+
clearTimeout(this.settling);
|
|
222
|
+
this.settling = setTimeout(() => void this.scan(), watchSettleMs);
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
async put(file) {
|
|
226
|
+
assertWritable(this.tag);
|
|
227
|
+
checkName(file.name);
|
|
228
|
+
const meta = this.schema.parse(file.meta ?? {});
|
|
229
|
+
const id = Bun.randomUUIDv7();
|
|
230
|
+
const staging = join(this.folder, tmpFolder, id);
|
|
231
|
+
await mkdir(staging, { recursive: true });
|
|
232
|
+
await Bun.write(join(staging, file.name), file.body);
|
|
233
|
+
await writeMeta(staging, { ...meta, ...file.folder ? { folder: file.folder } : {} });
|
|
234
|
+
await rename(staging, join(this.folder, id));
|
|
235
|
+
const { after } = await this.reindex(id);
|
|
236
|
+
return after;
|
|
237
|
+
}
|
|
238
|
+
async get(id) {
|
|
239
|
+
recordRead(this.tag);
|
|
240
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}" where id = $1`, [id]);
|
|
241
|
+
return rows[0] ? this.toFile(rows[0]) : null;
|
|
242
|
+
}
|
|
243
|
+
async file(id) {
|
|
244
|
+
const found = await this.get(id);
|
|
245
|
+
if (!found)
|
|
246
|
+
throw new MethodError("NOT_FOUND", `${this.name} has no file ${id}.`);
|
|
247
|
+
return Bun.file(found.path);
|
|
248
|
+
}
|
|
249
|
+
async list(options = {}) {
|
|
250
|
+
recordRead(this.tag);
|
|
251
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}"
|
|
252
|
+
where ($1 is null or folder = $1) and ($2 is null or status = $2)
|
|
253
|
+
order by created_at desc, id desc limit $3`, [options.folder ?? null, options.status ?? null, options.limit ?? 100]);
|
|
254
|
+
return rows.map((row) => this.toFile(row));
|
|
255
|
+
}
|
|
256
|
+
async search(query, options = {}) {
|
|
257
|
+
recordRead(this.tag);
|
|
258
|
+
const words = query.match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
259
|
+
if (words.length === 0)
|
|
260
|
+
return [];
|
|
261
|
+
const rows = await this.sql.unsafe(`select t.* from "${this.searchTable}" s join "${this.table}" t on t.id = s.id
|
|
262
|
+
where "${this.searchTable}" match $1 order by s.rank limit $2`, [words.map((word) => `"${word}"*`).join(" "), options.limit ?? 50]);
|
|
263
|
+
return rows.map((row) => this.toFile(row));
|
|
264
|
+
}
|
|
265
|
+
async update(id, meta, options = {}) {
|
|
266
|
+
assertWritable(this.tag);
|
|
267
|
+
return this.rewrite(id, options.revision, async (folder, current2) => {
|
|
268
|
+
const merged = Object.fromEntries(Object.entries({ ...current2, ...meta }).filter(([, value]) => value !== null));
|
|
269
|
+
const { folder: place, ...fields } = merged;
|
|
270
|
+
return { ...this.schema.parse(fields), ...place ? { folder: place } : {} };
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
async move(id, to, options = {}) {
|
|
274
|
+
assertWritable(this.tag);
|
|
275
|
+
if (to.name !== undefined)
|
|
276
|
+
checkName(to.name);
|
|
277
|
+
return this.rewrite(id, options.revision, async (folder, current2, name) => {
|
|
278
|
+
if (to.name !== undefined && to.name !== name)
|
|
279
|
+
await rename(join(folder, name), join(folder, to.name));
|
|
280
|
+
const { folder: _, ...fields } = current2;
|
|
281
|
+
return { ...fields, ...to.folder ?? current2.folder ? { folder: to.folder ?? current2.folder } : {} };
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
async delete(id) {
|
|
285
|
+
assertWritable(this.tag);
|
|
286
|
+
checkID(id);
|
|
287
|
+
await rm(join(this.folder, id), { recursive: true, force: true });
|
|
288
|
+
await this.reindex(id);
|
|
289
|
+
}
|
|
290
|
+
async tick(now) {
|
|
291
|
+
if (now - this.lastScan >= scanEveryMs)
|
|
292
|
+
await this.scan();
|
|
293
|
+
}
|
|
294
|
+
close() {
|
|
295
|
+
this.watcher?.close();
|
|
296
|
+
if (this.settling)
|
|
297
|
+
clearTimeout(this.settling);
|
|
298
|
+
}
|
|
299
|
+
scan() {
|
|
300
|
+
this.scanning = this.scanning.then(() => this.scanNow()).catch((error) => {
|
|
301
|
+
console.error(`[buckets] ${this.name}: scan failed:`, toCallError(error).message);
|
|
302
|
+
});
|
|
303
|
+
return this.scanning;
|
|
304
|
+
}
|
|
305
|
+
async scanNow() {
|
|
306
|
+
this.lastScan = Date.now();
|
|
307
|
+
const entries = await readdir(this.folder, { withFileTypes: true });
|
|
308
|
+
const ids = entries.filter((entry) => entry.isDirectory() && safeName.test(entry.name)).map((entry) => entry.name);
|
|
309
|
+
const rows = await this.sql.unsafe(`select id, file_print, meta_print from "${this.table}"`);
|
|
310
|
+
const known = new Map(rows.map((row) => [row.id, row]));
|
|
311
|
+
const present = new Set(ids);
|
|
312
|
+
const changed = [];
|
|
313
|
+
for (const id of ids) {
|
|
314
|
+
const row = known.get(id);
|
|
315
|
+
const prints = await fingerprints(join(this.folder, id));
|
|
316
|
+
if (!row || row.file_print !== prints.file || row.meta_print !== prints.meta)
|
|
317
|
+
changed.push(id);
|
|
318
|
+
}
|
|
319
|
+
const gone = rows.map((row) => row.id).filter((id) => !present.has(id));
|
|
320
|
+
for (const id of [...changed, ...gone])
|
|
321
|
+
await this.reindex(id, { announce: true });
|
|
322
|
+
}
|
|
323
|
+
async reindex(id, options = {}) {
|
|
324
|
+
const result = await this.options.database.transaction(async () => {
|
|
325
|
+
const before2 = (await this.sql.unsafe(`select * from "${this.table}" where id = $1`, [id]))[0];
|
|
326
|
+
const disk = await this.read(id);
|
|
327
|
+
if (!disk) {
|
|
328
|
+
if (!before2)
|
|
329
|
+
return { before: null, after: null };
|
|
330
|
+
await this.sql.unsafe(`delete from "${this.table}" where id = $1`, [id]);
|
|
331
|
+
await this.sql.unsafe(`delete from "${this.searchTable}" where id = $1`, [id]);
|
|
332
|
+
recordWrite(this.tag);
|
|
333
|
+
return { before: before2, after: null };
|
|
334
|
+
}
|
|
335
|
+
if (before2 && before2.file_print === disk.file_print && before2.meta_print === disk.meta_print) {
|
|
336
|
+
return { before: before2, after: before2 };
|
|
337
|
+
}
|
|
338
|
+
const row = {
|
|
339
|
+
...disk,
|
|
340
|
+
revision: (before2?.revision ?? 0) + 1,
|
|
341
|
+
created_at: before2?.created_at ?? disk.createdAt,
|
|
342
|
+
updated_at: disk.updatedAt
|
|
343
|
+
};
|
|
344
|
+
await this.sql.unsafe(`insert or replace into "${this.table}"
|
|
345
|
+
(id, name, folder, size, type, meta, status, problem, revision, file_print, meta_print, created_at, updated_at)
|
|
346
|
+
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, [
|
|
347
|
+
row.id,
|
|
348
|
+
row.name,
|
|
349
|
+
row.folder,
|
|
350
|
+
row.size,
|
|
351
|
+
row.type,
|
|
352
|
+
row.meta,
|
|
353
|
+
row.status,
|
|
354
|
+
row.problem,
|
|
355
|
+
row.revision,
|
|
356
|
+
row.file_print,
|
|
357
|
+
row.meta_print,
|
|
358
|
+
row.created_at,
|
|
359
|
+
row.updated_at
|
|
360
|
+
]);
|
|
361
|
+
await this.sql.unsafe(`delete from "${this.searchTable}" where id = $1`, [id]);
|
|
362
|
+
await this.sql.unsafe(`insert into "${this.searchTable}" (id, name, folder, text) values ($1, $2, $3, $4)`, [
|
|
363
|
+
id,
|
|
364
|
+
row.name,
|
|
365
|
+
row.folder,
|
|
366
|
+
searchText(JSON.parse(row.meta))
|
|
367
|
+
]);
|
|
368
|
+
recordWrite(this.tag);
|
|
369
|
+
return { before: before2, after: row };
|
|
370
|
+
});
|
|
371
|
+
const { before, after } = result;
|
|
372
|
+
if (before === after)
|
|
373
|
+
return { before: before && this.toFile(before), after: after && this.toFile(after) };
|
|
374
|
+
if (options.announce)
|
|
375
|
+
this.options.announce([this.tag]);
|
|
376
|
+
const file = after && this.toFile(after);
|
|
377
|
+
if (!before)
|
|
378
|
+
await this.emit("created", file);
|
|
379
|
+
else if (!after)
|
|
380
|
+
await this.emit("deleted", this.toFile(before));
|
|
381
|
+
else {
|
|
382
|
+
if (before.file_print !== after.file_print)
|
|
383
|
+
await this.emit("changed", file);
|
|
384
|
+
if (before.meta_print !== after.meta_print)
|
|
385
|
+
await this.emit("metaChanged", file);
|
|
386
|
+
}
|
|
387
|
+
return { before: before && this.toFile(before), after: file };
|
|
388
|
+
}
|
|
389
|
+
async read(id) {
|
|
390
|
+
const folder = join(this.folder, id);
|
|
391
|
+
const names = await readdir(folder).catch(() => null);
|
|
392
|
+
if (!names)
|
|
393
|
+
return null;
|
|
394
|
+
const files = names.filter((name) => name !== metaFile && !name.startsWith("."));
|
|
395
|
+
const prints = await fingerprints(folder);
|
|
396
|
+
const info = await stat(folder);
|
|
397
|
+
const file = files.length === 1 ? await stat(join(folder, files[0])) : null;
|
|
398
|
+
const disk = {
|
|
399
|
+
id,
|
|
400
|
+
name: files[0] ?? "",
|
|
401
|
+
folder: "",
|
|
402
|
+
size: file?.size ?? 0,
|
|
403
|
+
type: files.length === 1 ? Bun.file(join(folder, files[0])).type : "",
|
|
404
|
+
meta: "{}",
|
|
405
|
+
status: "empty",
|
|
406
|
+
problem: null,
|
|
407
|
+
file_print: prints.file,
|
|
408
|
+
meta_print: prints.meta,
|
|
409
|
+
createdAt: info.birthtime.toISOString(),
|
|
410
|
+
updatedAt: new Date(Math.max(info.mtimeMs, file?.mtimeMs ?? 0, prints.metaMtimeMs)).toISOString()
|
|
411
|
+
};
|
|
412
|
+
const raw = await readFile(join(folder, metaFile), "utf8").catch(() => null);
|
|
413
|
+
let meta = {};
|
|
414
|
+
if (raw !== null) {
|
|
415
|
+
try {
|
|
416
|
+
const parsed = JSON.parse(raw);
|
|
417
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
|
|
418
|
+
throw new Error("meta.json is not an object");
|
|
419
|
+
meta = parsed;
|
|
420
|
+
} catch (error) {
|
|
421
|
+
return { ...disk, status: "malformed", problem: `meta.json: ${error.message}` };
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
const { folder: place, ...fields } = meta;
|
|
425
|
+
disk.folder = typeof place === "string" ? place : "";
|
|
426
|
+
disk.meta = JSON.stringify(fields);
|
|
427
|
+
if (files.length !== 1) {
|
|
428
|
+
return { ...disk, status: "malformed", problem: `A file's folder holds one file and meta.json; found ${files.length}.` };
|
|
429
|
+
}
|
|
430
|
+
if (Object.keys(fields).length === 0)
|
|
431
|
+
return disk;
|
|
432
|
+
const checked = this.schema.safeParse(fields);
|
|
433
|
+
if (!checked.success)
|
|
434
|
+
return { ...disk, status: "invalid", problem: toCallError(checked.error).message };
|
|
435
|
+
return { ...disk, status: "ok" };
|
|
436
|
+
}
|
|
437
|
+
async rewrite(id, revision, change) {
|
|
438
|
+
checkID(id);
|
|
439
|
+
const folder = join(this.folder, id);
|
|
440
|
+
await this.options.database.transaction(async () => {
|
|
441
|
+
const row = (await this.sql.unsafe(`select * from "${this.table}" where id = $1`, [id]))[0];
|
|
442
|
+
if (!row)
|
|
443
|
+
throw new MethodError("NOT_FOUND", `${this.name} has no file ${id}.`);
|
|
444
|
+
if (revision !== undefined && row.revision !== revision) {
|
|
445
|
+
throw new MethodError("VALIDATION", `${this.name} ${id} changed since it was read (revision ${row.revision}, not ${revision}).`);
|
|
446
|
+
}
|
|
447
|
+
const raw = await readFile(join(folder, metaFile), "utf8").catch(() => "{}");
|
|
448
|
+
await writeMeta(folder, await change(folder, JSON.parse(raw), row.name));
|
|
449
|
+
});
|
|
450
|
+
const { after } = await this.reindex(id);
|
|
451
|
+
return after;
|
|
452
|
+
}
|
|
453
|
+
toFile(row) {
|
|
454
|
+
return {
|
|
455
|
+
id: row.id,
|
|
456
|
+
name: row.name,
|
|
457
|
+
folder: row.folder,
|
|
458
|
+
path: join(this.folder, row.id, row.name),
|
|
459
|
+
size: row.size,
|
|
460
|
+
type: row.type,
|
|
461
|
+
meta: JSON.parse(row.meta),
|
|
462
|
+
status: row.status,
|
|
463
|
+
problem: row.problem,
|
|
464
|
+
revision: row.revision,
|
|
465
|
+
createdAt: row.created_at,
|
|
466
|
+
updatedAt: row.updated_at
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
async function writeMeta(folder, meta) {
|
|
471
|
+
const temp = join(folder, `.${metaFile}.${crypto.randomUUID()}`);
|
|
472
|
+
await writeFile(temp, JSON.stringify(meta, null, 2) + `
|
|
473
|
+
`);
|
|
474
|
+
await rename(temp, join(folder, metaFile));
|
|
475
|
+
}
|
|
476
|
+
async function fingerprints(folder) {
|
|
477
|
+
const names = (await readdir(folder).catch(() => [])).filter((name) => !name.startsWith(".")).sort();
|
|
478
|
+
const file = [];
|
|
479
|
+
let meta = "";
|
|
480
|
+
let metaMtimeMs = 0;
|
|
481
|
+
for (const name of names) {
|
|
482
|
+
const info = await stat(join(folder, name), { bigint: true }).catch(() => null);
|
|
483
|
+
if (!info)
|
|
484
|
+
continue;
|
|
485
|
+
const print = `${name}:${info.size}:${info.mtimeNs}`;
|
|
486
|
+
if (name === metaFile) {
|
|
487
|
+
meta = print;
|
|
488
|
+
metaMtimeMs = Number(info.mtimeMs);
|
|
489
|
+
} else
|
|
490
|
+
file.push(print);
|
|
491
|
+
}
|
|
492
|
+
return { file: file.join("|"), meta, metaMtimeMs };
|
|
493
|
+
}
|
|
494
|
+
function searchText(value) {
|
|
495
|
+
if (typeof value === "string" || typeof value === "number")
|
|
496
|
+
return String(value);
|
|
497
|
+
if (Array.isArray(value))
|
|
498
|
+
return value.map(searchText).join(" ");
|
|
499
|
+
if (value && typeof value === "object")
|
|
500
|
+
return Object.values(value).map(searchText).join(" ");
|
|
501
|
+
return "";
|
|
502
|
+
}
|
|
503
|
+
function checkID(id) {
|
|
504
|
+
if (!safeName.test(id))
|
|
505
|
+
throw new MethodError("VALIDATION", `Not a file id: ${id}`);
|
|
506
|
+
}
|
|
507
|
+
function checkName(name) {
|
|
508
|
+
if (!/^[^/\\\0.][^/\\\0]{0,254}$/.test(name) || name === metaFile) {
|
|
509
|
+
throw new MethodError("VALIDATION", `A file name has no slashes, doesn't start with a dot and isn't meta.json: ${name}`);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// src/cache.ts
|
|
514
|
+
var maxBytes = 1e6;
|
|
515
|
+
var keepMs = 30 * 86400000;
|
|
516
|
+
|
|
517
|
+
class AnswerCache {
|
|
518
|
+
database;
|
|
519
|
+
namespace;
|
|
520
|
+
announce;
|
|
521
|
+
inflight = new Map;
|
|
522
|
+
constructor(database, namespace, announce) {
|
|
523
|
+
this.database = database;
|
|
524
|
+
this.namespace = namespace;
|
|
525
|
+
this.announce = announce;
|
|
526
|
+
}
|
|
527
|
+
async setup() {
|
|
528
|
+
await this.database.sql.unsafe(`create table if not exists cached_answers (
|
|
529
|
+
key text primary key,
|
|
530
|
+
value text not null,
|
|
531
|
+
checked_at integer not null
|
|
532
|
+
)`);
|
|
533
|
+
}
|
|
534
|
+
async read(key, options, fetch) {
|
|
535
|
+
const tag = `${this.namespace}.cached.${Bun.hash(key).toString(36)}`;
|
|
536
|
+
recordRead(tag);
|
|
537
|
+
const [row] = await this.database.sql.unsafe(`select value, checked_at from cached_answers where key = $1`, [key]);
|
|
538
|
+
if (!row)
|
|
539
|
+
return this.refresh(key, tag, options, fetch);
|
|
540
|
+
if (Date.now() - row.checked_at >= options.fresh) {
|
|
541
|
+
this.refresh(key, tag, options, fetch).catch((error) => record("error", [error]));
|
|
542
|
+
}
|
|
543
|
+
return JSON.parse(row.value);
|
|
544
|
+
}
|
|
545
|
+
async idle() {
|
|
546
|
+
while (this.inflight.size)
|
|
547
|
+
await Promise.allSettled([...this.inflight.values()]);
|
|
548
|
+
}
|
|
549
|
+
refresh(key, tag, options, fetch) {
|
|
550
|
+
const running = this.inflight.get(key);
|
|
551
|
+
if (running)
|
|
552
|
+
return running;
|
|
553
|
+
const started = Date.now();
|
|
554
|
+
const work = isolated(async () => withTimeout(Promise.resolve(fetch()), options.timeout)).then(async (value) => {
|
|
555
|
+
await this.save(key, tag, value ?? null, started);
|
|
556
|
+
return value ?? null;
|
|
557
|
+
}, async (error) => {
|
|
558
|
+
await this.database.transaction(() => this.database.sql.unsafe(`update cached_answers set checked_at = $1 where key = $2`, [Date.now(), key]));
|
|
559
|
+
throw error;
|
|
560
|
+
}).finally(() => this.inflight.delete(key));
|
|
561
|
+
this.inflight.set(key, work);
|
|
562
|
+
return work;
|
|
563
|
+
}
|
|
564
|
+
async save(key, tag, value, started) {
|
|
565
|
+
const text = JSON.stringify(value);
|
|
566
|
+
if (text.length > maxBytes)
|
|
567
|
+
return;
|
|
568
|
+
const changed = await this.database.transaction(async () => {
|
|
569
|
+
const [before] = await this.database.sql.unsafe(`select value, checked_at from cached_answers where key = $1`, [key]);
|
|
570
|
+
if (before && before.checked_at > started)
|
|
571
|
+
return false;
|
|
572
|
+
const now = Date.now();
|
|
573
|
+
await this.database.sql.unsafe(`insert into cached_answers (key, value, checked_at) values ($1, $2, $3)
|
|
574
|
+
on conflict (key) do update set value = excluded.value, checked_at = excluded.checked_at`, [key, text, now]);
|
|
575
|
+
await this.database.sql.unsafe(`delete from cached_answers where checked_at < $1`, [now - keepMs]);
|
|
576
|
+
return before?.value !== text;
|
|
577
|
+
});
|
|
578
|
+
if (changed)
|
|
579
|
+
this.announce([tag]);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
function cacheKey(method, root, input) {
|
|
583
|
+
return JSON.stringify([method, root, sorted(input)]);
|
|
584
|
+
}
|
|
585
|
+
function sorted(value) {
|
|
586
|
+
if (Array.isArray(value))
|
|
587
|
+
return value.map(sorted);
|
|
588
|
+
if (value === null || typeof value !== "object")
|
|
589
|
+
return value;
|
|
590
|
+
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sorted(value[key])]));
|
|
591
|
+
}
|
|
592
|
+
function withTimeout(work, ms) {
|
|
593
|
+
let timer;
|
|
594
|
+
const timeout = new Promise((_, reject) => {
|
|
595
|
+
timer = setTimeout(() => reject(new MethodError("INTERNAL", `The answer took longer than ${Math.round(ms / 1000)} s.`)), ms);
|
|
596
|
+
});
|
|
597
|
+
return Promise.race([work, timeout]).finally(() => clearTimeout(timer));
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// src/collections.ts
|
|
601
|
+
class Collection {
|
|
602
|
+
database;
|
|
603
|
+
events;
|
|
604
|
+
plugin;
|
|
605
|
+
name;
|
|
606
|
+
schema;
|
|
607
|
+
table;
|
|
608
|
+
tag;
|
|
609
|
+
constructor(database, events, plugin, namespace, name, schema) {
|
|
610
|
+
this.database = database;
|
|
611
|
+
this.events = events;
|
|
612
|
+
this.plugin = plugin;
|
|
613
|
+
this.name = name;
|
|
614
|
+
this.schema = schema;
|
|
615
|
+
this.table = name.replace(/[^a-zA-Z0-9_]/g, "_");
|
|
616
|
+
this.tag = `${namespace}.${name}`;
|
|
617
|
+
}
|
|
618
|
+
async setup() {
|
|
619
|
+
await this.sql.unsafe(`create table if not exists "${this.table}" (
|
|
620
|
+
id text primary key,
|
|
621
|
+
data text not null,
|
|
622
|
+
created_at text not null,
|
|
623
|
+
updated_at text not null
|
|
624
|
+
)`);
|
|
625
|
+
}
|
|
626
|
+
get sql() {
|
|
627
|
+
return this.database.sql;
|
|
628
|
+
}
|
|
629
|
+
async create(input) {
|
|
630
|
+
assertWritable(this.tag);
|
|
631
|
+
const record2 = await this.database.transaction(async () => {
|
|
632
|
+
const data = this.schema.parse(input);
|
|
633
|
+
const now = new Date().toISOString();
|
|
634
|
+
const record3 = { ...data, id: crypto.randomUUID(), createdAt: now, updatedAt: now };
|
|
635
|
+
await this.sql.unsafe(`insert into "${this.table}" (id, data, created_at, updated_at) values ($1, $2, $3, $4)`, [
|
|
636
|
+
record3.id,
|
|
637
|
+
JSON.stringify(data),
|
|
638
|
+
now,
|
|
639
|
+
now
|
|
640
|
+
]);
|
|
641
|
+
recordWrite(this.tag);
|
|
642
|
+
return record3;
|
|
643
|
+
});
|
|
644
|
+
await this.events.emit(this.eventName("created"), record2);
|
|
645
|
+
return record2;
|
|
646
|
+
}
|
|
647
|
+
async get(id) {
|
|
648
|
+
recordRead(this.tag);
|
|
649
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}" where id = $1`, [id]);
|
|
650
|
+
return rows[0] ? this.toRecord(rows[0]) : null;
|
|
651
|
+
}
|
|
652
|
+
async update(id, patch) {
|
|
653
|
+
assertWritable(this.tag);
|
|
654
|
+
const { before, after } = await this.database.transaction(async () => {
|
|
655
|
+
const before2 = await this.get(id);
|
|
656
|
+
if (!before2)
|
|
657
|
+
throw new Error(`${this.name} ${id} not found`);
|
|
658
|
+
const { id: _, createdAt, updatedAt: __, ...fields } = before2;
|
|
659
|
+
const data = this.schema.parse({ ...fields, ...patch });
|
|
660
|
+
const now = new Date().toISOString();
|
|
661
|
+
await this.sql.unsafe(`update "${this.table}" set data = $1, updated_at = $2 where id = $3`, [JSON.stringify(data), now, id]);
|
|
662
|
+
recordWrite(this.tag);
|
|
663
|
+
return { before: before2, after: { ...data, id, createdAt, updatedAt: now } };
|
|
664
|
+
});
|
|
665
|
+
await this.events.emit(this.eventName("updated"), { before, after });
|
|
666
|
+
return after;
|
|
667
|
+
}
|
|
668
|
+
async delete(id) {
|
|
669
|
+
assertWritable(this.tag);
|
|
670
|
+
const before = await this.database.transaction(async () => {
|
|
671
|
+
const before2 = await this.get(id);
|
|
672
|
+
if (!before2)
|
|
673
|
+
return null;
|
|
674
|
+
await this.sql.unsafe(`delete from "${this.table}" where id = $1`, [id]);
|
|
675
|
+
recordWrite(this.tag);
|
|
676
|
+
return before2;
|
|
677
|
+
});
|
|
678
|
+
if (before)
|
|
679
|
+
await this.events.emit(this.eventName("deleted"), before);
|
|
680
|
+
}
|
|
681
|
+
async list(options = {}) {
|
|
682
|
+
recordRead(this.tag);
|
|
683
|
+
const { sql, params } = this.select(options);
|
|
684
|
+
const rows = await this.sql.unsafe(sql, params);
|
|
685
|
+
return rows.map((row) => this.toRecord(row));
|
|
686
|
+
}
|
|
687
|
+
async deleteMany(options = {}) {
|
|
688
|
+
assertWritable(this.tag);
|
|
689
|
+
const deleted = await this.database.transaction(async () => {
|
|
690
|
+
const { sql, params } = this.select(options);
|
|
691
|
+
const rows = (await this.sql.unsafe(sql, params)).map((row) => this.toRecord(row));
|
|
692
|
+
if (!rows.length)
|
|
693
|
+
return rows;
|
|
694
|
+
const ids = rows.map((row) => row.id);
|
|
695
|
+
await this.sql.unsafe(`delete from "${this.table}" where id in (${ids.map((_, i) => `$${i + 1}`).join(", ")})`, ids);
|
|
696
|
+
recordWrite(this.tag);
|
|
697
|
+
return rows;
|
|
698
|
+
});
|
|
699
|
+
for (const row of deleted)
|
|
700
|
+
await this.events.emit(this.eventName("deleted"), row);
|
|
701
|
+
return deleted.length;
|
|
702
|
+
}
|
|
703
|
+
select(options) {
|
|
704
|
+
const params = [];
|
|
705
|
+
const field = (name) => {
|
|
706
|
+
if (/["\\]/.test(name))
|
|
707
|
+
throw new Error(`bad field name: ${name}`);
|
|
708
|
+
params.push(`$."${name}"`);
|
|
709
|
+
return `json_extract(data, $${params.length})`;
|
|
710
|
+
};
|
|
711
|
+
const clauses = Object.entries(options.where ?? {}).map(([name, value]) => {
|
|
712
|
+
const path = field(name);
|
|
713
|
+
if (value === null)
|
|
714
|
+
return `${path} is null`;
|
|
715
|
+
params.push(typeof value === "boolean" ? Number(value) : value);
|
|
716
|
+
return `${path} = $${params.length}`;
|
|
717
|
+
});
|
|
718
|
+
const where = clauses.length ? `where ${clauses.join(" and ")}` : "";
|
|
719
|
+
const orderBy = options.orderBy ?? "createdAt";
|
|
720
|
+
const columns = { id: "id", createdAt: "created_at", updatedAt: "updated_at" };
|
|
721
|
+
const column = columns[orderBy] ?? field(orderBy);
|
|
722
|
+
const order = options.order === "asc" ? "asc" : "desc";
|
|
723
|
+
params.push(options.limit ?? 100, options.offset ?? 0);
|
|
724
|
+
return {
|
|
725
|
+
sql: `select * from "${this.table}" ${where} order by ${column} ${order}, id ${order} limit $${params.length - 1} offset $${params.length}`,
|
|
726
|
+
params
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
eventName(kind) {
|
|
730
|
+
return `${this.plugin}.${this.name}.${kind}`;
|
|
731
|
+
}
|
|
732
|
+
toRecord(row) {
|
|
733
|
+
return { ...JSON.parse(row.data), id: row.id, createdAt: row.created_at, updatedAt: row.updated_at };
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
// src/cron.ts
|
|
738
|
+
var ranges = { second: [0, 59], minute: [0, 59], hour: [0, 23], day: [1, 31], month: [1, 12], weekday: [0, 7] };
|
|
739
|
+
function parseSchedule(source) {
|
|
740
|
+
const fields = source.trim().split(/\s+/);
|
|
741
|
+
if (fields.length !== 5 && fields.length !== 6) {
|
|
742
|
+
throw new Error(`"${source}" has ${fields.length} fields; a cron has 5 (minute hour day month weekday), or 6 with seconds first`);
|
|
743
|
+
}
|
|
744
|
+
const [second, minute, hour, day, month, weekday] = fields.length === 6 ? fields : [null, ...fields];
|
|
745
|
+
const weekdays = field(weekday, "weekday");
|
|
746
|
+
if (weekdays.has(7)) {
|
|
747
|
+
weekdays.delete(7);
|
|
748
|
+
weekdays.add(0);
|
|
749
|
+
}
|
|
750
|
+
return {
|
|
751
|
+
seconds: second === null ? null : field(second, "second"),
|
|
752
|
+
minutes: field(minute, "minute"),
|
|
753
|
+
hours: field(hour, "hour"),
|
|
754
|
+
days: field(day, "day"),
|
|
755
|
+
months: field(month, "month"),
|
|
756
|
+
weekdays,
|
|
757
|
+
daysRestricted: day !== "*",
|
|
758
|
+
weekdaysRestricted: weekday !== "*"
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
function field(source, name) {
|
|
762
|
+
const [min, max] = ranges[name];
|
|
763
|
+
const out = new Set;
|
|
764
|
+
for (const part of source.split(",")) {
|
|
765
|
+
const [span, stepText] = part.split("/");
|
|
766
|
+
const step = stepText === undefined ? 1 : whole(stepText, name);
|
|
767
|
+
if (step < 1)
|
|
768
|
+
throw new Error(`the ${name} step in "${part}" must be at least 1`);
|
|
769
|
+
let [from, to] = [min, max];
|
|
770
|
+
if (span !== "*") {
|
|
771
|
+
const [a, b] = span.split("-");
|
|
772
|
+
from = whole(a, name);
|
|
773
|
+
to = b === undefined ? stepText === undefined ? from : max : whole(b, name);
|
|
774
|
+
}
|
|
775
|
+
if (from < min || to > max || from > to)
|
|
776
|
+
throw new Error(`the ${name} "${part}" is outside ${min}-${max}`);
|
|
777
|
+
for (let v = from;v <= to; v += step)
|
|
778
|
+
out.add(v);
|
|
779
|
+
}
|
|
780
|
+
return out;
|
|
781
|
+
}
|
|
782
|
+
function whole(text, name) {
|
|
783
|
+
if (!/^\d+$/.test(text))
|
|
784
|
+
throw new Error(`the ${name} "${text}" isn't a whole number`);
|
|
785
|
+
return Number(text);
|
|
786
|
+
}
|
|
787
|
+
function checkTimezone(zone) {
|
|
788
|
+
new Intl.DateTimeFormat("en-US", { timeZone: zone });
|
|
789
|
+
}
|
|
790
|
+
var formats = new Map;
|
|
791
|
+
function wall(ms, zone) {
|
|
792
|
+
let format2 = formats.get(zone);
|
|
793
|
+
if (!format2) {
|
|
794
|
+
format2 = new Intl.DateTimeFormat("en-US", {
|
|
795
|
+
timeZone: zone,
|
|
796
|
+
hourCycle: "h23",
|
|
797
|
+
weekday: "short",
|
|
798
|
+
year: "numeric",
|
|
799
|
+
month: "numeric",
|
|
800
|
+
day: "numeric",
|
|
801
|
+
hour: "numeric",
|
|
802
|
+
minute: "numeric",
|
|
803
|
+
second: "numeric"
|
|
804
|
+
});
|
|
805
|
+
formats.set(zone, format2);
|
|
806
|
+
}
|
|
807
|
+
const parts = {};
|
|
808
|
+
for (const p of format2.formatToParts(new Date(ms)))
|
|
809
|
+
parts[p.type] = p.value;
|
|
810
|
+
return {
|
|
811
|
+
year: Number(parts.year),
|
|
812
|
+
month: Number(parts.month),
|
|
813
|
+
day: Number(parts.day),
|
|
814
|
+
weekday: ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"].indexOf(parts.weekday),
|
|
815
|
+
hour: Number(parts.hour),
|
|
816
|
+
minute: Number(parts.minute),
|
|
817
|
+
second: Number(parts.second)
|
|
818
|
+
};
|
|
819
|
+
}
|
|
820
|
+
var minute = 60000;
|
|
821
|
+
var horizon = 4 * 366 * 24 * 60 * minute;
|
|
822
|
+
function nextDue(s, after, zone) {
|
|
823
|
+
let at = Math.floor(after / minute) * minute;
|
|
824
|
+
const end = after + horizon;
|
|
825
|
+
if (s.seconds) {
|
|
826
|
+
const first = firstSecond(s, at, after, zone);
|
|
827
|
+
if (first !== null)
|
|
828
|
+
return first;
|
|
829
|
+
}
|
|
830
|
+
at += minute;
|
|
831
|
+
while (at < end) {
|
|
832
|
+
const w = wall(at, zone);
|
|
833
|
+
if (!s.months.has(w.month)) {
|
|
834
|
+
at += untilNextDay(w);
|
|
835
|
+
continue;
|
|
836
|
+
}
|
|
837
|
+
if (!dayMatches(s, w)) {
|
|
838
|
+
at += untilNextDay(w);
|
|
839
|
+
continue;
|
|
840
|
+
}
|
|
841
|
+
if (!s.hours.has(w.hour)) {
|
|
842
|
+
at += (60 - w.minute) * minute;
|
|
843
|
+
continue;
|
|
844
|
+
}
|
|
845
|
+
if (!s.minutes.has(w.minute) || repeated(at, w, zone)) {
|
|
846
|
+
at += minute;
|
|
847
|
+
continue;
|
|
848
|
+
}
|
|
849
|
+
if (!s.seconds)
|
|
850
|
+
return at;
|
|
851
|
+
const first = firstSecond(s, at, at - 1, zone);
|
|
852
|
+
if (first !== null)
|
|
853
|
+
return first;
|
|
854
|
+
at += minute;
|
|
855
|
+
}
|
|
856
|
+
return null;
|
|
857
|
+
}
|
|
858
|
+
function firstSecond(s, start, after, zone) {
|
|
859
|
+
const w = wall(start, zone);
|
|
860
|
+
if (!s.months.has(w.month) || !dayMatches(s, w) || !s.hours.has(w.hour) || !s.minutes.has(w.minute) || repeated(start, w, zone))
|
|
861
|
+
return null;
|
|
862
|
+
for (let second = 0;second < 60; second++) {
|
|
863
|
+
const at = start + second * 1000;
|
|
864
|
+
if (s.seconds.has(second) && at > after)
|
|
865
|
+
return at;
|
|
866
|
+
}
|
|
867
|
+
return null;
|
|
868
|
+
}
|
|
869
|
+
function dayMatches(s, w) {
|
|
870
|
+
const day = s.days.has(w.day), weekday = s.weekdays.has(w.weekday);
|
|
871
|
+
if (s.daysRestricted && s.weekdaysRestricted)
|
|
872
|
+
return day || weekday;
|
|
873
|
+
return day && weekday;
|
|
874
|
+
}
|
|
875
|
+
function untilNextDay(w) {
|
|
876
|
+
return ((23 - w.hour) * 60 + (60 - w.minute)) * minute;
|
|
877
|
+
}
|
|
878
|
+
function repeated(at, w, zone) {
|
|
879
|
+
for (const back of [60, 120]) {
|
|
880
|
+
const earlier = wall(at - back * minute, zone);
|
|
881
|
+
if (earlier.day === w.day && earlier.hour === w.hour && earlier.minute === w.minute)
|
|
882
|
+
return true;
|
|
883
|
+
}
|
|
884
|
+
return false;
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// src/database.ts
|
|
888
|
+
var {SQL } = globalThis.Bun;
|
|
889
|
+
|
|
890
|
+
class Database {
|
|
891
|
+
sql;
|
|
892
|
+
writes = Promise.resolve();
|
|
893
|
+
constructor(sql) {
|
|
894
|
+
this.sql = sql;
|
|
895
|
+
}
|
|
896
|
+
static async open(path) {
|
|
897
|
+
const sql = new SQL(`sqlite://${path}`);
|
|
898
|
+
await sql.unsafe("pragma busy_timeout = 5000");
|
|
899
|
+
await sql.unsafe("pragma journal_mode = wal");
|
|
900
|
+
return new Database(sql);
|
|
901
|
+
}
|
|
902
|
+
transaction(work) {
|
|
903
|
+
const result = this.writes.then(() => this.run(work), () => this.run(work));
|
|
904
|
+
this.writes = result.catch(() => {});
|
|
905
|
+
return result;
|
|
906
|
+
}
|
|
907
|
+
close() {
|
|
908
|
+
this.sql.close();
|
|
909
|
+
}
|
|
910
|
+
async run(work) {
|
|
911
|
+
await this.sql.unsafe("begin immediate");
|
|
912
|
+
try {
|
|
913
|
+
const value = await work();
|
|
914
|
+
await this.sql.unsafe("commit");
|
|
915
|
+
return value;
|
|
916
|
+
} catch (error) {
|
|
917
|
+
await this.sql.unsafe("rollback").catch(() => {});
|
|
918
|
+
throw error;
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
// src/events.ts
|
|
924
|
+
class EventBus {
|
|
925
|
+
listeners = new Map;
|
|
926
|
+
all = [];
|
|
927
|
+
on(name, listener) {
|
|
928
|
+
const list = this.listeners.get(name) ?? [];
|
|
929
|
+
list.push(listener);
|
|
930
|
+
this.listeners.set(name, list);
|
|
931
|
+
}
|
|
932
|
+
onAny(listener) {
|
|
933
|
+
this.all.push(listener);
|
|
934
|
+
}
|
|
935
|
+
async emit(name, payload) {
|
|
936
|
+
const event = { name, payload, at: new Date().toISOString() };
|
|
937
|
+
const listeners = [...this.listeners.get(name) ?? [], ...this.all];
|
|
938
|
+
const results = await Promise.allSettled(listeners.map(async (listener) => listener(event)));
|
|
939
|
+
for (const result of results) {
|
|
940
|
+
if (result.status === "rejected")
|
|
941
|
+
console.error(`[events] ${name}:`, result.reason);
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// src/generated.ts
|
|
947
|
+
import { isAbsolute } from "path";
|
|
948
|
+
import { z } from "zod";
|
|
949
|
+
var limit = z.number().int().positive().max(500).optional();
|
|
950
|
+
var ttl = z.union([z.number().positive(), z.string().regex(/^\d+(\.\d+)?[smhd]$/)]).optional();
|
|
951
|
+
function kvMethods(name, kv, schema) {
|
|
952
|
+
const key = z.string().min(1);
|
|
953
|
+
const entry = z.object({ key: z.string(), value: schema, expiresAt: z.string().nullable(), createdAt: z.string(), updatedAt: z.string() });
|
|
954
|
+
return {
|
|
955
|
+
get: {
|
|
956
|
+
kind: "query",
|
|
957
|
+
description: `Get one ${name} entry by key, or null`,
|
|
958
|
+
input: z.object({ key }),
|
|
959
|
+
output: entry.nullable(),
|
|
960
|
+
run: ({ key: key2 }) => kv.entry(key2)
|
|
961
|
+
},
|
|
962
|
+
list: {
|
|
963
|
+
kind: "query",
|
|
964
|
+
description: `List ${name} entries in key order, optionally by prefix`,
|
|
965
|
+
input: z.object({ prefix: z.string().optional(), limit }),
|
|
966
|
+
output: z.array(entry),
|
|
967
|
+
run: (input) => kv.list(input)
|
|
968
|
+
},
|
|
969
|
+
set: {
|
|
970
|
+
kind: "mutation",
|
|
971
|
+
description: `Set a ${name} key; ttl ("5m", "1h", "7d" or ms) makes it expire`,
|
|
972
|
+
input: z.object({ key, value: schema, ttl }),
|
|
973
|
+
output: entry,
|
|
974
|
+
run: ({ key: key2, value, ttl: ttl2 }) => kv.set(key2, value, { ttl: ttl2 })
|
|
975
|
+
},
|
|
976
|
+
delete: {
|
|
977
|
+
kind: "mutation",
|
|
978
|
+
description: `Delete a ${name} key`,
|
|
979
|
+
input: z.object({ key }),
|
|
980
|
+
output: z.null(),
|
|
981
|
+
confirm: "Delete this key?",
|
|
982
|
+
run: async ({ key: key2 }) => (await kv.delete(key2), null)
|
|
983
|
+
}
|
|
984
|
+
};
|
|
985
|
+
}
|
|
986
|
+
function streamMethods(name, stream, schema) {
|
|
987
|
+
const key = z.string().min(1);
|
|
988
|
+
const line = z.object({ id: z.string(), key: z.string(), offset: z.number(), at: z.string(), data: schema });
|
|
989
|
+
return {
|
|
990
|
+
tail: {
|
|
991
|
+
kind: "stream",
|
|
992
|
+
description: `Follow a ${name} key live: its last lines (100 unless last says), then each new one, until it ends`,
|
|
993
|
+
input: z.object({ key, from: z.number().int().min(0).optional(), last: z.number().int().positive().max(500).optional() }),
|
|
994
|
+
run: ({ key: key2, from, last }, signal) => stream.follow(key2, { from, last, signal })
|
|
995
|
+
},
|
|
996
|
+
read: {
|
|
997
|
+
kind: "query",
|
|
998
|
+
description: `Read a ${name} key's lines from an offset`,
|
|
999
|
+
input: z.object({ key, from: z.number().int().min(0).optional(), limit }),
|
|
1000
|
+
output: z.array(line),
|
|
1001
|
+
run: ({ key: key2, ...options }) => stream.read(key2, options)
|
|
1002
|
+
},
|
|
1003
|
+
keys: {
|
|
1004
|
+
kind: "query",
|
|
1005
|
+
description: `List ${name} keys, most recently written first`,
|
|
1006
|
+
input: z.object({ limit }),
|
|
1007
|
+
output: z.array(z.object({ key: z.string(), lines: z.number(), bytes: z.number(), endedAt: z.string().nullable(), updatedAt: z.string() })),
|
|
1008
|
+
run: (input) => stream.keys(input)
|
|
1009
|
+
},
|
|
1010
|
+
append: {
|
|
1011
|
+
kind: "mutation",
|
|
1012
|
+
description: `Append a line to a ${name} key`,
|
|
1013
|
+
input: z.object({ key, data: schema }),
|
|
1014
|
+
output: line,
|
|
1015
|
+
run: ({ key: key2, data }) => stream.append(key2, data)
|
|
1016
|
+
},
|
|
1017
|
+
end: {
|
|
1018
|
+
kind: "mutation",
|
|
1019
|
+
description: `End a ${name} key: followers stop and appends fail`,
|
|
1020
|
+
input: z.object({ key }),
|
|
1021
|
+
output: z.null(),
|
|
1022
|
+
run: async ({ key: key2 }) => (await stream.end(key2), null)
|
|
1023
|
+
},
|
|
1024
|
+
delete: {
|
|
1025
|
+
kind: "mutation",
|
|
1026
|
+
description: `Delete a ${name} key and its lines`,
|
|
1027
|
+
input: z.object({ key }),
|
|
1028
|
+
output: z.null(),
|
|
1029
|
+
confirm: "Delete this stream and its lines?",
|
|
1030
|
+
run: async ({ key: key2 }) => (await stream.delete(key2), null)
|
|
1031
|
+
}
|
|
1032
|
+
};
|
|
1033
|
+
}
|
|
1034
|
+
function bucketMethods(name, bucket, schema) {
|
|
1035
|
+
const id = z.string().min(1);
|
|
1036
|
+
const status = z.enum(["ok", "empty", "invalid", "malformed"]);
|
|
1037
|
+
const file = z.object({
|
|
1038
|
+
id: z.string(),
|
|
1039
|
+
name: z.string(),
|
|
1040
|
+
folder: z.string(),
|
|
1041
|
+
path: z.string(),
|
|
1042
|
+
size: z.number(),
|
|
1043
|
+
type: z.string(),
|
|
1044
|
+
meta: schema.partial(),
|
|
1045
|
+
status,
|
|
1046
|
+
problem: z.string().nullable(),
|
|
1047
|
+
revision: z.number(),
|
|
1048
|
+
createdAt: z.string(),
|
|
1049
|
+
updatedAt: z.string()
|
|
1050
|
+
});
|
|
1051
|
+
const revision = z.number().int().optional();
|
|
1052
|
+
return {
|
|
1053
|
+
list: {
|
|
1054
|
+
kind: "query",
|
|
1055
|
+
description: `List ${name} files, newest first, optionally in one folder or with one status`,
|
|
1056
|
+
input: z.object({ folder: z.string().optional(), status: status.optional(), limit }),
|
|
1057
|
+
output: z.array(file),
|
|
1058
|
+
run: (input) => bucket.list(input)
|
|
1059
|
+
},
|
|
1060
|
+
search: {
|
|
1061
|
+
kind: "query",
|
|
1062
|
+
description: `Search ${name} files by name, folder and meta`,
|
|
1063
|
+
input: z.object({ query: z.string(), limit }),
|
|
1064
|
+
output: z.array(file),
|
|
1065
|
+
run: ({ query, limit: limit2 }) => bucket.search(query, { limit: limit2 })
|
|
1066
|
+
},
|
|
1067
|
+
get: {
|
|
1068
|
+
kind: "query",
|
|
1069
|
+
description: `Get one ${name} file by id, or null`,
|
|
1070
|
+
input: z.object({ id }),
|
|
1071
|
+
output: file.nullable(),
|
|
1072
|
+
run: ({ id: id2 }) => bucket.get(id2)
|
|
1073
|
+
},
|
|
1074
|
+
put: {
|
|
1075
|
+
kind: "mutation",
|
|
1076
|
+
description: `Add a file to ${name}: its name, and text or an absolute path to copy`,
|
|
1077
|
+
input: z.object({
|
|
1078
|
+
name: z.string(),
|
|
1079
|
+
folder: z.string().optional(),
|
|
1080
|
+
text: z.string().optional(),
|
|
1081
|
+
from: z.string().refine(isAbsolute, "from is an absolute path").optional(),
|
|
1082
|
+
meta: schema.optional()
|
|
1083
|
+
}),
|
|
1084
|
+
output: file,
|
|
1085
|
+
run: async ({ name: name2, folder, text, from, meta }) => {
|
|
1086
|
+
if (text === undefined === (from === undefined))
|
|
1087
|
+
throw new MethodError("VALIDATION", "Send text or from, one of them.");
|
|
1088
|
+
return bucket.put({ name: name2, folder, meta, body: text ?? Bun.file(from) });
|
|
1089
|
+
}
|
|
1090
|
+
},
|
|
1091
|
+
update: {
|
|
1092
|
+
kind: "mutation",
|
|
1093
|
+
description: `Change fields of a ${name} file's meta; null removes one`,
|
|
1094
|
+
input: z.object({ id, meta: z.record(z.string(), z.unknown()), revision }),
|
|
1095
|
+
output: file,
|
|
1096
|
+
run: ({ id: id2, meta, revision: revision2 }) => bucket.update(id2, meta, { revision: revision2 })
|
|
1097
|
+
},
|
|
1098
|
+
move: {
|
|
1099
|
+
kind: "mutation",
|
|
1100
|
+
description: `Rename a ${name} file or move it to another folder`,
|
|
1101
|
+
input: z.object({ id, name: z.string().optional(), folder: z.string().optional(), revision }),
|
|
1102
|
+
output: file,
|
|
1103
|
+
run: ({ id: id2, name: name2, folder, revision: revision2 }) => bucket.move(id2, { name: name2, folder }, { revision: revision2 })
|
|
1104
|
+
},
|
|
1105
|
+
delete: {
|
|
1106
|
+
kind: "mutation",
|
|
1107
|
+
description: `Delete a ${name} file and its meta`,
|
|
1108
|
+
input: z.object({ id }),
|
|
1109
|
+
output: z.null(),
|
|
1110
|
+
confirm: "Delete this file?",
|
|
1111
|
+
run: async ({ id: id2 }) => (await bucket.delete(id2), null)
|
|
1112
|
+
}
|
|
1113
|
+
};
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
// src/kv.ts
|
|
1117
|
+
class Kv extends Store {
|
|
1118
|
+
schema;
|
|
1119
|
+
table;
|
|
1120
|
+
constructor(options, name, schema) {
|
|
1121
|
+
super(options, "kv", name);
|
|
1122
|
+
this.schema = schema;
|
|
1123
|
+
this.table = tableName("kv", name);
|
|
1124
|
+
}
|
|
1125
|
+
async setup() {
|
|
1126
|
+
await this.sql.unsafe(`create table if not exists "${this.table}" (
|
|
1127
|
+
key text primary key,
|
|
1128
|
+
value text not null,
|
|
1129
|
+
expires_at integer,
|
|
1130
|
+
created_at text not null,
|
|
1131
|
+
updated_at text not null
|
|
1132
|
+
)`);
|
|
1133
|
+
}
|
|
1134
|
+
async get(key) {
|
|
1135
|
+
return (await this.entry(key))?.value ?? null;
|
|
1136
|
+
}
|
|
1137
|
+
async entry(key) {
|
|
1138
|
+
recordRead(this.tag);
|
|
1139
|
+
const row = await this.live(key);
|
|
1140
|
+
return row ? toEntry(row) : null;
|
|
1141
|
+
}
|
|
1142
|
+
async set(key, value, options = {}) {
|
|
1143
|
+
assertWritable(this.tag);
|
|
1144
|
+
const parsed = this.schema.parse(value);
|
|
1145
|
+
const expiresAt = options.ttl === undefined ? null : Date.now() + parseDuration(options.ttl);
|
|
1146
|
+
const { entry, created } = await this.options.database.transaction(async () => {
|
|
1147
|
+
const before = await this.live(key);
|
|
1148
|
+
const now = new Date().toISOString();
|
|
1149
|
+
const row = {
|
|
1150
|
+
key,
|
|
1151
|
+
value: JSON.stringify(parsed),
|
|
1152
|
+
expires_at: expiresAt,
|
|
1153
|
+
created_at: before?.created_at ?? now,
|
|
1154
|
+
updated_at: now
|
|
1155
|
+
};
|
|
1156
|
+
await this.sql.unsafe(`insert into "${this.table}" (key, value, expires_at, created_at, updated_at) values ($1, $2, $3, $4, $5)
|
|
1157
|
+
on conflict (key) do update set value = excluded.value, expires_at = excluded.expires_at,
|
|
1158
|
+
created_at = excluded.created_at, updated_at = excluded.updated_at`, [row.key, row.value, row.expires_at, row.created_at, row.updated_at]);
|
|
1159
|
+
recordWrite(this.tag);
|
|
1160
|
+
return { entry: toEntry(row), created: !before };
|
|
1161
|
+
});
|
|
1162
|
+
await this.emit(created ? "created" : "updated", entry);
|
|
1163
|
+
return entry;
|
|
1164
|
+
}
|
|
1165
|
+
async delete(key) {
|
|
1166
|
+
assertWritable(this.tag);
|
|
1167
|
+
const before = await this.options.database.transaction(async () => {
|
|
1168
|
+
const before2 = await this.live(key);
|
|
1169
|
+
await this.sql.unsafe(`delete from "${this.table}" where key = $1`, [key]);
|
|
1170
|
+
if (before2)
|
|
1171
|
+
recordWrite(this.tag);
|
|
1172
|
+
return before2;
|
|
1173
|
+
});
|
|
1174
|
+
if (before)
|
|
1175
|
+
await this.emit("deleted", toEntry(before));
|
|
1176
|
+
}
|
|
1177
|
+
async list(options = {}) {
|
|
1178
|
+
recordRead(this.tag);
|
|
1179
|
+
const prefix = options.prefix ?? "";
|
|
1180
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}"
|
|
1181
|
+
where substr(key, 1, length($1)) = $1 and (expires_at is null or expires_at > $2)
|
|
1182
|
+
order by key limit $3`, [prefix, Date.now(), options.limit ?? 100]);
|
|
1183
|
+
return rows.map(toEntry);
|
|
1184
|
+
}
|
|
1185
|
+
async tick(now) {
|
|
1186
|
+
const expired = await this.options.database.transaction(async () => {
|
|
1187
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}" where expires_at <= $1`, [now]);
|
|
1188
|
+
if (rows.length > 0)
|
|
1189
|
+
await this.sql.unsafe(`delete from "${this.table}" where expires_at <= $1`, [now]);
|
|
1190
|
+
return rows;
|
|
1191
|
+
});
|
|
1192
|
+
if (expired.length === 0)
|
|
1193
|
+
return;
|
|
1194
|
+
this.options.announce([this.tag]);
|
|
1195
|
+
for (const row of expired)
|
|
1196
|
+
await this.emit("deleted", toEntry(row));
|
|
1197
|
+
}
|
|
1198
|
+
async live(key) {
|
|
1199
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}" where key = $1 and (expires_at is null or expires_at > $2)`, [key, Date.now()]);
|
|
1200
|
+
return rows[0] ?? null;
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
function toEntry(row) {
|
|
1204
|
+
return {
|
|
1205
|
+
key: row.key,
|
|
1206
|
+
value: JSON.parse(row.value),
|
|
1207
|
+
expiresAt: row.expires_at === null ? null : new Date(row.expires_at).toISOString(),
|
|
1208
|
+
createdAt: row.created_at,
|
|
1209
|
+
updatedAt: row.updated_at
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
// src/streams.ts
|
|
1214
|
+
import { appendFile, mkdir as mkdir2, readdir as readdir2, readFile as readFile2, rm as rm2, stat as stat2, truncate } from "fs/promises";
|
|
1215
|
+
import { join as join2 } from "path";
|
|
1216
|
+
var defaultRetain = { age: "7d", bytes: 50000000 };
|
|
1217
|
+
var segmentBytes = 1e6;
|
|
1218
|
+
var retainEveryMs = 60000;
|
|
1219
|
+
var followPollMs = 1000;
|
|
1220
|
+
|
|
1221
|
+
class Stream extends Store {
|
|
1222
|
+
schema;
|
|
1223
|
+
table;
|
|
1224
|
+
folder;
|
|
1225
|
+
ageMs;
|
|
1226
|
+
maxBytes;
|
|
1227
|
+
lastRetain = 0;
|
|
1228
|
+
waiting = new Map;
|
|
1229
|
+
constructor(options, name, schema, retain = defaultRetain) {
|
|
1230
|
+
super(options, "streams", name);
|
|
1231
|
+
this.schema = schema;
|
|
1232
|
+
this.table = tableName("streams", name);
|
|
1233
|
+
this.folder = join2(options.dir, "streams", name);
|
|
1234
|
+
this.ageMs = retain === "forever" || retain.age === undefined ? null : parseDuration(retain.age);
|
|
1235
|
+
this.maxBytes = retain === "forever" ? null : retain.bytes ?? null;
|
|
1236
|
+
}
|
|
1237
|
+
get path() {
|
|
1238
|
+
return this.folder;
|
|
1239
|
+
}
|
|
1240
|
+
async setup() {
|
|
1241
|
+
await mkdir2(this.folder, { recursive: true });
|
|
1242
|
+
await this.sql.unsafe(`create table if not exists "${this.table}" (
|
|
1243
|
+
key text primary key,
|
|
1244
|
+
first_offset integer not null,
|
|
1245
|
+
next_offset integer not null,
|
|
1246
|
+
segment integer not null,
|
|
1247
|
+
segment_bytes integer not null,
|
|
1248
|
+
bytes integer not null,
|
|
1249
|
+
ended_at text,
|
|
1250
|
+
updated_at text not null
|
|
1251
|
+
)`);
|
|
1252
|
+
}
|
|
1253
|
+
async append(key, data) {
|
|
1254
|
+
assertWritable(this.tag);
|
|
1255
|
+
checkKey(key);
|
|
1256
|
+
const parsed = this.schema.parse(data);
|
|
1257
|
+
const entry = await this.options.database.transaction(async () => {
|
|
1258
|
+
const row = await this.row(key) ?? await this.start(key);
|
|
1259
|
+
if (row.ended_at)
|
|
1260
|
+
throw new MethodError("VALIDATION", `${this.name} ${key} has ended; append to a new key.`);
|
|
1261
|
+
await this.cutUncommitted(row);
|
|
1262
|
+
const at = new Date().toISOString();
|
|
1263
|
+
const line = JSON.stringify({ offset: row.next_offset, at, data: parsed }) + `
|
|
1264
|
+
`;
|
|
1265
|
+
const size = Buffer.byteLength(line);
|
|
1266
|
+
const segment = row.segment_bytes > 0 && row.segment_bytes + size > segmentBytes ? row.next_offset : row.segment;
|
|
1267
|
+
const written = segment === row.segment ? row.segment_bytes : 0;
|
|
1268
|
+
await appendFile(this.segmentPath(key, segment), line);
|
|
1269
|
+
await this.sql.unsafe(`update "${this.table}" set next_offset = $1, segment = $2, segment_bytes = $3, bytes = bytes + $4, updated_at = $5 where key = $6`, [row.next_offset + 1, segment, written + size, size, at, key]);
|
|
1270
|
+
recordWrite(this.tag);
|
|
1271
|
+
return { id: `${key}:${row.next_offset}`, key, offset: row.next_offset, at, data: parsed };
|
|
1272
|
+
});
|
|
1273
|
+
this.wake(key);
|
|
1274
|
+
return entry;
|
|
1275
|
+
}
|
|
1276
|
+
async read(key, options = {}) {
|
|
1277
|
+
recordRead(this.tag);
|
|
1278
|
+
const row = await this.row(key);
|
|
1279
|
+
return row ? this.lines(row, options.from ?? 0, options.limit ?? 100) : [];
|
|
1280
|
+
}
|
|
1281
|
+
async* follow(key, options) {
|
|
1282
|
+
recordRead(this.tag);
|
|
1283
|
+
let next = options.from ?? Math.max(0, ((await this.row(key))?.next_offset ?? 0) - (options.last ?? 100));
|
|
1284
|
+
while (!options.signal.aborted) {
|
|
1285
|
+
const appended = this.nextAppend(key, options.signal);
|
|
1286
|
+
const row = await this.row(key);
|
|
1287
|
+
const lines = row ? await this.lines(row, next, 500) : [];
|
|
1288
|
+
for (const line of lines)
|
|
1289
|
+
yield line;
|
|
1290
|
+
next = lines.at(-1) ? lines.at(-1).offset + 1 : next;
|
|
1291
|
+
if (lines.length === 500)
|
|
1292
|
+
continue;
|
|
1293
|
+
if (row?.ended_at)
|
|
1294
|
+
return;
|
|
1295
|
+
await appended;
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
async end(key) {
|
|
1299
|
+
assertWritable(this.tag);
|
|
1300
|
+
await this.options.database.transaction(async () => {
|
|
1301
|
+
if (!await this.row(key))
|
|
1302
|
+
throw new MethodError("NOT_FOUND", `${this.name} has no key ${key}.`);
|
|
1303
|
+
const now = new Date().toISOString();
|
|
1304
|
+
await this.sql.unsafe(`update "${this.table}" set ended_at = $1, updated_at = $1 where key = $2`, [now, key]);
|
|
1305
|
+
recordWrite(this.tag);
|
|
1306
|
+
});
|
|
1307
|
+
this.wake(key);
|
|
1308
|
+
}
|
|
1309
|
+
async keys(options = {}) {
|
|
1310
|
+
recordRead(this.tag);
|
|
1311
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}" order by updated_at desc limit $1`, [
|
|
1312
|
+
options.limit ?? 100
|
|
1313
|
+
]);
|
|
1314
|
+
return rows.map(toKey);
|
|
1315
|
+
}
|
|
1316
|
+
async delete(key) {
|
|
1317
|
+
assertWritable(this.tag);
|
|
1318
|
+
await this.options.database.transaction(async () => {
|
|
1319
|
+
await rm2(join2(this.folder, key), { recursive: true, force: true });
|
|
1320
|
+
await this.sql.unsafe(`delete from "${this.table}" where key = $1`, [key]);
|
|
1321
|
+
recordWrite(this.tag);
|
|
1322
|
+
});
|
|
1323
|
+
this.wake(key);
|
|
1324
|
+
}
|
|
1325
|
+
async tick(now) {
|
|
1326
|
+
if (now - this.lastRetain < retainEveryMs || this.ageMs === null && this.maxBytes === null)
|
|
1327
|
+
return;
|
|
1328
|
+
this.lastRetain = now;
|
|
1329
|
+
const changed = await this.options.database.transaction(async () => {
|
|
1330
|
+
let changed2 = false;
|
|
1331
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}"`);
|
|
1332
|
+
for (const row of rows)
|
|
1333
|
+
changed2 = await this.retain(row, now) || changed2;
|
|
1334
|
+
return changed2;
|
|
1335
|
+
});
|
|
1336
|
+
if (changed)
|
|
1337
|
+
this.options.announce([this.tag]);
|
|
1338
|
+
}
|
|
1339
|
+
close() {
|
|
1340
|
+
for (const wakers of this.waiting.values())
|
|
1341
|
+
for (const wake of wakers)
|
|
1342
|
+
wake();
|
|
1343
|
+
this.waiting.clear();
|
|
1344
|
+
}
|
|
1345
|
+
async retain(row, now) {
|
|
1346
|
+
if (this.ageMs !== null && Date.parse(row.updated_at) <= now - this.ageMs) {
|
|
1347
|
+
await rm2(join2(this.folder, row.key), { recursive: true, force: true });
|
|
1348
|
+
await this.sql.unsafe(`delete from "${this.table}" where key = $1`, [row.key]);
|
|
1349
|
+
return true;
|
|
1350
|
+
}
|
|
1351
|
+
const segments = (await this.segments(row.key)).filter((start) => start < row.segment);
|
|
1352
|
+
let { bytes, first_offset: first } = row;
|
|
1353
|
+
for (const [index, start] of segments.entries()) {
|
|
1354
|
+
const path = this.segmentPath(row.key, start);
|
|
1355
|
+
const info = await stat2(path);
|
|
1356
|
+
const old = this.ageMs !== null && info.mtimeMs <= now - this.ageMs;
|
|
1357
|
+
const over = this.maxBytes !== null && bytes > this.maxBytes;
|
|
1358
|
+
if (!old && !over)
|
|
1359
|
+
break;
|
|
1360
|
+
await rm2(path, { force: true });
|
|
1361
|
+
bytes -= info.size;
|
|
1362
|
+
first = segments[index + 1] ?? row.segment;
|
|
1363
|
+
}
|
|
1364
|
+
if (first === row.first_offset)
|
|
1365
|
+
return false;
|
|
1366
|
+
await this.sql.unsafe(`update "${this.table}" set first_offset = $1, bytes = $2 where key = $3`, [first, bytes, row.key]);
|
|
1367
|
+
return true;
|
|
1368
|
+
}
|
|
1369
|
+
async lines(row, from, limit2) {
|
|
1370
|
+
const start = Math.max(from, row.first_offset);
|
|
1371
|
+
const segments = await this.segments(row.key);
|
|
1372
|
+
const firstSegment = segments.findLastIndex((segment) => segment <= start);
|
|
1373
|
+
const lines = [];
|
|
1374
|
+
for (const segment of segments.slice(Math.max(0, firstSegment))) {
|
|
1375
|
+
const text = await readFile2(this.segmentPath(row.key, segment), "utf8").catch(() => "");
|
|
1376
|
+
for (const raw of text.split(`
|
|
1377
|
+
`)) {
|
|
1378
|
+
if (!raw)
|
|
1379
|
+
continue;
|
|
1380
|
+
const line = JSON.parse(raw);
|
|
1381
|
+
if (line.offset < start || line.offset >= row.next_offset)
|
|
1382
|
+
continue;
|
|
1383
|
+
lines.push({ id: `${row.key}:${line.offset}`, key: row.key, ...line });
|
|
1384
|
+
if (lines.length >= limit2)
|
|
1385
|
+
return lines;
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
return lines;
|
|
1389
|
+
}
|
|
1390
|
+
async cutUncommitted(row) {
|
|
1391
|
+
const path = this.segmentPath(row.key, row.segment);
|
|
1392
|
+
const size = await stat2(path).then((info) => info.size, () => 0);
|
|
1393
|
+
if (size > row.segment_bytes)
|
|
1394
|
+
await truncate(path, row.segment_bytes);
|
|
1395
|
+
}
|
|
1396
|
+
async start(key) {
|
|
1397
|
+
await mkdir2(join2(this.folder, key), { recursive: true });
|
|
1398
|
+
const row = {
|
|
1399
|
+
key,
|
|
1400
|
+
first_offset: 0,
|
|
1401
|
+
next_offset: 0,
|
|
1402
|
+
segment: 0,
|
|
1403
|
+
segment_bytes: 0,
|
|
1404
|
+
bytes: 0,
|
|
1405
|
+
ended_at: null,
|
|
1406
|
+
updated_at: new Date().toISOString()
|
|
1407
|
+
};
|
|
1408
|
+
await this.sql.unsafe(`insert into "${this.table}" (key, first_offset, next_offset, segment, segment_bytes, bytes, updated_at)
|
|
1409
|
+
values ($1, 0, 0, 0, 0, 0, $2)`, [key, row.updated_at]);
|
|
1410
|
+
return row;
|
|
1411
|
+
}
|
|
1412
|
+
async row(key) {
|
|
1413
|
+
const rows = await this.sql.unsafe(`select * from "${this.table}" where key = $1`, [key]);
|
|
1414
|
+
return rows[0] ?? null;
|
|
1415
|
+
}
|
|
1416
|
+
async segments(key) {
|
|
1417
|
+
const names = await readdir2(join2(this.folder, key)).catch(() => []);
|
|
1418
|
+
return names.filter((name) => /^\d+\.jsonl$/.test(name)).map((name) => Number.parseInt(name)).sort((a, b) => a - b);
|
|
1419
|
+
}
|
|
1420
|
+
segmentPath(key, segment) {
|
|
1421
|
+
return join2(this.folder, key, `${String(segment).padStart(12, "0")}.jsonl`);
|
|
1422
|
+
}
|
|
1423
|
+
nextAppend(key, signal) {
|
|
1424
|
+
return new Promise((resolve) => {
|
|
1425
|
+
const wakers = this.waiting.get(key) ?? new Set;
|
|
1426
|
+
this.waiting.set(key, wakers);
|
|
1427
|
+
const done = () => {
|
|
1428
|
+
clearTimeout(timer);
|
|
1429
|
+
signal.removeEventListener("abort", done);
|
|
1430
|
+
wakers.delete(done);
|
|
1431
|
+
resolve();
|
|
1432
|
+
};
|
|
1433
|
+
const timer = setTimeout(done, followPollMs);
|
|
1434
|
+
signal.addEventListener("abort", done);
|
|
1435
|
+
wakers.add(done);
|
|
1436
|
+
});
|
|
1437
|
+
}
|
|
1438
|
+
wake(key) {
|
|
1439
|
+
for (const wake of [...this.waiting.get(key) ?? []])
|
|
1440
|
+
wake();
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
function checkKey(key) {
|
|
1444
|
+
if (!safeName.test(key)) {
|
|
1445
|
+
throw new MethodError("VALIDATION", `A stream key is letters, digits, dots, dashes and underscores, up to 128: ${key}`);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
function toKey(row) {
|
|
1449
|
+
return {
|
|
1450
|
+
key: row.key,
|
|
1451
|
+
lines: row.next_offset - row.first_offset,
|
|
1452
|
+
bytes: row.bytes,
|
|
1453
|
+
endedAt: row.ended_at,
|
|
1454
|
+
updatedAt: row.updated_at
|
|
1455
|
+
};
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
// src/workflow/register.ts
|
|
1459
|
+
import { z as z4 } from "zod";
|
|
1460
|
+
|
|
1461
|
+
// src/workflow/contracts.ts
|
|
1462
|
+
import { z as z3 } from "zod";
|
|
1463
|
+
import { isAbsolute as isAbsolute2 } from "path";
|
|
1464
|
+
|
|
1465
|
+
// src/workflow/policy.ts
|
|
1466
|
+
import { z as z2 } from "zod";
|
|
1467
|
+
var duration = z2.custom((value) => typeof value === "number" || typeof value === "string");
|
|
1468
|
+
var StepOptionsSchema = z2.object({
|
|
1469
|
+
kind: z2.enum(["query", "mutation"]).optional(),
|
|
1470
|
+
attempts: z2.number().optional(),
|
|
1471
|
+
timeout: duration.optional(),
|
|
1472
|
+
backoff: z2.object({ initial: duration.optional(), factor: z2.number().optional(), max: duration.optional() }).strict().optional()
|
|
1473
|
+
}).strict();
|
|
1474
|
+
var maxDurationMs = 365 * 86400000;
|
|
1475
|
+
var DurationMsSchema = z2.number().int().positive().max(maxDurationMs);
|
|
1476
|
+
var StepPolicySchema = z2.object({
|
|
1477
|
+
kind: z2.enum(["query", "mutation"]),
|
|
1478
|
+
attempts: z2.number().int().min(1).max(100),
|
|
1479
|
+
timeoutMs: DurationMsSchema,
|
|
1480
|
+
backoff: z2.object({ initialMs: DurationMsSchema, factor: z2.number().min(1).max(1000), maxMs: DurationMsSchema }).strict()
|
|
1481
|
+
}).strict().refine((value) => value.kind === "query" || value.attempts === 1, "Only query steps may retry.").refine((value) => value.backoff.initialMs <= value.backoff.maxMs, "Initial backoff must not exceed its maximum.");
|
|
1482
|
+
function durationMs(value) {
|
|
1483
|
+
return DurationMsSchema.parse(Math.ceil(parseDuration(value)));
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
// src/workflow/contracts.ts
|
|
1487
|
+
var RunSummarySchema = z3.object({
|
|
1488
|
+
id: z3.string(),
|
|
1489
|
+
workflow: z3.string(),
|
|
1490
|
+
status: z3.enum(["pending", "running", "waiting", "attention", "completed", "failed", "cancelled", "expired", "unknown", "blocked"]),
|
|
1491
|
+
currentStep: z3.string().nullable(),
|
|
1492
|
+
attempt: z3.number().int(),
|
|
1493
|
+
createdAt: z3.string(),
|
|
1494
|
+
updatedAt: z3.string(),
|
|
1495
|
+
nextAttemptAt: z3.string().nullable(),
|
|
1496
|
+
deadlineAt: z3.string().nullable()
|
|
1497
|
+
});
|
|
1498
|
+
var RunMetadataSchema = z3.object({
|
|
1499
|
+
id: z3.string().uuid(),
|
|
1500
|
+
dir: z3.string().refine(isAbsolute2),
|
|
1501
|
+
root: z3.string().refine(isAbsolute2),
|
|
1502
|
+
namespace: z3.string().regex(/^[a-z0-9][a-z0-9.-]*$/),
|
|
1503
|
+
plugin: z3.string().regex(/^[a-z0-9][a-z0-9-]*$/),
|
|
1504
|
+
caller: z3.enum(["card", "agent", "cron", "code", "http"]),
|
|
1505
|
+
workflow: z3.string().regex(/^[a-z][a-zA-Z0-9_-]*$/),
|
|
1506
|
+
input: z3.json(),
|
|
1507
|
+
key: z3.string().nullable(),
|
|
1508
|
+
timeoutMs: DurationMsSchema,
|
|
1509
|
+
maxOfflineMs: DurationMsSchema,
|
|
1510
|
+
retentionMs: DurationMsSchema,
|
|
1511
|
+
createdAt: z3.iso.datetime()
|
|
1512
|
+
}).strict();
|
|
1513
|
+
var RunLocationSchema = RunMetadataSchema.omit({ input: true, timeoutMs: true, workflow: true, createdAt: true });
|
|
1514
|
+
|
|
1515
|
+
// src/workflow/register.ts
|
|
1516
|
+
function workflowTag(namespace) {
|
|
1517
|
+
return `${namespace}.workflows`;
|
|
1518
|
+
}
|
|
1519
|
+
function workflowMethods(plugin, namespace, name, definition, ctx, bridge, directory) {
|
|
1520
|
+
const invoke = (operation, input) => {
|
|
1521
|
+
const dir = directory();
|
|
1522
|
+
if (!bridge || !dir)
|
|
1523
|
+
throw new MethodError("INTERNAL", "Durable workflows run through the plugins service with source files.");
|
|
1524
|
+
return bridge({ operation, dir, plugin, namespace, workflow: name, root: ctx.root, caller: ctx.caller, input, settings: workflowSettings(definition) });
|
|
1525
|
+
};
|
|
1526
|
+
const id = z4.object({ id: z4.string().uuid() }).strict();
|
|
1527
|
+
const spec = (method, kind, input, description, confirm, output) => ({
|
|
1528
|
+
name: `${plugin}.${name}.${method}`,
|
|
1529
|
+
plugin,
|
|
1530
|
+
kind,
|
|
1531
|
+
input,
|
|
1532
|
+
description,
|
|
1533
|
+
confirm,
|
|
1534
|
+
output,
|
|
1535
|
+
run: async (input2, raw) => {
|
|
1536
|
+
if (kind === "query")
|
|
1537
|
+
recordRead(workflowTag(namespace));
|
|
1538
|
+
const request = method === "start" ? { ...input2, input: raw.input } : input2;
|
|
1539
|
+
return invoke(method, request);
|
|
1540
|
+
}
|
|
1541
|
+
});
|
|
1542
|
+
return [
|
|
1543
|
+
spec("list", "query", z4.object({ limit: z4.number().int().min(1).max(100).default(20), status: z4.string().optional(), cursor: z4.string().optional() }).strict(), "List workflow runs", undefined, z4.object({ items: z4.array(RunSummarySchema), cursor: z4.string().nullable() })),
|
|
1544
|
+
spec("get", "query", id, "Get one workflow run"),
|
|
1545
|
+
spec("start", "mutation", z4.object({ input: definition.input, key: z4.string().min(1).max(200).optional() }).strict(), definition.description ?? "Start a durable workflow", definition.confirm, RunSummarySchema),
|
|
1546
|
+
spec("retry", "mutation", id, "Retry a safely failed query step", undefined, RunSummarySchema),
|
|
1547
|
+
spec("restart", "mutation", id, "Start a run that needs attention over from the beginning, on the current code", undefined, RunSummarySchema),
|
|
1548
|
+
spec("cancel", "mutation", id, "Request cancellation of a workflow", undefined, RunSummarySchema),
|
|
1549
|
+
spec("cancelAll", "mutation", z4.object({ status: z4.string().optional() }).strict(), "Cancel every unfinished run, or only those in one status", undefined, z4.object({ cancelled: z4.number().int() }))
|
|
1550
|
+
];
|
|
1551
|
+
}
|
|
1552
|
+
function workflowSettings(definition) {
|
|
1553
|
+
return {
|
|
1554
|
+
timeoutMs: durationMs(definition.timeout ?? "30d"),
|
|
1555
|
+
maxOfflineMs: durationMs(definition.maxOffline ?? "14d"),
|
|
1556
|
+
retentionMs: durationMs(definition.retention ?? "30d")
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
// src/runtime.ts
|
|
1561
|
+
var sampleMs = Number(process.env.REPOGO_PLUGINS_SAMPLE_MS ?? 5000);
|
|
1562
|
+
var sampleLimit = 200;
|
|
1563
|
+
var cronLateMs = 2 * 60000;
|
|
1564
|
+
var never = new AbortController().signal;
|
|
1565
|
+
var agentTimeoutMs = 10 * 60000;
|
|
1566
|
+
var agentWaitMs = 50000;
|
|
1567
|
+
var pluginName = /^[a-z][a-z0-9-]*$/;
|
|
1568
|
+
var namespaceName = /^[a-z][a-z0-9.-]*$/;
|
|
1569
|
+
|
|
1570
|
+
class Runtime {
|
|
1571
|
+
dataDir;
|
|
1572
|
+
root;
|
|
1573
|
+
host;
|
|
1574
|
+
workflows;
|
|
1575
|
+
events = new EventBus;
|
|
1576
|
+
methods = new Map;
|
|
1577
|
+
plugins = new Map;
|
|
1578
|
+
crons = [];
|
|
1579
|
+
contexts = new Map;
|
|
1580
|
+
databases = new Map;
|
|
1581
|
+
answerCaches = new Map;
|
|
1582
|
+
stores = new Map;
|
|
1583
|
+
invalidationListeners = new Set;
|
|
1584
|
+
noticeListeners = new Set;
|
|
1585
|
+
pluginsDir = null;
|
|
1586
|
+
constructor(dataDir, root, host, workflows) {
|
|
1587
|
+
this.dataDir = dataDir;
|
|
1588
|
+
this.root = root;
|
|
1589
|
+
this.host = host;
|
|
1590
|
+
this.workflows = workflows;
|
|
1591
|
+
}
|
|
1592
|
+
callRoot = new AsyncLocalStorage3;
|
|
1593
|
+
callSignal = new AsyncLocalStorage3;
|
|
1594
|
+
callCaller = new AsyncLocalStorage3;
|
|
1595
|
+
callConfirmed = new AsyncLocalStorage3;
|
|
1596
|
+
cronsTag = `crons.${crypto.randomUUID()}`;
|
|
1597
|
+
static async open(options) {
|
|
1598
|
+
await mkdir3(options.dataDir, { recursive: true });
|
|
1599
|
+
return new Runtime(options.dataDir, options.root ?? process.cwd(), options.host, options.workflows);
|
|
1600
|
+
}
|
|
1601
|
+
async loadDir(pluginsDir, only = null) {
|
|
1602
|
+
this.pluginsDir = pluginsDir;
|
|
1603
|
+
const entries = await readdir3(pluginsDir, { withFileTypes: true });
|
|
1604
|
+
for (const entry of entries) {
|
|
1605
|
+
if (!entry.isDirectory() || !pluginName.test(entry.name))
|
|
1606
|
+
continue;
|
|
1607
|
+
if (only && !only.includes(entry.name))
|
|
1608
|
+
continue;
|
|
1609
|
+
const module = await logContext.run({ plugin: entry.name }, () => import(join3(pluginsDir, entry.name, "app.ts")));
|
|
1610
|
+
await this.add(entry.name, module.default);
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
async add(name, plugin) {
|
|
1614
|
+
if (!pluginName.test(name))
|
|
1615
|
+
throw new Error(`plugin names are lowercase letters, digits and hyphens: ${name}`);
|
|
1616
|
+
const collections = {};
|
|
1617
|
+
const buckets = {};
|
|
1618
|
+
const kv = {};
|
|
1619
|
+
const streams = {};
|
|
1620
|
+
const declared = Object.entries(plugin.collections ?? {});
|
|
1621
|
+
const stores = [...declared.map(([n]) => n), ...[plugin.kv, plugin.streams, plugin.buckets].flatMap((group) => Object.keys(group ?? {}))];
|
|
1622
|
+
const hidden = new Set(plugin.private ?? []);
|
|
1623
|
+
for (const store of hidden) {
|
|
1624
|
+
if (!stores.includes(store))
|
|
1625
|
+
throw new Error(`plugin ${name} lists ${store} as private, but keeps no collection, kv, stream or bucket by that name`);
|
|
1626
|
+
}
|
|
1627
|
+
const hasCache = Object.values(plugin.methods ?? {}).some((method) => method.cache);
|
|
1628
|
+
const hasData = hasCache || Object.keys(plugin.workflows ?? {}).length > 0 || declared.length > 0 || [plugin.buckets, plugin.kv, plugin.streams].some((stores2) => Object.keys(stores2 ?? {}).length > 0);
|
|
1629
|
+
const namespace = plugin.namespace ?? "";
|
|
1630
|
+
if (hasData && !namespace)
|
|
1631
|
+
throw new Error(`plugin ${name} keeps data, so it needs a namespace: "<project>.${name}" for one project's, "${name}" to share it`);
|
|
1632
|
+
if (namespace && !namespaceName.test(namespace))
|
|
1633
|
+
throw new Error(`a plugin's namespace is lowercase letters, digits, dots and hyphens: ${namespace}`);
|
|
1634
|
+
const file = hasData ? await this.database(namespace) : null;
|
|
1635
|
+
for (const [collectionName, schema] of declared) {
|
|
1636
|
+
const collection = new Collection(file, this.events, name, namespace, collectionName, schema);
|
|
1637
|
+
await collection.setup();
|
|
1638
|
+
collections[collectionName] = collection;
|
|
1639
|
+
if (!hidden.has(collectionName))
|
|
1640
|
+
this.registerCrud(name, collectionName, collection, schema);
|
|
1641
|
+
}
|
|
1642
|
+
const options = {
|
|
1643
|
+
database: file,
|
|
1644
|
+
events: this.events,
|
|
1645
|
+
namespace,
|
|
1646
|
+
dir: join3(this.dataDir, namespace),
|
|
1647
|
+
announce: (tags) => this.notifyInvalidated(tags)
|
|
1648
|
+
};
|
|
1649
|
+
for (const [storeName, schema] of Object.entries(plugin.kv ?? {})) {
|
|
1650
|
+
const store = await this.store(name, "kv", storeName, schema, () => new Kv(options, storeName, schema));
|
|
1651
|
+
kv[storeName] = store;
|
|
1652
|
+
if (!hidden.has(storeName))
|
|
1653
|
+
this.registerGenerated(name, storeName, kvMethods(storeName, store, schema));
|
|
1654
|
+
}
|
|
1655
|
+
for (const [storeName, definition] of Object.entries(plugin.streams ?? {})) {
|
|
1656
|
+
const { item, retain } = "item" in definition ? definition : { item: definition, retain: defaultRetain };
|
|
1657
|
+
const store = await this.store(name, "streams", storeName, item, () => new Stream(options, storeName, item, retain), retain);
|
|
1658
|
+
streams[storeName] = store;
|
|
1659
|
+
if (!hidden.has(storeName))
|
|
1660
|
+
this.registerGenerated(name, storeName, streamMethods(storeName, store, item));
|
|
1661
|
+
}
|
|
1662
|
+
for (const [storeName, schema] of Object.entries(plugin.buckets ?? {})) {
|
|
1663
|
+
const store = await this.store(name, "buckets", storeName, schema, () => new Bucket(options, storeName, schema));
|
|
1664
|
+
buckets[storeName] = store;
|
|
1665
|
+
if (!hidden.has(storeName))
|
|
1666
|
+
this.registerGenerated(name, storeName, bucketMethods(storeName, store, schema));
|
|
1667
|
+
}
|
|
1668
|
+
const runtime = this;
|
|
1669
|
+
const ctx = {
|
|
1670
|
+
plugin: name,
|
|
1671
|
+
get caller() {
|
|
1672
|
+
return runtime.callCaller.getStore() ?? "code";
|
|
1673
|
+
},
|
|
1674
|
+
get root() {
|
|
1675
|
+
return runtime.callRoot.getStore() ?? runtime.root;
|
|
1676
|
+
},
|
|
1677
|
+
get signal() {
|
|
1678
|
+
return attemptScope.getStore()?.signal ?? runtime.callSignal.getStore() ?? never;
|
|
1679
|
+
},
|
|
1680
|
+
collections,
|
|
1681
|
+
buckets,
|
|
1682
|
+
kv,
|
|
1683
|
+
streams,
|
|
1684
|
+
call: async (method, input) => (await this.call(method.includes(".") ? method : `${name}.${method}`, input)).result,
|
|
1685
|
+
emit: (event, payload) => this.events.emit(`${name}.${event}`, payload),
|
|
1686
|
+
host: async (method, params) => {
|
|
1687
|
+
if (workflowScope.getStore())
|
|
1688
|
+
assertWritable("host methods");
|
|
1689
|
+
if (!runtime.host)
|
|
1690
|
+
throw new MethodError("INTERNAL", "ctx.host runs only under the RepoGo host.");
|
|
1691
|
+
return runtime.host(name, method, params ?? {});
|
|
1692
|
+
},
|
|
1693
|
+
mcp: async (server, tool, input) => {
|
|
1694
|
+
if (!runtime.host)
|
|
1695
|
+
throw new MethodError("INTERNAL", "ctx.mcp runs only under the RepoGo host.");
|
|
1696
|
+
const answer = await runtime.host(name, "mcp.call", {
|
|
1697
|
+
project: ctx.root,
|
|
1698
|
+
server,
|
|
1699
|
+
tool,
|
|
1700
|
+
input: input ?? {},
|
|
1701
|
+
read: currentAccess()?.readOnly ?? false,
|
|
1702
|
+
confirmed: runtime.callConfirmed.getStore() ?? false
|
|
1703
|
+
});
|
|
1704
|
+
if (answer.error)
|
|
1705
|
+
throw mcpFailure(server, tool, answer.error);
|
|
1706
|
+
return answer.result ?? null;
|
|
1707
|
+
},
|
|
1708
|
+
plugins: () => {
|
|
1709
|
+
recordRead(this.cronsTag);
|
|
1710
|
+
return this.manifest().plugins;
|
|
1711
|
+
},
|
|
1712
|
+
notify: (notice) => {
|
|
1713
|
+
assertWritable("notifications");
|
|
1714
|
+
const sent = { ...notice, plugin: name, root: runtime.callRoot.getStore() ?? runtime.root };
|
|
1715
|
+
for (const listener of runtime.noticeListeners)
|
|
1716
|
+
listener(sent);
|
|
1717
|
+
},
|
|
1718
|
+
activity: async (activity) => {
|
|
1719
|
+
assertWritable("activities");
|
|
1720
|
+
if (!runtime.host)
|
|
1721
|
+
throw new MethodError("INTERNAL", "ctx.activity runs only under the RepoGo host.");
|
|
1722
|
+
await runtime.host(name, "activities.set", activityParams(activity, ctx.root));
|
|
1723
|
+
},
|
|
1724
|
+
agent: (prompt, options2) => this.agent(name, ctx, prompt, options2 ?? {}),
|
|
1725
|
+
log: (...args) => console.log(...args)
|
|
1726
|
+
};
|
|
1727
|
+
const cache = hasCache ? await this.answerCache(namespace, file) : null;
|
|
1728
|
+
for (const [methodName, method] of Object.entries(plugin.methods ?? {})) {
|
|
1729
|
+
const fullName = `${name}.${methodName}`;
|
|
1730
|
+
const options2 = method.cache;
|
|
1731
|
+
this.register({
|
|
1732
|
+
name: fullName,
|
|
1733
|
+
plugin: name,
|
|
1734
|
+
kind: method.kind,
|
|
1735
|
+
description: method.description,
|
|
1736
|
+
confirm: method.confirm,
|
|
1737
|
+
http: method.http,
|
|
1738
|
+
input: method.input,
|
|
1739
|
+
output: method.output,
|
|
1740
|
+
run: options2 && cache ? async (input) => cache.read(cacheKey(fullName, options2.shared ? null : ctx.root, input), options2, () => method.run(input, ctx)) : async (input) => method.run(input, ctx)
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
this.contexts.set(name, ctx);
|
|
1744
|
+
for (const [workflowName, definition] of Object.entries(plugin.workflows ?? {})) {
|
|
1745
|
+
if (!/^[a-z][a-zA-Z0-9_-]*$/.test(workflowName))
|
|
1746
|
+
throw new Error(`Invalid workflow name ${workflowName}.`);
|
|
1747
|
+
workflowSettings(definition);
|
|
1748
|
+
for (const method of workflowMethods(name, namespace, workflowName, definition, ctx, this.workflows, () => this.pluginsDir))
|
|
1749
|
+
this.register(method);
|
|
1750
|
+
}
|
|
1751
|
+
for (const [event, handler] of Object.entries(plugin.on ?? {})) {
|
|
1752
|
+
this.events.on(`${name}.${event}`, (e) => logContext.run(this.logContextFor(name), () => handler(e.payload, ctx)));
|
|
1753
|
+
}
|
|
1754
|
+
this.plugins.set(name, plugin);
|
|
1755
|
+
}
|
|
1756
|
+
async executeWorkflow(plugin, name, input, context) {
|
|
1757
|
+
const definition = this.plugins.get(plugin)?.workflows?.[name];
|
|
1758
|
+
if (!definition)
|
|
1759
|
+
throw new MethodError("NOT_FOUND", `No workflow ${plugin}.${name}.`);
|
|
1760
|
+
const ctx = this.contexts.get(plugin);
|
|
1761
|
+
return workflowScope.run(true, () => this.callCaller.run(context.caller, () => this.callRoot.run(context.root, () => this.callSignal.run(context.signal, () => runWith({ plugin, method: `${plugin}.${name}`, caller: context.caller, callId: context.runId }, async () => {
|
|
1762
|
+
const result = await definition.run(definition.input.parse(input), workflowContext(ctx, context));
|
|
1763
|
+
return definition.output ? definition.output.parse(result) : result;
|
|
1764
|
+
})))));
|
|
1765
|
+
}
|
|
1766
|
+
async call(name, input = {}, options = {}) {
|
|
1767
|
+
if (options.files) {
|
|
1768
|
+
const { files, ...rest } = options;
|
|
1769
|
+
const fields = input ?? {};
|
|
1770
|
+
if (Object.hasOwn(fields, "files"))
|
|
1771
|
+
throw new MethodError("VALIDATION", "files comes from the upload; leave it out of the input.");
|
|
1772
|
+
return callFiles.run(files, () => this.call(name, { ...fields, files }, rest));
|
|
1773
|
+
}
|
|
1774
|
+
if (options.root && options.root !== this.callRoot.getStore()) {
|
|
1775
|
+
const { root, ...rest } = options;
|
|
1776
|
+
return this.callRoot.run(root, () => this.call(name, input, rest));
|
|
1777
|
+
}
|
|
1778
|
+
if (options.caller && options.caller !== this.callCaller.getStore()) {
|
|
1779
|
+
const { caller, ...rest } = options;
|
|
1780
|
+
return this.callCaller.run(caller, () => this.call(name, input, rest));
|
|
1781
|
+
}
|
|
1782
|
+
const method = this.methods.get(name);
|
|
1783
|
+
if (!method)
|
|
1784
|
+
throw new MethodError("NOT_FOUND", `There is no method ${name}.`);
|
|
1785
|
+
const where = this.logContextFor(method.plugin, name, options.chat);
|
|
1786
|
+
if (!logContext.getStore())
|
|
1787
|
+
return runWith(where, () => this.call(name, input, options));
|
|
1788
|
+
if (where !== logContext.getStore())
|
|
1789
|
+
return logContext.run(where, () => this.call(name, input, options));
|
|
1790
|
+
if ((options.expect === "query" || currentAccess()?.readOnly) && method.kind === "mutation") {
|
|
1791
|
+
throw new MethodError("FORBIDDEN", `${name} changes data, so it only runs when the user acts.`);
|
|
1792
|
+
}
|
|
1793
|
+
const raw = input ?? {};
|
|
1794
|
+
const parsed = method.input.parse(raw);
|
|
1795
|
+
if (method.confirm && (options.confirmed === false || workflowScope.getStore()))
|
|
1796
|
+
throw new MethodError("CONFIRM", fillConfirm(method.confirm, parsed));
|
|
1797
|
+
if (method.kind === "stream")
|
|
1798
|
+
return { result: await this.sample(name, input), reads: [], invalidates: [] };
|
|
1799
|
+
const confirmed = this.callConfirmed.getStore() ?? false;
|
|
1800
|
+
const run = await tracked(method.kind === "query", (tags) => this.notifyInvalidated(tags), () => this.callConfirmed.run(confirmed || method.confirm !== undefined, async () => {
|
|
1801
|
+
const result = await method.run(parsed, raw);
|
|
1802
|
+
await this.events.emit(name, { input: parsed, result });
|
|
1803
|
+
return result;
|
|
1804
|
+
}));
|
|
1805
|
+
const invalidates = run.outermost ? [...run.access.writes] : [];
|
|
1806
|
+
if (invalidates.length > 0)
|
|
1807
|
+
this.notifyInvalidated(invalidates);
|
|
1808
|
+
if (!run.ok) {
|
|
1809
|
+
if (!(run.error instanceof MethodError && run.error.code === "CONFIRM"))
|
|
1810
|
+
record("error", [run.error]);
|
|
1811
|
+
throw run.error;
|
|
1812
|
+
}
|
|
1813
|
+
return { result: run.value ?? null, reads: [...run.access.reads], invalidates };
|
|
1814
|
+
}
|
|
1815
|
+
stream(name, input, options, onItem) {
|
|
1816
|
+
const method = this.methods.get(name);
|
|
1817
|
+
if (!method)
|
|
1818
|
+
throw new MethodError("NOT_FOUND", `There is no method ${name}.`);
|
|
1819
|
+
if (method.kind !== "stream")
|
|
1820
|
+
throw new MethodError("VALIDATION", `${name} is a ${method.kind}, not a stream.`);
|
|
1821
|
+
const raw = input ?? {};
|
|
1822
|
+
const parsed = method.input.parse(raw);
|
|
1823
|
+
const root = options.root ?? this.callRoot.getStore() ?? this.root;
|
|
1824
|
+
const caller = options.caller ?? this.callCaller.getStore() ?? "code";
|
|
1825
|
+
const where = this.logContextFor(method.plugin, name);
|
|
1826
|
+
return this.callRoot.run(root, () => this.callCaller.run(caller, () => this.callSignal.run(options.signal, () => runWith(where, async () => {
|
|
1827
|
+
const run = await tracked(true, (tags) => this.notifyInvalidated(tags), async () => {
|
|
1828
|
+
const items = await method.run(parsed, raw);
|
|
1829
|
+
if (!isAsyncIterable(items))
|
|
1830
|
+
throw new MethodError("INTERNAL", `${name} is a stream but returned no async iterable.`);
|
|
1831
|
+
for await (const item of items) {
|
|
1832
|
+
if (options.signal.aborted)
|
|
1833
|
+
break;
|
|
1834
|
+
onItem(item);
|
|
1835
|
+
}
|
|
1836
|
+
});
|
|
1837
|
+
if (!run.ok && !options.signal.aborted) {
|
|
1838
|
+
record("error", [run.error]);
|
|
1839
|
+
throw run.error;
|
|
1840
|
+
}
|
|
1841
|
+
}))));
|
|
1842
|
+
}
|
|
1843
|
+
async sample(name, input) {
|
|
1844
|
+
const items = [];
|
|
1845
|
+
const stop = new AbortController;
|
|
1846
|
+
const timer = setTimeout(() => stop.abort(), sampleMs);
|
|
1847
|
+
try {
|
|
1848
|
+
const stopped = new Promise((resolve) => stop.signal.addEventListener("abort", () => resolve()));
|
|
1849
|
+
const ended = this.stream(name, input, { signal: stop.signal }, (item) => {
|
|
1850
|
+
items.push(item);
|
|
1851
|
+
if (items.length >= sampleLimit)
|
|
1852
|
+
stop.abort();
|
|
1853
|
+
});
|
|
1854
|
+
await Promise.race([ended, stopped]);
|
|
1855
|
+
return { items, done: !stop.signal.aborted };
|
|
1856
|
+
} finally {
|
|
1857
|
+
clearTimeout(timer);
|
|
1858
|
+
stop.abort();
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
onNotify(listener) {
|
|
1862
|
+
this.noticeListeners.add(listener);
|
|
1863
|
+
return () => this.noticeListeners.delete(listener);
|
|
1864
|
+
}
|
|
1865
|
+
onInvalidate(listener) {
|
|
1866
|
+
this.invalidationListeners.add(listener);
|
|
1867
|
+
return () => this.invalidationListeners.delete(listener);
|
|
1868
|
+
}
|
|
1869
|
+
setCrons(entries) {
|
|
1870
|
+
const kept = new Map(this.crons.map((job) => [job.signature, job]));
|
|
1871
|
+
const jobs = [];
|
|
1872
|
+
const statuses = [];
|
|
1873
|
+
for (const entry of entries) {
|
|
1874
|
+
const timezone = entry.timezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
1875
|
+
const signature = `${entry.plugin}.${entry.method} ${entry.schedule} ${timezone}`;
|
|
1876
|
+
try {
|
|
1877
|
+
const job = kept.get(signature) ?? this.cronJob(entry, timezone, signature);
|
|
1878
|
+
jobs.push(job);
|
|
1879
|
+
statuses.push({ ...cronState(job), plugin: job.plugin, error: null });
|
|
1880
|
+
} catch (error) {
|
|
1881
|
+
statuses.push({
|
|
1882
|
+
plugin: entry.plugin,
|
|
1883
|
+
method: `${entry.plugin}.${entry.method}`,
|
|
1884
|
+
schedule: entry.schedule,
|
|
1885
|
+
timezone,
|
|
1886
|
+
error: error.message,
|
|
1887
|
+
nextRunAt: null,
|
|
1888
|
+
lastRunAt: null,
|
|
1889
|
+
lastError: null,
|
|
1890
|
+
lastDurationMs: null,
|
|
1891
|
+
running: false,
|
|
1892
|
+
runs: 0
|
|
1893
|
+
});
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
this.crons.splice(0, this.crons.length, ...jobs);
|
|
1897
|
+
this.notifyInvalidated([this.cronsTag]);
|
|
1898
|
+
return statuses;
|
|
1899
|
+
}
|
|
1900
|
+
cronJob(entry, timezone, signature) {
|
|
1901
|
+
if (!this.plugins.has(entry.plugin))
|
|
1902
|
+
throw new Error(`${entry.plugin} isn't loaded here`);
|
|
1903
|
+
const name = `${entry.plugin}.${entry.method}`;
|
|
1904
|
+
const method = this.methods.get(name);
|
|
1905
|
+
if (!method || method.plugin !== entry.plugin)
|
|
1906
|
+
throw new Error(`${entry.method} isn't one of ${entry.plugin}'s methods`);
|
|
1907
|
+
if (method.kind === "stream")
|
|
1908
|
+
throw new Error(`${name} is a stream; a cron runs a query or a mutation`);
|
|
1909
|
+
if (method.confirm)
|
|
1910
|
+
throw new Error(`${name} asks before it runs, so nothing may run it on a schedule`);
|
|
1911
|
+
if (!method.input.safeParse({}).success)
|
|
1912
|
+
throw new Error(`${name} needs input; a cron calls it with {}`);
|
|
1913
|
+
const parsed = parseSchedule(entry.schedule);
|
|
1914
|
+
try {
|
|
1915
|
+
checkTimezone(timezone);
|
|
1916
|
+
} catch {
|
|
1917
|
+
throw new Error(`"${timezone}" isn't a time zone`);
|
|
1918
|
+
}
|
|
1919
|
+
const nextRunAt = nextDue(parsed, Date.now(), timezone);
|
|
1920
|
+
if (nextRunAt === null)
|
|
1921
|
+
throw new Error(`"${entry.schedule}" is never due`);
|
|
1922
|
+
return {
|
|
1923
|
+
signature,
|
|
1924
|
+
plugin: entry.plugin,
|
|
1925
|
+
method: name,
|
|
1926
|
+
schedule: entry.schedule,
|
|
1927
|
+
timezone,
|
|
1928
|
+
parsed,
|
|
1929
|
+
nextRunAt,
|
|
1930
|
+
lastRunAt: null,
|
|
1931
|
+
lastDurationMs: null,
|
|
1932
|
+
lastError: null,
|
|
1933
|
+
running: false,
|
|
1934
|
+
runs: 0
|
|
1935
|
+
};
|
|
1936
|
+
}
|
|
1937
|
+
tickCrons(now = Date.now()) {
|
|
1938
|
+
const started = [];
|
|
1939
|
+
for (const job of this.crons) {
|
|
1940
|
+
if (job.nextRunAt === null || job.nextRunAt > now)
|
|
1941
|
+
continue;
|
|
1942
|
+
const due = job.nextRunAt;
|
|
1943
|
+
job.nextRunAt = nextDue(job.parsed, now, job.timezone);
|
|
1944
|
+
if (job.running || now - due > cronLateMs)
|
|
1945
|
+
continue;
|
|
1946
|
+
job.running = true;
|
|
1947
|
+
job.lastRunAt = now;
|
|
1948
|
+
const startedAt = Date.now();
|
|
1949
|
+
started.push(this.call(job.method, {}, { caller: "cron" }).then(() => {
|
|
1950
|
+
job.lastError = null;
|
|
1951
|
+
}, (error) => {
|
|
1952
|
+
job.lastError = toCallError(error).message;
|
|
1953
|
+
}).finally(() => {
|
|
1954
|
+
job.running = false;
|
|
1955
|
+
job.runs += 1;
|
|
1956
|
+
job.lastDurationMs = Date.now() - startedAt;
|
|
1957
|
+
this.notifyInvalidated([this.cronsTag]);
|
|
1958
|
+
}));
|
|
1959
|
+
}
|
|
1960
|
+
if (started.length)
|
|
1961
|
+
this.notifyInvalidated([this.cronsTag]);
|
|
1962
|
+
return started;
|
|
1963
|
+
}
|
|
1964
|
+
logContextFor(plugin, method, chat) {
|
|
1965
|
+
const outer = logContext.getStore();
|
|
1966
|
+
if (outer && outer.plugin === plugin && outer.method === method)
|
|
1967
|
+
return outer;
|
|
1968
|
+
return {
|
|
1969
|
+
plugin,
|
|
1970
|
+
method: method ?? outer?.method,
|
|
1971
|
+
caller: this.callCaller.getStore() ?? outer?.caller,
|
|
1972
|
+
callId: outer?.callId ?? crypto.randomUUID().slice(0, 8),
|
|
1973
|
+
chatId: chat ?? outer?.chatId
|
|
1974
|
+
};
|
|
1975
|
+
}
|
|
1976
|
+
async agent(plugin, ctx, prompt, options) {
|
|
1977
|
+
assertWritable("agents");
|
|
1978
|
+
if (!this.host)
|
|
1979
|
+
throw new MethodError("INTERNAL", "ctx.agent runs only under the RepoGo host.");
|
|
1980
|
+
if (!options.agent)
|
|
1981
|
+
throw new MethodError("VALIDATION", 'ctx.agent needs the agent to run: { agent: "claude" } or "codex".');
|
|
1982
|
+
const started = await this.host(plugin, "chats.start", {
|
|
1983
|
+
cwd: options.project ?? ctx.root,
|
|
1984
|
+
agent: options.agent,
|
|
1985
|
+
title: options.title ?? "",
|
|
1986
|
+
turn: { prompt, config: {}, attachments: [] }
|
|
1987
|
+
});
|
|
1988
|
+
const deadline = Date.now() + parseDuration(options.timeout ?? agentTimeoutMs);
|
|
1989
|
+
let chatId = "";
|
|
1990
|
+
for (;; ) {
|
|
1991
|
+
const remaining = deadline - Date.now();
|
|
1992
|
+
if (remaining <= 0) {
|
|
1993
|
+
throw new MethodError("INTERNAL", `The agent didn't answer in time; its chat ${chatId || started.turn_id} is still running.`);
|
|
1994
|
+
}
|
|
1995
|
+
const waited = await this.host(plugin, "chats.wait", { turn_id: started.turn_id, timeout_ms: Math.min(remaining, agentWaitMs) });
|
|
1996
|
+
chatId = waited.chat_id ?? chatId;
|
|
1997
|
+
if (waited.state === "queued" || waited.state === "running")
|
|
1998
|
+
continue;
|
|
1999
|
+
if (waited.state !== "done")
|
|
2000
|
+
throw new MethodError("INTERNAL", `The agent's turn ${waited.state}${waited.error ? `: ${waited.error}` : ""} (chat ${chatId}).`);
|
|
2001
|
+
return { chatId, text: waited.reply ?? "" };
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
manifest() {
|
|
2005
|
+
const plugins = [...this.plugins.entries()].map(([name, plugin]) => ({
|
|
2006
|
+
name,
|
|
2007
|
+
description: plugin.description ?? null,
|
|
2008
|
+
methods: [...this.methods.values()].filter((method) => method.plugin === name).map((method) => ({
|
|
2009
|
+
name: method.name,
|
|
2010
|
+
kind: method.kind,
|
|
2011
|
+
description: method.description ?? null,
|
|
2012
|
+
confirm: method.confirm ?? null,
|
|
2013
|
+
http: method.http ?? null,
|
|
2014
|
+
input: z5.toJSONSchema(method.input, { io: "input", unrepresentable: "any" }),
|
|
2015
|
+
output: method.output ? z5.toJSONSchema(method.output, { io: "output", unrepresentable: "any" }) : null
|
|
2016
|
+
})),
|
|
2017
|
+
crons: this.crons.filter((job) => job.plugin === name).map(cronState),
|
|
2018
|
+
storage: {
|
|
2019
|
+
buckets: this.storesOf(name, "buckets").map((store) => ({ name: store.name, path: store.path })),
|
|
2020
|
+
kv: this.storesOf(name, "kv").map((store) => ({ name: store.name })),
|
|
2021
|
+
streams: this.storesOf(name, "streams").map((store) => ({ name: store.name, path: store.path }))
|
|
2022
|
+
}
|
|
2023
|
+
}));
|
|
2024
|
+
return { plugins };
|
|
2025
|
+
}
|
|
2026
|
+
get runningCrons() {
|
|
2027
|
+
return this.crons.filter((job) => job.running).map((job) => job.method);
|
|
2028
|
+
}
|
|
2029
|
+
get hasStorage() {
|
|
2030
|
+
return this.stores.size > 0;
|
|
2031
|
+
}
|
|
2032
|
+
async tickStorage(now = Date.now()) {
|
|
2033
|
+
for (const { store } of this.stores.values()) {
|
|
2034
|
+
await store.tick(now).catch((error) => console.error(`[${store.kind}] ${store.name}:`, toCallError(error).message));
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
close() {
|
|
2038
|
+
for (const { store } of this.stores.values())
|
|
2039
|
+
store.close();
|
|
2040
|
+
for (const database of this.databases.values())
|
|
2041
|
+
database.close();
|
|
2042
|
+
}
|
|
2043
|
+
async database(namespace) {
|
|
2044
|
+
let database = this.databases.get(namespace);
|
|
2045
|
+
if (database)
|
|
2046
|
+
return database;
|
|
2047
|
+
await mkdir3(join3(this.dataDir, namespace), { recursive: true });
|
|
2048
|
+
database = await Database.open(join3(this.dataDir, namespace, "data.db"));
|
|
2049
|
+
this.databases.set(namespace, database);
|
|
2050
|
+
return database;
|
|
2051
|
+
}
|
|
2052
|
+
async answerCache(namespace, database) {
|
|
2053
|
+
let cache = this.answerCaches.get(namespace);
|
|
2054
|
+
if (cache)
|
|
2055
|
+
return cache;
|
|
2056
|
+
cache = new AnswerCache(database, namespace, (tags) => this.notifyInvalidated(tags));
|
|
2057
|
+
await cache.setup();
|
|
2058
|
+
this.answerCaches.set(namespace, cache);
|
|
2059
|
+
return cache;
|
|
2060
|
+
}
|
|
2061
|
+
async store(plugin, kind, name, schema, make, extra) {
|
|
2062
|
+
const signature = JSON.stringify([z5.toJSONSchema(schema, { unrepresentable: "any" }), extra ?? null]);
|
|
2063
|
+
const made = make();
|
|
2064
|
+
const found = this.stores.get(made.tag);
|
|
2065
|
+
if (found && found.schema !== signature) {
|
|
2066
|
+
throw new Error(`${plugin}: ${kind} ${name} is declared by another plugin in this namespace with a different schema`);
|
|
2067
|
+
}
|
|
2068
|
+
const store = found?.store ?? made;
|
|
2069
|
+
if (!found) {
|
|
2070
|
+
await store.setup();
|
|
2071
|
+
this.stores.set(store.tag, { store, schema: signature });
|
|
2072
|
+
}
|
|
2073
|
+
store.plugins.add(plugin);
|
|
2074
|
+
return store;
|
|
2075
|
+
}
|
|
2076
|
+
storesOf(plugin, kind) {
|
|
2077
|
+
return [...this.stores.values()].map(({ store }) => store).filter((store) => store.kind === kind && store.plugins.has(plugin));
|
|
2078
|
+
}
|
|
2079
|
+
notifyInvalidated(tags) {
|
|
2080
|
+
for (const listener of this.invalidationListeners)
|
|
2081
|
+
listener(tags);
|
|
2082
|
+
}
|
|
2083
|
+
register(method) {
|
|
2084
|
+
if (this.methods.has(method.name))
|
|
2085
|
+
throw new Error(`${method.name} is defined twice; rename the method or the data`);
|
|
2086
|
+
this.methods.set(method.name, method);
|
|
2087
|
+
}
|
|
2088
|
+
registerGenerated(plugin, name, methods) {
|
|
2089
|
+
for (const [method, { run, ...spec }] of Object.entries(methods)) {
|
|
2090
|
+
this.register({
|
|
2091
|
+
name: `${plugin}.${name}.${method}`,
|
|
2092
|
+
plugin,
|
|
2093
|
+
...spec,
|
|
2094
|
+
run: async (input) => run(input, this.callSignal.getStore() ?? never)
|
|
2095
|
+
});
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
registerCrud(plugin, name, collection, schema) {
|
|
2099
|
+
const prefix = `${plugin}.${name}`;
|
|
2100
|
+
const id = z5.object({ id: z5.string() });
|
|
2101
|
+
const record2 = schema.extend({ id: z5.string(), createdAt: z5.string(), updatedAt: z5.string() });
|
|
2102
|
+
const crud = (method, kind, input, output, description, run, confirm) => this.register({
|
|
2103
|
+
name: `${prefix}.${method}`,
|
|
2104
|
+
plugin,
|
|
2105
|
+
kind,
|
|
2106
|
+
description,
|
|
2107
|
+
confirm,
|
|
2108
|
+
input,
|
|
2109
|
+
output,
|
|
2110
|
+
run: async (value, raw) => run(value, raw)
|
|
2111
|
+
});
|
|
2112
|
+
crud("list", "query", z5.object({
|
|
2113
|
+
limit: z5.number().int().positive().max(500).optional(),
|
|
2114
|
+
offset: z5.number().int().min(0).optional(),
|
|
2115
|
+
orderBy: z5.string().optional(),
|
|
2116
|
+
order: z5.enum(["asc", "desc"]).optional()
|
|
2117
|
+
}), z5.array(record2), `List ${name} records, newest first unless orderBy says`, (input) => collection.list(input));
|
|
2118
|
+
crud("get", "query", id, record2.nullable(), `Get one ${name} record by id, or null`, (input) => collection.get(input.id));
|
|
2119
|
+
crud("create", "mutation", schema, record2, `Create a ${name} record`, (input) => collection.create(input));
|
|
2120
|
+
crud("update", "mutation", id.extend(schema.partial().shape), record2, `Update fields on a ${name} record`, ({ id: id2, ...fields }, raw) => collection.update(id2, Object.fromEntries(Object.entries(fields).filter(([key]) => Object.hasOwn(raw ?? {}, key)))));
|
|
2121
|
+
crud("delete", "mutation", id, z5.null(), `Delete a ${name} record by id`, async (input) => {
|
|
2122
|
+
await collection.delete(input.id);
|
|
2123
|
+
return null;
|
|
2124
|
+
}, "Delete this record?");
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
function fillConfirm(question, input) {
|
|
2128
|
+
const fields = input ?? {};
|
|
2129
|
+
return question.replace(/\{\{\s*([A-Za-z0-9_]+)\s*\}\}/g, (_, field2) => {
|
|
2130
|
+
const value = Object.hasOwn(fields, field2) ? fields[field2] : undefined;
|
|
2131
|
+
return value === undefined || value === null ? "" : String(value);
|
|
2132
|
+
});
|
|
2133
|
+
}
|
|
2134
|
+
function cronState(job) {
|
|
2135
|
+
return {
|
|
2136
|
+
method: job.method,
|
|
2137
|
+
schedule: job.schedule,
|
|
2138
|
+
timezone: job.timezone,
|
|
2139
|
+
nextRunAt: job.nextRunAt,
|
|
2140
|
+
lastRunAt: job.lastRunAt,
|
|
2141
|
+
lastError: job.lastError,
|
|
2142
|
+
lastDurationMs: job.lastDurationMs,
|
|
2143
|
+
running: job.running,
|
|
2144
|
+
runs: job.runs
|
|
2145
|
+
};
|
|
2146
|
+
}
|
|
2147
|
+
function mcpFailure(server, tool, error) {
|
|
2148
|
+
switch (error.code) {
|
|
2149
|
+
case "NOT_READ_ONLY":
|
|
2150
|
+
return new MethodError("FORBIDDEN", `${server}.${tool} changes data, so it runs from a mutation, not a query.`);
|
|
2151
|
+
case "CONFIRM":
|
|
2152
|
+
return new MethodError("FORBIDDEN", `${server}.${tool} may destroy data, so it runs only from a method with a confirm.`);
|
|
2153
|
+
}
|
|
2154
|
+
return new MethodError("INTERNAL", error.message);
|
|
2155
|
+
}
|
|
2156
|
+
function activityParams(activity, project) {
|
|
2157
|
+
const { compactLabel, timer, ...rest } = activity;
|
|
2158
|
+
return {
|
|
2159
|
+
...rest,
|
|
2160
|
+
compact_label: compactLabel,
|
|
2161
|
+
timer: timer && {
|
|
2162
|
+
kind: timer.kind,
|
|
2163
|
+
started_at: timer.kind === "elapsed" ? timer.startedAt : undefined,
|
|
2164
|
+
ends_at: timer.kind === "countdown" ? timer.endsAt : undefined
|
|
2165
|
+
},
|
|
2166
|
+
project
|
|
2167
|
+
};
|
|
2168
|
+
}
|
|
2169
|
+
function isAsyncIterable(value) {
|
|
2170
|
+
return typeof value?.[Symbol.asyncIterator] === "function";
|
|
2171
|
+
}
|
|
2172
|
+
|
|
2173
|
+
// src/self.ts
|
|
2174
|
+
Bun.plugin({
|
|
2175
|
+
name: "@repogo/plugins",
|
|
2176
|
+
setup(build) {
|
|
2177
|
+
build.module("@repogo/plugins", () => ({ exports: exports_src, loader: "object" }));
|
|
2178
|
+
}
|
|
2179
|
+
});
|
|
2180
|
+
|
|
2181
|
+
// src/workflow/testing.ts
|
|
2182
|
+
import { mkdtemp, rm as rm4 } from "fs/promises";
|
|
2183
|
+
import { tmpdir } from "os";
|
|
2184
|
+
import { join as join7, resolve as resolve2 } from "path";
|
|
2185
|
+
|
|
2186
|
+
// src/workflow/service.ts
|
|
2187
|
+
import { isDeepStrictEqual as isDeepStrictEqual2 } from "util";
|
|
2188
|
+
import { readdir as readdir4, rm as rm3, stat as stat3 } from "fs/promises";
|
|
2189
|
+
import { join as join6 } from "path";
|
|
2190
|
+
import { fileURLToPath } from "url";
|
|
2191
|
+
import { z as z7 } from "zod";
|
|
2192
|
+
|
|
2193
|
+
// src/workflow/catalog.ts
|
|
2194
|
+
var {SQL: SQL3 } = globalThis.Bun;
|
|
2195
|
+
import { mkdir as mkdir4 } from "fs/promises";
|
|
2196
|
+
import { join as join4 } from "path";
|
|
2197
|
+
|
|
2198
|
+
// src/workflow/lease.ts
|
|
2199
|
+
var {SQL: SQL2 } = globalThis.Bun;
|
|
2200
|
+
|
|
2201
|
+
// src/workflow/events.ts
|
|
2202
|
+
import { z as z6 } from "zod";
|
|
2203
|
+
var ID = z6.string().min(1).max(200);
|
|
2204
|
+
var sleepId = (position) => `@sleep:${position}`;
|
|
2205
|
+
var SleepID = z6.string().regex(/^@sleep:[1-9]\d*$/);
|
|
2206
|
+
var StepID = ID.refine((value) => !value.startsWith("@sleep:"), "Sleep IDs are reserved for the workflow core.");
|
|
2207
|
+
var Attempt = z6.number().int().positive();
|
|
2208
|
+
var Failure = z6.object({ message: z6.string(), code: z6.string().optional() }).strict();
|
|
2209
|
+
var Step = { stepId: StepID, attempt: Attempt };
|
|
2210
|
+
var EventInputSchema = z6.discriminatedUnion("type", [
|
|
2211
|
+
z6.object({ type: z6.literal("run.started"), workflow: ID, input: z6.json(), timeoutMs: DurationMsSchema, location: RunLocationSchema.optional() }).strict(),
|
|
2212
|
+
z6.object({ type: z6.literal("sleep.scheduled"), sleepId: SleepID, durationMs: DurationMsSchema }).strict(),
|
|
2213
|
+
z6.object({ type: z6.literal("sleep.completed"), sleepId: SleepID }).strict(),
|
|
2214
|
+
z6.object({ type: z6.literal("step.started"), ...Step, policy: StepPolicySchema }).strict(),
|
|
2215
|
+
z6.object({ type: z6.literal("step.completed"), ...Step, result: z6.json() }).strict(),
|
|
2216
|
+
z6.object({ type: z6.literal("step.failed"), ...Step, error: Failure }).strict(),
|
|
2217
|
+
z6.object({ type: z6.literal("step.unknown"), ...Step, error: Failure }).strict(),
|
|
2218
|
+
z6.object({ type: z6.literal("step.retryScheduled"), ...Step, error: Failure, delayMs: z6.number().int().min(0).max(maxDurationMs) }).strict(),
|
|
2219
|
+
z6.object({ type: z6.literal("run.retryRequested"), stepId: StepID }).strict(),
|
|
2220
|
+
z6.object({ type: z6.literal("run.completed"), result: z6.json() }).strict(),
|
|
2221
|
+
z6.object({ type: z6.literal("run.failed"), error: Failure }).strict(),
|
|
2222
|
+
z6.object({ type: z6.literal("run.cancelRequested") }).strict(),
|
|
2223
|
+
z6.object({ type: z6.literal("run.cancelled") }).strict(),
|
|
2224
|
+
z6.object({ type: z6.literal("run.expired"), reason: z6.string().min(1) }).strict(),
|
|
2225
|
+
z6.object({ type: z6.literal("run.unknown"), error: Failure }).strict(),
|
|
2226
|
+
z6.object({ type: z6.literal("run.attention"), error: Failure }).strict(),
|
|
2227
|
+
z6.object({ type: z6.literal("run.restarted") }).strict()
|
|
2228
|
+
]);
|
|
2229
|
+
var Envelope = z6.object({ version: z6.literal(1), seq: z6.number().int().positive(), at: z6.iso.datetime() }).strict();
|
|
2230
|
+
|
|
2231
|
+
class JournalError extends Error {
|
|
2232
|
+
code;
|
|
2233
|
+
constructor(code, message) {
|
|
2234
|
+
super(message);
|
|
2235
|
+
this.code = code;
|
|
2236
|
+
this.name = "JournalError";
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
function eventInput(value) {
|
|
2240
|
+
const parsed = EventInputSchema.safeParse(value);
|
|
2241
|
+
if (!parsed.success)
|
|
2242
|
+
throw new JournalError("INVALID_EVENT", parsed.error.message);
|
|
2243
|
+
return JSON.parse(JSON.stringify(parsed.data));
|
|
2244
|
+
}
|
|
2245
|
+
function storedEvent(value) {
|
|
2246
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
2247
|
+
throw new Error("Expected an event object.");
|
|
2248
|
+
const { version, seq, at, ...input } = value;
|
|
2249
|
+
return { ...eventInput(input), ...Envelope.parse({ version, seq, at }) };
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
// src/workflow/lease.ts
|
|
2253
|
+
async function acquireOwner(path) {
|
|
2254
|
+
const owner = new SQL2({ adapter: "sqlite", filename: path });
|
|
2255
|
+
try {
|
|
2256
|
+
await owner.unsafe("pragma busy_timeout = 0");
|
|
2257
|
+
await owner.unsafe("begin exclusive");
|
|
2258
|
+
} catch (error) {
|
|
2259
|
+
await owner.close();
|
|
2260
|
+
if (error instanceof SQL2.SQLiteError && [5, 6].includes(error.errno & 255))
|
|
2261
|
+
throw new JournalError("LOCKED", "Another executor owns this storage.");
|
|
2262
|
+
throw error;
|
|
2263
|
+
}
|
|
2264
|
+
let closed = false;
|
|
2265
|
+
return async () => {
|
|
2266
|
+
if (closed)
|
|
2267
|
+
return;
|
|
2268
|
+
closed = true;
|
|
2269
|
+
try {
|
|
2270
|
+
await owner.unsafe("rollback");
|
|
2271
|
+
} finally {
|
|
2272
|
+
await owner.close();
|
|
2273
|
+
}
|
|
2274
|
+
};
|
|
2275
|
+
}
|
|
2276
|
+
|
|
2277
|
+
// src/workflow/catalog.ts
|
|
2278
|
+
class WorkflowCatalog {
|
|
2279
|
+
sql;
|
|
2280
|
+
release;
|
|
2281
|
+
constructor(sql, release) {
|
|
2282
|
+
this.sql = sql;
|
|
2283
|
+
this.release = release;
|
|
2284
|
+
}
|
|
2285
|
+
static async open(directory) {
|
|
2286
|
+
await mkdir4(directory, { recursive: true, mode: 448 });
|
|
2287
|
+
const release = await acquireOwner(join4(directory, "owner.db"));
|
|
2288
|
+
let sql = null;
|
|
2289
|
+
try {
|
|
2290
|
+
sql = new SQL3({ adapter: "sqlite", filename: join4(directory, "catalog.db") });
|
|
2291
|
+
await sql.unsafe("pragma journal_mode = wal");
|
|
2292
|
+
await sql.unsafe("pragma synchronous = full");
|
|
2293
|
+
await sql.unsafe("create table if not exists runs (id text primary key, metadata text not null, summary text not null, seen real not null, terminal real, error text)");
|
|
2294
|
+
await sql.unsafe("create table if not exists availability (id integer primary key check(id = 1), seen real not null, offline_since real)");
|
|
2295
|
+
return new WorkflowCatalog(sql, release);
|
|
2296
|
+
} catch (error) {
|
|
2297
|
+
await sql?.close();
|
|
2298
|
+
await release();
|
|
2299
|
+
throw error;
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
2302
|
+
async all() {
|
|
2303
|
+
const rows = await this.sql.unsafe("select metadata, summary, seen, terminal, error from runs order by id");
|
|
2304
|
+
return rows.map((row) => ({ run: JSON.parse(row.metadata), summary: JSON.parse(row.summary), seen: row.seen, terminal: row.terminal, error: row.error }));
|
|
2305
|
+
}
|
|
2306
|
+
async put(value) {
|
|
2307
|
+
await this.sql.unsafe("insert into runs (id,metadata,summary,seen,terminal,error) values (?,?,?,?,?,?) on conflict(id) do update set metadata=excluded.metadata,summary=excluded.summary,seen=excluded.seen,terminal=excluded.terminal,error=excluded.error", [
|
|
2308
|
+
value.run.id,
|
|
2309
|
+
JSON.stringify(value.run),
|
|
2310
|
+
JSON.stringify(value.summary),
|
|
2311
|
+
value.seen,
|
|
2312
|
+
value.terminal,
|
|
2313
|
+
value.error
|
|
2314
|
+
]);
|
|
2315
|
+
}
|
|
2316
|
+
async availability() {
|
|
2317
|
+
const [row] = await this.sql.unsafe("select seen, offline_since from availability where id = 1");
|
|
2318
|
+
return row ? { seen: row.seen, offlineSince: row.offline_since } : null;
|
|
2319
|
+
}
|
|
2320
|
+
async heartbeat(value) {
|
|
2321
|
+
await this.sql.unsafe("insert into availability (id,seen,offline_since) values (1,?,?) on conflict(id) do update set seen=excluded.seen,offline_since=excluded.offline_since", [value.seen, value.offlineSince]);
|
|
2322
|
+
}
|
|
2323
|
+
async remove(id) {
|
|
2324
|
+
await this.sql.unsafe("delete from runs where id = ?", [id]);
|
|
2325
|
+
}
|
|
2326
|
+
async close() {
|
|
2327
|
+
try {
|
|
2328
|
+
await this.sql.close();
|
|
2329
|
+
} finally {
|
|
2330
|
+
await this.release();
|
|
2331
|
+
}
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
// src/workflow/availability.ts
|
|
2336
|
+
import { networkInterfaces } from "os";
|
|
2337
|
+
var networkAvailable = () => {
|
|
2338
|
+
try {
|
|
2339
|
+
return Object.values(networkInterfaces()).flat().some((address) => address && !address.internal && (address.family === "IPv4" ? !address.address.startsWith("169.254.") : !address.address.startsWith("fe80:")));
|
|
2340
|
+
} catch {
|
|
2341
|
+
return null;
|
|
2342
|
+
}
|
|
2343
|
+
};
|
|
2344
|
+
function nextAvailability(saved, now, online) {
|
|
2345
|
+
const offlineSince = online === true ? null : online === false ? saved?.offlineSince ?? now : saved?.offlineSince ?? null;
|
|
2346
|
+
return { seen: now, offlineSince };
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
// src/workflow/clock.ts
|
|
2350
|
+
var systemClock = {
|
|
2351
|
+
now: () => Date.now(),
|
|
2352
|
+
schedule(at, callback) {
|
|
2353
|
+
let timer;
|
|
2354
|
+
let cancelled = false;
|
|
2355
|
+
const tick = () => {
|
|
2356
|
+
if (cancelled)
|
|
2357
|
+
return;
|
|
2358
|
+
const remaining = at - Date.now();
|
|
2359
|
+
if (remaining <= 0)
|
|
2360
|
+
callback();
|
|
2361
|
+
else
|
|
2362
|
+
timer = setTimeout(tick, Math.min(remaining, 2147483647));
|
|
2363
|
+
};
|
|
2364
|
+
timer = setTimeout(tick, Math.min(Math.max(0, at - Date.now()), 2147483647));
|
|
2365
|
+
return () => {
|
|
2366
|
+
cancelled = true;
|
|
2367
|
+
if (timer !== undefined)
|
|
2368
|
+
clearTimeout(timer);
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
};
|
|
2372
|
+
function deadlineFrom(at, milliseconds) {
|
|
2373
|
+
return new Date(Date.parse(at) + milliseconds).toISOString();
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
// src/workflow/interrupted.ts
|
|
2377
|
+
async function settleInterrupted(journal, code) {
|
|
2378
|
+
for (const [stepId, step] of Object.entries(journal.state.steps)) {
|
|
2379
|
+
if (step.status !== "running")
|
|
2380
|
+
continue;
|
|
2381
|
+
await journal.append({
|
|
2382
|
+
type: step.policy.kind === "mutation" ? "step.unknown" : "step.failed",
|
|
2383
|
+
stepId,
|
|
2384
|
+
attempt: step.attempt,
|
|
2385
|
+
error: { code, message: "The workflow stopped before this attempt's outcome was recorded." }
|
|
2386
|
+
});
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
// src/workflow/state.ts
|
|
2391
|
+
import { isDeepStrictEqual } from "util";
|
|
2392
|
+
function initialState() {
|
|
2393
|
+
return {
|
|
2394
|
+
version: 2,
|
|
2395
|
+
lastSeq: 0,
|
|
2396
|
+
status: "pending",
|
|
2397
|
+
workflow: null,
|
|
2398
|
+
input: null,
|
|
2399
|
+
currentStep: null,
|
|
2400
|
+
operationOrder: [],
|
|
2401
|
+
steps: {},
|
|
2402
|
+
sleeps: {},
|
|
2403
|
+
result: null,
|
|
2404
|
+
error: null,
|
|
2405
|
+
expiryReason: null,
|
|
2406
|
+
deadlineAt: null,
|
|
2407
|
+
timeoutMs: null,
|
|
2408
|
+
cancelRequested: false,
|
|
2409
|
+
updatedAt: null,
|
|
2410
|
+
endedAt: null,
|
|
2411
|
+
metadata: null
|
|
2412
|
+
};
|
|
2413
|
+
}
|
|
2414
|
+
function applyEvent(state, event) {
|
|
2415
|
+
if (event.seq !== state.lastSeq + 1)
|
|
2416
|
+
refuse("Event sequence is not contiguous.");
|
|
2417
|
+
const next = { ...state, lastSeq: event.seq, updatedAt: event.at };
|
|
2418
|
+
if (event.type === "run.started") {
|
|
2419
|
+
if (state.status !== "pending")
|
|
2420
|
+
refuse("The run already started.");
|
|
2421
|
+
return { ...next, metadata: event.location ? { ...event.location, workflow: event.workflow, input: event.input, timeoutMs: event.timeoutMs, createdAt: event.at } : null, status: "running", workflow: event.workflow, input: event.input, timeoutMs: event.timeoutMs, deadlineAt: deadlineFrom(event.at, event.timeoutMs) };
|
|
2422
|
+
}
|
|
2423
|
+
if (event.type === "run.retryRequested") {
|
|
2424
|
+
const step2 = Object.hasOwn(state.steps, event.stepId) ? state.steps[event.stepId] : null;
|
|
2425
|
+
if (state.status !== "failed" || state.cancelRequested || !step2 || step2.status !== "failed" || step2.policy.kind !== "query" || !["STEP", "TIMEOUT", "INTERRUPTED"].includes(step2.error?.code ?? "") || Object.values(state.sleeps).some((sleep) => sleep.status !== "completed") || Object.entries(state.steps).some(([id, value]) => id !== event.stepId && value.status !== "completed"))
|
|
2426
|
+
refuse("Only a safely failed query step can be retried.");
|
|
2427
|
+
if (state.deadlineAt && Date.parse(event.at) >= Date.parse(state.deadlineAt))
|
|
2428
|
+
refuse("The workflow lifetime has ended.");
|
|
2429
|
+
const limit2 = step2.attempt + step2.policy.attempts;
|
|
2430
|
+
if (limit2 > 1e4)
|
|
2431
|
+
refuse("The run's attempt limit is exhausted.");
|
|
2432
|
+
return { ...next, endedAt: null, status: "waiting", error: null, currentStep: event.stepId, steps: {
|
|
2433
|
+
...state.steps,
|
|
2434
|
+
[event.stepId]: { ...step2, status: "waiting", limit: limit2, nextAttemptAt: event.at }
|
|
2435
|
+
} };
|
|
2436
|
+
}
|
|
2437
|
+
if (event.type === "run.restarted") {
|
|
2438
|
+
if (state.status !== "attention")
|
|
2439
|
+
refuse("Only a run that needs attention can restart.");
|
|
2440
|
+
const deadlineAt = state.timeoutMs === null ? state.deadlineAt : deadlineFrom(event.at, state.timeoutMs);
|
|
2441
|
+
return { ...next, status: "running", error: null, currentStep: null, operationOrder: [], steps: {}, sleeps: {}, deadlineAt };
|
|
2442
|
+
}
|
|
2443
|
+
if (state.status === "attention") {
|
|
2444
|
+
if (event.type === "run.cancelRequested")
|
|
2445
|
+
return { ...next, cancelRequested: true };
|
|
2446
|
+
if (event.type === "run.cancelled")
|
|
2447
|
+
return { ...next, endedAt: event.at, status: "cancelled" };
|
|
2448
|
+
if (event.type === "run.expired")
|
|
2449
|
+
return { ...next, endedAt: event.at, status: "expired", expiryReason: event.reason };
|
|
2450
|
+
refuse("The run needs attention: restart or cancel it.");
|
|
2451
|
+
}
|
|
2452
|
+
if (state.status !== "running" && state.status !== "waiting")
|
|
2453
|
+
refuse("The run is not active.");
|
|
2454
|
+
if (event.type === "run.attention") {
|
|
2455
|
+
if (Object.values(state.steps).some((step2) => step2.status === "running"))
|
|
2456
|
+
refuse("A step is running.");
|
|
2457
|
+
return { ...next, status: "attention", error: event.error };
|
|
2458
|
+
}
|
|
2459
|
+
if (event.type === "run.cancelRequested")
|
|
2460
|
+
return { ...next, cancelRequested: true };
|
|
2461
|
+
if (event.type === "run.completed") {
|
|
2462
|
+
if (state.cancelRequested)
|
|
2463
|
+
refuse("Cancellation prevents run completion.");
|
|
2464
|
+
if (Object.values(state.sleeps).some((sleep) => sleep.status !== "completed") || Object.values(state.steps).some((step2) => step2.status !== "completed"))
|
|
2465
|
+
refuse("An unfinished or failed step prevents run completion.");
|
|
2466
|
+
return { ...next, endedAt: event.at, status: "completed", result: event.result };
|
|
2467
|
+
}
|
|
2468
|
+
if (event.type === "run.failed" || event.type === "run.unknown") {
|
|
2469
|
+
return { ...next, endedAt: event.at, status: event.type === "run.failed" ? "failed" : "unknown", error: event.error };
|
|
2470
|
+
}
|
|
2471
|
+
if (event.type === "run.cancelled")
|
|
2472
|
+
return { ...next, endedAt: event.at, status: "cancelled" };
|
|
2473
|
+
if (event.type === "run.expired")
|
|
2474
|
+
return { ...next, endedAt: event.at, status: "expired", expiryReason: event.reason };
|
|
2475
|
+
if (event.type === "sleep.scheduled") {
|
|
2476
|
+
if (state.cancelRequested)
|
|
2477
|
+
refuse("Cancellation prevents starting sleeps.");
|
|
2478
|
+
if (event.sleepId !== sleepId(state.operationOrder.length + 1) || Object.hasOwn(state.sleeps, event.sleepId))
|
|
2479
|
+
refuse("Sleep identity must match its operation position.");
|
|
2480
|
+
if (Object.values(state.steps).some((step2) => step2.status !== "completed") || Object.values(state.sleeps).some((sleep2) => sleep2.status !== "completed"))
|
|
2481
|
+
refuse("Another operation is unfinished.");
|
|
2482
|
+
const sleep = { status: "waiting", durationMs: event.durationMs, wakeAt: deadlineFrom(event.at, event.durationMs) };
|
|
2483
|
+
return { ...next, status: "waiting", currentStep: event.sleepId, operationOrder: [...state.operationOrder, event.sleepId], sleeps: { ...state.sleeps, [event.sleepId]: sleep } };
|
|
2484
|
+
}
|
|
2485
|
+
if (event.type === "sleep.completed") {
|
|
2486
|
+
const sleep = Object.hasOwn(state.sleeps, event.sleepId) ? state.sleeps[event.sleepId] : null;
|
|
2487
|
+
if (!sleep || sleep.status !== "waiting")
|
|
2488
|
+
refuse("Only a waiting sleep can complete.");
|
|
2489
|
+
if (state.cancelRequested)
|
|
2490
|
+
refuse("Cancellation prevents completing sleeps.");
|
|
2491
|
+
if (Date.parse(event.at) < Date.parse(sleep.wakeAt))
|
|
2492
|
+
throw new JournalError("NOT_DUE", "The sleep is not due.");
|
|
2493
|
+
return { ...next, status: "running", currentStep: event.sleepId, sleeps: { ...state.sleeps, [event.sleepId]: { ...sleep, status: "completed" } } };
|
|
2494
|
+
}
|
|
2495
|
+
const previous = Object.hasOwn(state.steps, event.stepId) ? state.steps[event.stepId] : undefined;
|
|
2496
|
+
let step;
|
|
2497
|
+
if (event.type === "step.started") {
|
|
2498
|
+
if (state.cancelRequested)
|
|
2499
|
+
refuse("Cancellation prevents starting steps.");
|
|
2500
|
+
if (Object.values(state.sleeps).some((sleep) => sleep.status !== "completed"))
|
|
2501
|
+
refuse("A sleep is unfinished.");
|
|
2502
|
+
if (Object.values(state.steps).some((step2) => step2.status === "failed"))
|
|
2503
|
+
refuse("A failed step prevents starting steps.");
|
|
2504
|
+
if (previous) {
|
|
2505
|
+
if (!isDeepStrictEqual(previous.policy, event.policy))
|
|
2506
|
+
refuse("Retry policy differs from its recorded policy.");
|
|
2507
|
+
if (previous.status !== "waiting" || event.attempt !== previous.attempt + 1)
|
|
2508
|
+
refuse("Only a scheduled retry can start the next attempt.");
|
|
2509
|
+
if (Date.parse(event.at) < Date.parse(previous.nextAttemptAt))
|
|
2510
|
+
throw new JournalError("NOT_DUE", "The retry is not due.");
|
|
2511
|
+
} else if (event.attempt !== 1) {
|
|
2512
|
+
refuse("A new step starts at attempt one.");
|
|
2513
|
+
}
|
|
2514
|
+
if (event.attempt > (previous?.limit ?? event.policy.attempts))
|
|
2515
|
+
refuse("The attempt budget is exhausted.");
|
|
2516
|
+
if (Object.entries(state.steps).some(([id, value]) => id !== event.stepId && active(value)))
|
|
2517
|
+
refuse("Another step is unfinished.");
|
|
2518
|
+
step = { status: "running", attempt: event.attempt, limit: previous?.limit ?? event.policy.attempts, result: null, error: null, nextAttemptAt: null, policy: event.policy };
|
|
2519
|
+
} else {
|
|
2520
|
+
if (!previous || previous.status !== "running" || previous.attempt !== event.attempt)
|
|
2521
|
+
refuse("The event does not match a running attempt.");
|
|
2522
|
+
if (event.type === "step.completed") {
|
|
2523
|
+
step = { ...previous, status: "completed", result: event.result, error: null, nextAttemptAt: null };
|
|
2524
|
+
} else if (event.type === "step.failed" || event.type === "step.unknown") {
|
|
2525
|
+
step = { ...previous, status: event.type === "step.failed" ? "failed" : "unknown", error: event.error };
|
|
2526
|
+
} else {
|
|
2527
|
+
if (previous.policy.kind !== "query" || previous.attempt >= previous.limit)
|
|
2528
|
+
refuse("The step cannot retry.");
|
|
2529
|
+
if (state.cancelRequested)
|
|
2530
|
+
refuse("Cancellation prevents retries.");
|
|
2531
|
+
const nextAttemptAt = deadlineFrom(event.at, event.delayMs);
|
|
2532
|
+
step = { ...previous, status: "waiting", error: event.error, nextAttemptAt };
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
return {
|
|
2536
|
+
...next,
|
|
2537
|
+
status: step.status === "waiting" ? "waiting" : "running",
|
|
2538
|
+
currentStep: event.stepId,
|
|
2539
|
+
operationOrder: previous ? state.operationOrder : [...state.operationOrder, event.stepId],
|
|
2540
|
+
steps: { ...state.steps, [event.stepId]: step }
|
|
2541
|
+
};
|
|
2542
|
+
}
|
|
2543
|
+
function active(step) {
|
|
2544
|
+
return step.status === "running" || step.status === "waiting" || step.status === "unknown";
|
|
2545
|
+
}
|
|
2546
|
+
function refuse(message) {
|
|
2547
|
+
throw new JournalError("INVALID_EVENT", message);
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
// src/workflow/storage.ts
|
|
2551
|
+
import { constants } from "fs";
|
|
2552
|
+
import { mkdir as mkdir5, open, rename as rename2 } from "fs/promises";
|
|
2553
|
+
import { dirname, join as join5, resolve } from "path";
|
|
2554
|
+
var maxRecordBytes = 1024 * 1024;
|
|
2555
|
+
var maxJournalBytes = 64 * 1024 * 1024;
|
|
2556
|
+
|
|
2557
|
+
class FileJournalStorage {
|
|
2558
|
+
directory;
|
|
2559
|
+
events;
|
|
2560
|
+
release;
|
|
2561
|
+
constructor(directory, events, release) {
|
|
2562
|
+
this.directory = directory;
|
|
2563
|
+
this.events = events;
|
|
2564
|
+
this.release = release;
|
|
2565
|
+
}
|
|
2566
|
+
static async open(directory, create = true) {
|
|
2567
|
+
directory = resolve(directory);
|
|
2568
|
+
const created = create ? await mkdir5(directory, { recursive: true, mode: 448 }) : undefined;
|
|
2569
|
+
const ancestor = created ? dirname(resolve(created)) : dirname(directory);
|
|
2570
|
+
for (let parent = dirname(directory);; parent = dirname(parent)) {
|
|
2571
|
+
await syncDirectory(parent);
|
|
2572
|
+
if (parent === ancestor)
|
|
2573
|
+
break;
|
|
2574
|
+
}
|
|
2575
|
+
const release = await acquireOwner(join5(directory, "owner.db"));
|
|
2576
|
+
let events;
|
|
2577
|
+
try {
|
|
2578
|
+
events = await open(join5(directory, "events.jsonl"), constants.O_RDWR | constants.O_APPEND | (create ? constants.O_CREAT : 0), 384);
|
|
2579
|
+
await events.sync();
|
|
2580
|
+
await syncDirectory(directory);
|
|
2581
|
+
return new FileJournalStorage(directory, events, release);
|
|
2582
|
+
} catch (error) {
|
|
2583
|
+
try {
|
|
2584
|
+
await events?.close();
|
|
2585
|
+
} finally {
|
|
2586
|
+
await release();
|
|
2587
|
+
}
|
|
2588
|
+
throw error;
|
|
2589
|
+
}
|
|
2590
|
+
}
|
|
2591
|
+
async readEvents() {
|
|
2592
|
+
if ((await this.events.stat()).size > maxJournalBytes)
|
|
2593
|
+
throw new JournalError("LIMIT", "The run journal exceeds its size limit.");
|
|
2594
|
+
return this.events.readFile();
|
|
2595
|
+
}
|
|
2596
|
+
async append(record2) {
|
|
2597
|
+
await this.events.writeFile(record2);
|
|
2598
|
+
await this.events.sync();
|
|
2599
|
+
}
|
|
2600
|
+
async truncateEvents(bytes) {
|
|
2601
|
+
await this.events.truncate(bytes);
|
|
2602
|
+
await this.events.sync();
|
|
2603
|
+
}
|
|
2604
|
+
async replaceSnapshot(snapshot) {
|
|
2605
|
+
const temporary = join5(this.directory, "state.json.tmp");
|
|
2606
|
+
const file = await open(temporary, "w", 384);
|
|
2607
|
+
try {
|
|
2608
|
+
await file.writeFile(snapshot);
|
|
2609
|
+
await file.sync();
|
|
2610
|
+
} finally {
|
|
2611
|
+
await file.close();
|
|
2612
|
+
}
|
|
2613
|
+
await rename2(temporary, join5(this.directory, "state.json"));
|
|
2614
|
+
await syncDirectory(this.directory);
|
|
2615
|
+
}
|
|
2616
|
+
async close() {
|
|
2617
|
+
try {
|
|
2618
|
+
await this.events.close();
|
|
2619
|
+
} finally {
|
|
2620
|
+
await this.release();
|
|
2621
|
+
}
|
|
2622
|
+
}
|
|
2623
|
+
}
|
|
2624
|
+
async function syncDirectory(path) {
|
|
2625
|
+
const directory = await open(path, "r");
|
|
2626
|
+
try {
|
|
2627
|
+
await directory.sync();
|
|
2628
|
+
} finally {
|
|
2629
|
+
await directory.close();
|
|
2630
|
+
}
|
|
2631
|
+
}
|
|
2632
|
+
|
|
2633
|
+
// src/workflow/journal.ts
|
|
2634
|
+
class RunJournal {
|
|
2635
|
+
storage;
|
|
2636
|
+
clock;
|
|
2637
|
+
value = initialState();
|
|
2638
|
+
listeners = new Set;
|
|
2639
|
+
bytes = 0;
|
|
2640
|
+
queue = Promise.resolve();
|
|
2641
|
+
closing = null;
|
|
2642
|
+
poisoned = false;
|
|
2643
|
+
snapshotFailure = null;
|
|
2644
|
+
constructor(storage, clock) {
|
|
2645
|
+
this.storage = storage;
|
|
2646
|
+
this.clock = clock;
|
|
2647
|
+
}
|
|
2648
|
+
static async open(directory, clock = systemClock) {
|
|
2649
|
+
return RunJournal.recover(await FileJournalStorage.open(directory), clock);
|
|
2650
|
+
}
|
|
2651
|
+
static async openExisting(directory, clock = systemClock) {
|
|
2652
|
+
return RunJournal.recover(await FileJournalStorage.open(directory, false), clock);
|
|
2653
|
+
}
|
|
2654
|
+
static async recover(storage, clock) {
|
|
2655
|
+
const journal = new RunJournal(storage, clock);
|
|
2656
|
+
try {
|
|
2657
|
+
const bytes = Buffer.from(await storage.readEvents());
|
|
2658
|
+
if (bytes.length > maxJournalBytes)
|
|
2659
|
+
throw new JournalError("LIMIT", "The run journal exceeds its size limit.");
|
|
2660
|
+
const complete = bytes.lastIndexOf(10) + 1;
|
|
2661
|
+
let offset = 0;
|
|
2662
|
+
while (offset < complete) {
|
|
2663
|
+
const end = bytes.indexOf(10, offset);
|
|
2664
|
+
if (end - offset + 1 > maxRecordBytes)
|
|
2665
|
+
throw new JournalError("CORRUPT_JOURNAL", "A committed record exceeds its size limit.");
|
|
2666
|
+
try {
|
|
2667
|
+
const line = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(offset, end));
|
|
2668
|
+
journal.value = applyEvent(journal.value, storedEvent(JSON.parse(line)));
|
|
2669
|
+
} catch (error) {
|
|
2670
|
+
throw new JournalError("CORRUPT_JOURNAL", `Invalid committed event at byte ${offset}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2671
|
+
}
|
|
2672
|
+
offset = end + 1;
|
|
2673
|
+
}
|
|
2674
|
+
if (complete !== bytes.length)
|
|
2675
|
+
await storage.truncateEvents(complete);
|
|
2676
|
+
journal.bytes = complete;
|
|
2677
|
+
await journal.checkpoint();
|
|
2678
|
+
return journal;
|
|
2679
|
+
} catch (error) {
|
|
2680
|
+
await storage.close();
|
|
2681
|
+
throw error;
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
onChange(listener) {
|
|
2685
|
+
this.listeners.add(listener);
|
|
2686
|
+
return () => {
|
|
2687
|
+
this.listeners.delete(listener);
|
|
2688
|
+
};
|
|
2689
|
+
}
|
|
2690
|
+
get state() {
|
|
2691
|
+
return structuredClone(this.value);
|
|
2692
|
+
}
|
|
2693
|
+
get snapshotError() {
|
|
2694
|
+
return this.snapshotFailure;
|
|
2695
|
+
}
|
|
2696
|
+
append(input) {
|
|
2697
|
+
if (this.closing)
|
|
2698
|
+
return Promise.reject(new JournalError("CLOSED", "The journal is closing or closed."));
|
|
2699
|
+
let captured;
|
|
2700
|
+
try {
|
|
2701
|
+
captured = structuredClone(eventInput(input));
|
|
2702
|
+
} catch (error) {
|
|
2703
|
+
return Promise.reject(error);
|
|
2704
|
+
}
|
|
2705
|
+
const writing = this.queue.then(async () => {
|
|
2706
|
+
if (this.poisoned)
|
|
2707
|
+
throw new JournalError("POISONED", "A journal write failed. Close and recover before continuing.");
|
|
2708
|
+
const event = { ...captured, version: 1, seq: this.value.lastSeq + 1, at: new Date(this.clock.now()).toISOString() };
|
|
2709
|
+
const next = applyEvent(this.value, event);
|
|
2710
|
+
const record2 = Buffer.from(JSON.stringify(event) + `
|
|
2711
|
+
`);
|
|
2712
|
+
const reserve = event.type === "step.started" || event.type === "sleep.scheduled" ? 3 * maxRecordBytes : 0;
|
|
2713
|
+
if (record2.length > maxRecordBytes || this.bytes + record2.length + reserve > maxJournalBytes) {
|
|
2714
|
+
throw new JournalError("LIMIT", "The event or run journal exceeds its size limit.");
|
|
2715
|
+
}
|
|
2716
|
+
try {
|
|
2717
|
+
await this.storage.append(record2);
|
|
2718
|
+
} catch (error) {
|
|
2719
|
+
this.poisoned = true;
|
|
2720
|
+
throw error;
|
|
2721
|
+
}
|
|
2722
|
+
this.value = next;
|
|
2723
|
+
this.bytes += record2.length;
|
|
2724
|
+
await this.checkpoint();
|
|
2725
|
+
for (const listener of this.listeners) {
|
|
2726
|
+
try {
|
|
2727
|
+
listener();
|
|
2728
|
+
} catch (error) {
|
|
2729
|
+
console.error("Workflow observer failed:", error);
|
|
2730
|
+
}
|
|
2731
|
+
}
|
|
2732
|
+
return structuredClone(event);
|
|
2733
|
+
});
|
|
2734
|
+
this.queue = writing.catch(() => {});
|
|
2735
|
+
return writing;
|
|
2736
|
+
}
|
|
2737
|
+
close() {
|
|
2738
|
+
this.closing ??= this.queue.then(async () => {
|
|
2739
|
+
try {
|
|
2740
|
+
if (!this.poisoned && this.snapshotFailure) {
|
|
2741
|
+
await this.checkpoint();
|
|
2742
|
+
if (this.snapshotFailure)
|
|
2743
|
+
throw this.snapshotFailure;
|
|
2744
|
+
}
|
|
2745
|
+
} finally {
|
|
2746
|
+
await this.storage.close();
|
|
2747
|
+
}
|
|
2748
|
+
});
|
|
2749
|
+
return this.closing;
|
|
2750
|
+
}
|
|
2751
|
+
async checkpoint() {
|
|
2752
|
+
try {
|
|
2753
|
+
await this.storage.replaceSnapshot(JSON.stringify(this.value) + `
|
|
2754
|
+
`);
|
|
2755
|
+
this.snapshotFailure = null;
|
|
2756
|
+
} catch (error) {
|
|
2757
|
+
this.snapshotFailure = error;
|
|
2758
|
+
}
|
|
2759
|
+
}
|
|
2760
|
+
}
|
|
2761
|
+
|
|
2762
|
+
// src/workflow/summary.ts
|
|
2763
|
+
function summarize(run, state) {
|
|
2764
|
+
const step = state.currentStep && Object.hasOwn(state.steps, state.currentStep) ? state.steps[state.currentStep] : null;
|
|
2765
|
+
const sleep = state.currentStep && Object.hasOwn(state.sleeps, state.currentStep) ? state.sleeps[state.currentStep] : null;
|
|
2766
|
+
return {
|
|
2767
|
+
id: run.id,
|
|
2768
|
+
workflow: run.workflow,
|
|
2769
|
+
status: state.status,
|
|
2770
|
+
currentStep: state.currentStep,
|
|
2771
|
+
attempt: step?.attempt ?? 0,
|
|
2772
|
+
createdAt: run.createdAt,
|
|
2773
|
+
updatedAt: state.updatedAt ?? run.createdAt,
|
|
2774
|
+
nextAttemptAt: !activeStatuses.has(state.status) ? null : sleep?.status === "waiting" ? sleep.wakeAt : step?.nextAttemptAt ?? null,
|
|
2775
|
+
deadlineAt: state.deadlineAt
|
|
2776
|
+
};
|
|
2777
|
+
}
|
|
2778
|
+
var terminalStatuses = new Set(["completed", "failed", "cancelled", "expired", "unknown"]);
|
|
2779
|
+
var activeStatuses = new Set(["pending", "running", "waiting"]);
|
|
2780
|
+
var openStatuses = new Set([...activeStatuses, "attention"]);
|
|
2781
|
+
function describe(state) {
|
|
2782
|
+
const bounded = (value) => JSON.stringify(value).length <= 16 * 1024 ? value : null;
|
|
2783
|
+
const result = bounded(state.result);
|
|
2784
|
+
return {
|
|
2785
|
+
error: briefError(state.error),
|
|
2786
|
+
input: bounded(state.input),
|
|
2787
|
+
result,
|
|
2788
|
+
resultTruncated: result === null && state.result !== null,
|
|
2789
|
+
inputTruncated: bounded(state.input) === null && state.input !== null,
|
|
2790
|
+
expiryReason: state.expiryReason,
|
|
2791
|
+
cancelRequested: state.cancelRequested,
|
|
2792
|
+
steps: state.operationOrder.slice(-20).map((id) => {
|
|
2793
|
+
const sleep = Object.hasOwn(state.sleeps, id) ? state.sleeps[id] : null;
|
|
2794
|
+
if (sleep)
|
|
2795
|
+
return { id, kind: "sleep", status: sleep.status, attempt: 0, attempts: 0, error: null, nextAttemptAt: activeStatuses.has(state.status) && sleep.status === "waiting" ? sleep.wakeAt : null, durationMs: sleep.durationMs };
|
|
2796
|
+
const step = state.steps[id];
|
|
2797
|
+
return { id, kind: "step", durationMs: null, status: step.status, attempt: step.attempt, attempts: step.limit, error: briefError(step.error), nextAttemptAt: activeStatuses.has(state.status) ? step.nextAttemptAt : null };
|
|
2798
|
+
})
|
|
2799
|
+
};
|
|
2800
|
+
}
|
|
2801
|
+
function briefError(error) {
|
|
2802
|
+
return error ? { ...error, message: error.message.slice(0, 4000) } : null;
|
|
2803
|
+
}
|
|
2804
|
+
function unavailableDetail(input, error) {
|
|
2805
|
+
return describe({ ...initialState(), input, error: error ? { message: error } : null });
|
|
2806
|
+
}
|
|
2807
|
+
|
|
2808
|
+
// src/workflow/service.ts
|
|
2809
|
+
var scopeMatches = (run, request) => run.dir === request.dir && run.plugin === request.plugin && run.namespace === request.namespace && run.workflow === request.workflow && run.root === request.root;
|
|
2810
|
+
var runReserve = 130 * 1024 * 1024;
|
|
2811
|
+
var storageFull = "The workflow storage budget is fully reserved by unfinished runs and retained history. Finish or cancel runs, or wait for retention cleanup.";
|
|
2812
|
+
var executor = fileURLToPath(new URL(import.meta.url.endsWith(".ts") ? "./executor.ts" : "./workflow-executor.js", import.meta.url));
|
|
2813
|
+
|
|
2814
|
+
class WorkflowService {
|
|
2815
|
+
dataDir;
|
|
2816
|
+
catalog;
|
|
2817
|
+
callbacks;
|
|
2818
|
+
clock;
|
|
2819
|
+
storageLimit;
|
|
2820
|
+
network;
|
|
2821
|
+
queue = Promise.resolve();
|
|
2822
|
+
details = new Map;
|
|
2823
|
+
rows = new Map;
|
|
2824
|
+
unrecovered = new Set;
|
|
2825
|
+
maintenanceErrors = new Map;
|
|
2826
|
+
cancelling = new Set;
|
|
2827
|
+
active = new Map;
|
|
2828
|
+
observers = new Set;
|
|
2829
|
+
timer = null;
|
|
2830
|
+
closing = false;
|
|
2831
|
+
started = false;
|
|
2832
|
+
constructor(dataDir, catalog, callbacks, clock, storageLimit, network) {
|
|
2833
|
+
this.dataDir = dataDir;
|
|
2834
|
+
this.catalog = catalog;
|
|
2835
|
+
this.callbacks = callbacks;
|
|
2836
|
+
this.clock = clock;
|
|
2837
|
+
this.storageLimit = storageLimit;
|
|
2838
|
+
this.network = network;
|
|
2839
|
+
}
|
|
2840
|
+
static async open(dataDir, callbacks, options = {}) {
|
|
2841
|
+
const service = new WorkflowService(dataDir, await WorkflowCatalog.open(join6(dataDir, ".workflows")), callbacks, options.clock ?? systemClock, options.storageLimit ?? 512 * 1024 * 1024, options.network ?? networkAvailable);
|
|
2842
|
+
try {
|
|
2843
|
+
for (const row of await service.catalog.all()) {
|
|
2844
|
+
row.run = RunMetadataSchema.parse(row.run);
|
|
2845
|
+
service.rows.set(row.run.id, row);
|
|
2846
|
+
}
|
|
2847
|
+
await service.recoverCatalog();
|
|
2848
|
+
return service;
|
|
2849
|
+
} catch (error) {
|
|
2850
|
+
await service.catalog.close();
|
|
2851
|
+
throw error;
|
|
2852
|
+
}
|
|
2853
|
+
}
|
|
2854
|
+
directory(run) {
|
|
2855
|
+
return join6(this.dataDir, run.namespace, "workflows", run.id);
|
|
2856
|
+
}
|
|
2857
|
+
serial(work) {
|
|
2858
|
+
const result = this.queue.then(work);
|
|
2859
|
+
this.queue = result.catch(() => {});
|
|
2860
|
+
return result;
|
|
2861
|
+
}
|
|
2862
|
+
observe(listener) {
|
|
2863
|
+
this.observers.add(listener);
|
|
2864
|
+
return () => {
|
|
2865
|
+
this.observers.delete(listener);
|
|
2866
|
+
};
|
|
2867
|
+
}
|
|
2868
|
+
start() {
|
|
2869
|
+
if (!this.closing) {
|
|
2870
|
+
this.started = true;
|
|
2871
|
+
this.wake();
|
|
2872
|
+
}
|
|
2873
|
+
}
|
|
2874
|
+
wake() {
|
|
2875
|
+
this.timer?.();
|
|
2876
|
+
this.timer = null;
|
|
2877
|
+
if (!this.started || this.closing)
|
|
2878
|
+
return;
|
|
2879
|
+
let due = this.clock.now() + (this.rows.size ? 5000 : 60000);
|
|
2880
|
+
for (const row of this.rows.values()) {
|
|
2881
|
+
if (!activeStatuses.has(row.summary.status) && row.summary.status !== "blocked")
|
|
2882
|
+
continue;
|
|
2883
|
+
due = Math.min(due, this.clock.now() + row.run.maxOfflineMs / 2);
|
|
2884
|
+
if (this.active.has(row.run.id) || this.active.size >= 4)
|
|
2885
|
+
continue;
|
|
2886
|
+
due = Math.min(due, row.summary.nextAttemptAt ? Date.parse(row.summary.nextAttemptAt) : this.clock.now());
|
|
2887
|
+
if (row.summary.status !== "blocked" && row.summary.deadlineAt)
|
|
2888
|
+
due = Math.min(due, Date.parse(row.summary.deadlineAt));
|
|
2889
|
+
}
|
|
2890
|
+
this.timer = this.clock.schedule(due, () => {
|
|
2891
|
+
this.tick().finally(() => this.wake()).catch(() => {});
|
|
2892
|
+
});
|
|
2893
|
+
}
|
|
2894
|
+
async save(row) {
|
|
2895
|
+
this.rows.set(row.run.id, row);
|
|
2896
|
+
await this.catalog.put(row);
|
|
2897
|
+
this.callbacks.invalidate([workflowTag(row.run.namespace)]);
|
|
2898
|
+
for (const observer of this.observers) {
|
|
2899
|
+
try {
|
|
2900
|
+
observer(row.summary);
|
|
2901
|
+
} catch (error) {
|
|
2902
|
+
console.error(error);
|
|
2903
|
+
}
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2906
|
+
async request(request) {
|
|
2907
|
+
if (this.closing)
|
|
2908
|
+
throw new MethodError("INTERNAL", "The workflow service is closing.");
|
|
2909
|
+
if (request.operation === "list") {
|
|
2910
|
+
const rows = [...this.rows.values()].filter((row2) => scopeMatches(row2.run, request) && (!request.input.status || row2.summary.status === request.input.status)).sort((a, b) => b.run.createdAt.localeCompare(a.run.createdAt) || b.run.id.localeCompare(a.run.id));
|
|
2911
|
+
const cursor = request.input.cursor;
|
|
2912
|
+
const start = cursor ? rows.findIndex((row2) => row2.run.id === cursor) + 1 : 0;
|
|
2913
|
+
if (cursor && start === 0)
|
|
2914
|
+
throw new MethodError("VALIDATION", "Invalid workflow cursor.");
|
|
2915
|
+
const count = Math.min(100, Math.max(1, Number(request.input.limit ?? 20)));
|
|
2916
|
+
const page = rows.slice(start, start + count);
|
|
2917
|
+
return { items: page.map((row2) => row2.summary), cursor: start + count < rows.length ? page.at(-1).run.id : null };
|
|
2918
|
+
}
|
|
2919
|
+
if (request.operation === "start")
|
|
2920
|
+
return this.serial(() => this.create(request));
|
|
2921
|
+
if (request.operation === "cancelAll")
|
|
2922
|
+
return this.cancelAll(request);
|
|
2923
|
+
const row = this.rows.get(String(request.input.id));
|
|
2924
|
+
if (!row || !scopeMatches(row.run, request))
|
|
2925
|
+
throw new MethodError("NOT_FOUND", "No workflow run in this project.");
|
|
2926
|
+
if (request.operation === "retry")
|
|
2927
|
+
return this.serial(async () => {
|
|
2928
|
+
if (await this.bytes(row.run.id) > this.storageLimit)
|
|
2929
|
+
throw new MethodError("VALIDATION", storageFull);
|
|
2930
|
+
const active2 = this.active.get(row.run.id);
|
|
2931
|
+
if (active2) {
|
|
2932
|
+
await active2.ended;
|
|
2933
|
+
this.active.delete(row.run.id);
|
|
2934
|
+
}
|
|
2935
|
+
const journal = await this.openJournal(row.run);
|
|
2936
|
+
try {
|
|
2937
|
+
const stepId = journal.state.currentStep;
|
|
2938
|
+
if (!stepId)
|
|
2939
|
+
throw new MethodError("VALIDATION", "This run has no failed query to retry.");
|
|
2940
|
+
try {
|
|
2941
|
+
await journal.append({ type: "run.retryRequested", stepId });
|
|
2942
|
+
} catch (error) {
|
|
2943
|
+
throw new MethodError("VALIDATION", toCallError(error).message);
|
|
2944
|
+
}
|
|
2945
|
+
row.summary = summarize(row.run, journal.state);
|
|
2946
|
+
row.terminal = null;
|
|
2947
|
+
row.error = null;
|
|
2948
|
+
row.seen = this.clock.now();
|
|
2949
|
+
await this.save(row);
|
|
2950
|
+
this.wake();
|
|
2951
|
+
return row.summary;
|
|
2952
|
+
} finally {
|
|
2953
|
+
await journal.close();
|
|
2954
|
+
}
|
|
2955
|
+
});
|
|
2956
|
+
if (request.operation === "restart")
|
|
2957
|
+
return this.serial(async () => {
|
|
2958
|
+
const journal = await this.openJournal(row.run);
|
|
2959
|
+
try {
|
|
2960
|
+
if (journal.state.status !== "attention")
|
|
2961
|
+
throw new MethodError("VALIDATION", "Only a run that needs attention can restart.");
|
|
2962
|
+
await journal.append({ type: "run.restarted" });
|
|
2963
|
+
row.summary = summarize(row.run, journal.state);
|
|
2964
|
+
row.error = null;
|
|
2965
|
+
row.seen = this.clock.now();
|
|
2966
|
+
await this.save(row);
|
|
2967
|
+
this.wake();
|
|
2968
|
+
return row.summary;
|
|
2969
|
+
} finally {
|
|
2970
|
+
await journal.close();
|
|
2971
|
+
}
|
|
2972
|
+
});
|
|
2973
|
+
if (request.operation === "cancel")
|
|
2974
|
+
return this.cancel(row);
|
|
2975
|
+
return this.serial(async () => {
|
|
2976
|
+
const fallback = unavailableDetail(row.run.input, row.error);
|
|
2977
|
+
if (this.active.has(row.run.id))
|
|
2978
|
+
return { ...row.summary, ...this.details.get(row.run.id) ?? fallback };
|
|
2979
|
+
let journal;
|
|
2980
|
+
try {
|
|
2981
|
+
journal = await this.openJournal(row.run);
|
|
2982
|
+
} catch (error) {
|
|
2983
|
+
if (row.summary.status === "blocked" || row.error)
|
|
2984
|
+
return { ...row.summary, ...fallback };
|
|
2985
|
+
throw error;
|
|
2986
|
+
}
|
|
2987
|
+
try {
|
|
2988
|
+
const state = journal.state;
|
|
2989
|
+
const summary = row.summary.status === "blocked" ? row.summary : summarize(row.run, state);
|
|
2990
|
+
return { ...summary, ...describe(state), error: briefError(state.error ?? (row.error ? { message: row.error } : null)) };
|
|
2991
|
+
} finally {
|
|
2992
|
+
await journal.close();
|
|
2993
|
+
}
|
|
2994
|
+
});
|
|
2995
|
+
}
|
|
2996
|
+
async cancelAll(request) {
|
|
2997
|
+
const status = request.input.status;
|
|
2998
|
+
const rows = [...this.rows.values()].filter((row) => scopeMatches(row.run, request) && (openStatuses.has(row.summary.status) || row.summary.status === "blocked") && (!status || row.summary.status === status));
|
|
2999
|
+
for (const row of rows)
|
|
3000
|
+
await this.cancel(row);
|
|
3001
|
+
return { cancelled: rows.length };
|
|
3002
|
+
}
|
|
3003
|
+
cancel(row) {
|
|
3004
|
+
const active2 = this.active.get(row.run.id);
|
|
3005
|
+
if (active2 && active2.worker.exitCode === null)
|
|
3006
|
+
return this.cancelActive(row, active2);
|
|
3007
|
+
return this.serial(async () => {
|
|
3008
|
+
if (active2) {
|
|
3009
|
+
await active2.ended;
|
|
3010
|
+
this.active.delete(row.run.id);
|
|
3011
|
+
}
|
|
3012
|
+
return this.cancelIdle(row);
|
|
3013
|
+
});
|
|
3014
|
+
}
|
|
3015
|
+
async cancelIdle(row) {
|
|
3016
|
+
const journal = await this.openJournal(row.run);
|
|
3017
|
+
try {
|
|
3018
|
+
if (openStatuses.has(journal.state.status)) {
|
|
3019
|
+
await journal.append({ type: "run.cancelRequested" });
|
|
3020
|
+
await settleInterrupted(journal, "CANCELLED");
|
|
3021
|
+
await journal.append({ type: "run.cancelled" });
|
|
3022
|
+
}
|
|
3023
|
+
row.summary = summarize(row.run, journal.state);
|
|
3024
|
+
if (terminalStatuses.has(row.summary.status))
|
|
3025
|
+
row.terminal ??= Date.parse(row.summary.updatedAt);
|
|
3026
|
+
await this.save(row);
|
|
3027
|
+
return row.summary;
|
|
3028
|
+
} finally {
|
|
3029
|
+
await journal.close();
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
async cancelActive(row, active2) {
|
|
3033
|
+
this.cancelling.add(row.run.id);
|
|
3034
|
+
const ack = Promise.withResolvers();
|
|
3035
|
+
active2.acknowledge = ack.resolve;
|
|
3036
|
+
let timer;
|
|
3037
|
+
try {
|
|
3038
|
+
active2.worker.send({ type: "cancel" });
|
|
3039
|
+
const accepted = await Promise.race([ack.promise, active2.worker.exited.then(() => null), new Promise((resolve2) => {
|
|
3040
|
+
timer = setTimeout(() => {
|
|
3041
|
+
active2.worker.kill("SIGKILL");
|
|
3042
|
+
resolve2(null);
|
|
3043
|
+
}, 1000);
|
|
3044
|
+
})]);
|
|
3045
|
+
if (accepted)
|
|
3046
|
+
return accepted;
|
|
3047
|
+
await active2.worker.exited;
|
|
3048
|
+
return await this.serial(async () => {
|
|
3049
|
+
this.active.delete(row.run.id);
|
|
3050
|
+
return this.cancelIdle(row);
|
|
3051
|
+
});
|
|
3052
|
+
} finally {
|
|
3053
|
+
if (timer)
|
|
3054
|
+
clearTimeout(timer);
|
|
3055
|
+
delete active2.acknowledge;
|
|
3056
|
+
this.cancelling.delete(row.run.id);
|
|
3057
|
+
this.wake();
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3060
|
+
async create(request) {
|
|
3061
|
+
const input = z7.json().parse(request.input.input);
|
|
3062
|
+
const key = request.input.key ? String(request.input.key) : null;
|
|
3063
|
+
const found = key ? [...this.rows.values()].find((row2) => scopeMatches(row2.run, request) && row2.run.key === key) : undefined;
|
|
3064
|
+
if (found) {
|
|
3065
|
+
if (!isDeepStrictEqual2(found.run.input, input))
|
|
3066
|
+
throw new MethodError("VALIDATION", "This workflow key already has different input.");
|
|
3067
|
+
return found.summary;
|
|
3068
|
+
}
|
|
3069
|
+
if (!request.settings)
|
|
3070
|
+
throw new MethodError("VALIDATION", "Workflow settings are required.");
|
|
3071
|
+
if (await this.bytes() + runReserve > this.storageLimit)
|
|
3072
|
+
throw new MethodError("VALIDATION", storageFull);
|
|
3073
|
+
let run = RunMetadataSchema.parse({ dir: request.dir, root: request.root, namespace: request.namespace, plugin: request.plugin, workflow: request.workflow, caller: request.caller ?? "code", input, key, id: crypto.randomUUID(), createdAt: new Date(this.clock.now()).toISOString(), ...request.settings });
|
|
3074
|
+
const journal = await RunJournal.open(this.directory(run), this.clock);
|
|
3075
|
+
let summary;
|
|
3076
|
+
try {
|
|
3077
|
+
const { workflow, input: savedInput, timeoutMs, createdAt, ...location } = run;
|
|
3078
|
+
await journal.append({ type: "run.started", workflow, input: savedInput, timeoutMs, location });
|
|
3079
|
+
run = journal.state.metadata;
|
|
3080
|
+
summary = summarize(run, journal.state);
|
|
3081
|
+
} finally {
|
|
3082
|
+
await journal.close();
|
|
3083
|
+
}
|
|
3084
|
+
const row = { run, summary, seen: this.clock.now(), terminal: null, error: null };
|
|
3085
|
+
await this.save(row);
|
|
3086
|
+
if (!await this.catalog.availability())
|
|
3087
|
+
await this.catalog.heartbeat(nextAvailability(null, this.clock.now(), this.network()));
|
|
3088
|
+
this.wake();
|
|
3089
|
+
return summary;
|
|
3090
|
+
}
|
|
3091
|
+
tick() {
|
|
3092
|
+
return this.serial(async () => {
|
|
3093
|
+
if (this.closing)
|
|
3094
|
+
return;
|
|
3095
|
+
const now = this.clock.now();
|
|
3096
|
+
if (!this.rows.size) {
|
|
3097
|
+
await this.recoverCatalog();
|
|
3098
|
+
return;
|
|
3099
|
+
}
|
|
3100
|
+
const availability = await this.catalog.availability();
|
|
3101
|
+
const next = nextAvailability(availability, now, this.network());
|
|
3102
|
+
const offlineSince = availability?.offlineSince ?? next.offlineSince;
|
|
3103
|
+
for (const row of [...this.rows.values()]) {
|
|
3104
|
+
try {
|
|
3105
|
+
const networkExpired = offlineSince !== null && now - Math.max(offlineSince, Date.parse(row.run.createdAt)) >= row.run.maxOfflineMs;
|
|
3106
|
+
const unavailable = availability !== null && now - Math.max(availability.seen, Date.parse(row.run.createdAt)) >= row.run.maxOfflineMs;
|
|
3107
|
+
const deadlineReached = row.summary.deadlineAt !== null && now >= Date.parse(row.summary.deadlineAt);
|
|
3108
|
+
if (this.active.has(row.run.id)) {
|
|
3109
|
+
if (!networkExpired && !unavailable && !deadlineReached) {
|
|
3110
|
+
row.seen = now;
|
|
3111
|
+
continue;
|
|
3112
|
+
}
|
|
3113
|
+
const active2 = this.active.get(row.run.id);
|
|
3114
|
+
active2.worker.kill("SIGKILL");
|
|
3115
|
+
await active2.worker.exited;
|
|
3116
|
+
this.active.delete(row.run.id);
|
|
3117
|
+
}
|
|
3118
|
+
if (row.terminal !== null && now - row.terminal < row.run.retentionMs)
|
|
3119
|
+
continue;
|
|
3120
|
+
if (availability && !networkExpired && !unavailable && !deadlineReached && row.summary.nextAttemptAt && Date.parse(row.summary.nextAttemptAt) > now) {
|
|
3121
|
+
row.seen = now;
|
|
3122
|
+
continue;
|
|
3123
|
+
}
|
|
3124
|
+
const blockedUntil = row.summary.status === "blocked" ? row.summary.nextAttemptAt : null;
|
|
3125
|
+
try {
|
|
3126
|
+
const journal = await this.openJournal(row.run);
|
|
3127
|
+
try {
|
|
3128
|
+
const state = journal.state;
|
|
3129
|
+
const offline = networkExpired || unavailable || !availability && now - row.seen >= row.run.maxOfflineMs;
|
|
3130
|
+
const expired = state.deadlineAt !== null && now >= Date.parse(state.deadlineAt);
|
|
3131
|
+
if (openStatuses.has(state.status) && (offline || expired)) {
|
|
3132
|
+
await settleInterrupted(journal, "EXPIRED");
|
|
3133
|
+
await journal.append({ type: "run.expired", reason: offline ? "Environment unavailable beyond the workflow offline limit." : "Workflow deadline reached." });
|
|
3134
|
+
}
|
|
3135
|
+
row.summary = summarize(row.run, journal.state);
|
|
3136
|
+
if (activeStatuses.has(row.summary.status) && blockedUntil && Date.parse(blockedUntil) > now)
|
|
3137
|
+
row.summary = { ...row.summary, status: "blocked", nextAttemptAt: blockedUntil };
|
|
3138
|
+
row.seen = now;
|
|
3139
|
+
if (journal.state.endedAt)
|
|
3140
|
+
row.terminal ??= Date.parse(journal.state.endedAt);
|
|
3141
|
+
await this.save(row);
|
|
3142
|
+
} finally {
|
|
3143
|
+
await journal.close();
|
|
3144
|
+
}
|
|
3145
|
+
} catch (error) {
|
|
3146
|
+
row.error = toCallError(error).message;
|
|
3147
|
+
if (!terminalStatuses.has(row.summary.status)) {
|
|
3148
|
+
const expired = now - row.seen >= row.run.maxOfflineMs || row.summary.deadlineAt !== null && now >= Date.parse(row.summary.deadlineAt);
|
|
3149
|
+
row.summary = { ...row.summary, status: expired ? "expired" : "blocked", nextAttemptAt: expired ? null : new Date(now + 60000).toISOString() };
|
|
3150
|
+
if (expired)
|
|
3151
|
+
row.terminal ??= now;
|
|
3152
|
+
}
|
|
3153
|
+
await this.save(row);
|
|
3154
|
+
}
|
|
3155
|
+
if (row.terminal !== null && now - row.terminal >= row.run.retentionMs) {
|
|
3156
|
+
await rm3(this.directory(row.run), { recursive: true, force: true });
|
|
3157
|
+
await this.catalog.remove(row.run.id);
|
|
3158
|
+
this.rows.delete(row.run.id);
|
|
3159
|
+
this.details.delete(row.run.id);
|
|
3160
|
+
this.maintenanceErrors.delete(row.run.id);
|
|
3161
|
+
this.callbacks.invalidate([workflowTag(row.run.namespace)]);
|
|
3162
|
+
continue;
|
|
3163
|
+
}
|
|
3164
|
+
if (!this.cancelling.has(row.run.id) && activeStatuses.has(row.summary.status) && this.active.size < 4 && (!row.summary.nextAttemptAt || now >= Date.parse(row.summary.nextAttemptAt)))
|
|
3165
|
+
await this.launch(row);
|
|
3166
|
+
this.maintenanceErrors.delete(row.run.id);
|
|
3167
|
+
} catch (error) {
|
|
3168
|
+
row.error = toCallError(error).message;
|
|
3169
|
+
if (this.maintenanceErrors.get(row.run.id) !== row.error)
|
|
3170
|
+
console.error("Workflow maintenance failed:", row.run.id, row.error);
|
|
3171
|
+
this.maintenanceErrors.set(row.run.id, row.error);
|
|
3172
|
+
}
|
|
3173
|
+
}
|
|
3174
|
+
await this.catalog.heartbeat(next);
|
|
3175
|
+
await this.recoverCatalog();
|
|
3176
|
+
});
|
|
3177
|
+
}
|
|
3178
|
+
async launch(row) {
|
|
3179
|
+
try {
|
|
3180
|
+
const worker = Bun.spawn([process.execPath, "--no-env-file", executor], {
|
|
3181
|
+
cwd: row.run.dir,
|
|
3182
|
+
stdin: "ignore",
|
|
3183
|
+
stdout: "ignore",
|
|
3184
|
+
stderr: "pipe",
|
|
3185
|
+
ipc: (message) => onMessage(message),
|
|
3186
|
+
serialization: "advanced"
|
|
3187
|
+
});
|
|
3188
|
+
const stderr = stderrTail(worker.stderr);
|
|
3189
|
+
let finish;
|
|
3190
|
+
const ended = new Promise((resolve2) => {
|
|
3191
|
+
finish = resolve2;
|
|
3192
|
+
});
|
|
3193
|
+
const active2 = { worker, ended, finish, error: null };
|
|
3194
|
+
this.active.set(row.run.id, active2);
|
|
3195
|
+
let finished = false;
|
|
3196
|
+
const complete = () => {
|
|
3197
|
+
if (finished)
|
|
3198
|
+
return;
|
|
3199
|
+
finished = true;
|
|
3200
|
+
active2.finish();
|
|
3201
|
+
if (this.closing)
|
|
3202
|
+
return;
|
|
3203
|
+
this.serial(async () => {
|
|
3204
|
+
if (this.active.get(row.run.id) !== active2)
|
|
3205
|
+
return;
|
|
3206
|
+
this.active.delete(row.run.id);
|
|
3207
|
+
row.error = active2.error;
|
|
3208
|
+
if (active2.error && !terminalStatuses.has(row.summary.status)) {
|
|
3209
|
+
row.summary = { ...row.summary, status: "blocked", nextAttemptAt: new Date(this.clock.now() + 60000).toISOString() };
|
|
3210
|
+
await this.save(row);
|
|
3211
|
+
}
|
|
3212
|
+
}).then(() => this.tick()).then(() => this.wake()).catch(() => {});
|
|
3213
|
+
};
|
|
3214
|
+
const onMessage = (message) => {
|
|
3215
|
+
if (this.closing)
|
|
3216
|
+
return;
|
|
3217
|
+
if (message.type === "host") {
|
|
3218
|
+
this.callbacks.host(message.plugin, message.method, message.params).then((result) => worker.send({ type: "hosted", id: message.id, result }), (error) => worker.send({ type: "hosted", id: message.id, error: toCallError(error) })).catch(() => {});
|
|
3219
|
+
} else if (message.type === "state") {
|
|
3220
|
+
const summary = RunSummarySchema.safeParse(message.summary);
|
|
3221
|
+
if (!summary.success || summary.data.id !== row.run.id || summary.data.workflow !== row.run.workflow || summary.data.createdAt !== row.run.createdAt) {
|
|
3222
|
+
active2.error = "The workflow executor reported an invalid run identity.";
|
|
3223
|
+
return;
|
|
3224
|
+
}
|
|
3225
|
+
if (message.detail.cancelRequested)
|
|
3226
|
+
active2.acknowledge?.(message.summary);
|
|
3227
|
+
this.serial(async () => {
|
|
3228
|
+
if (this.active.get(row.run.id) !== active2)
|
|
3229
|
+
return;
|
|
3230
|
+
row.summary = message.summary;
|
|
3231
|
+
this.details.set(row.run.id, message.detail);
|
|
3232
|
+
row.seen = this.clock.now();
|
|
3233
|
+
await this.save(row);
|
|
3234
|
+
});
|
|
3235
|
+
} else if (message.type === "invalidated")
|
|
3236
|
+
this.callbacks.invalidate(message.tags);
|
|
3237
|
+
else if (message.type === "notice")
|
|
3238
|
+
this.callbacks.notify(message.notice);
|
|
3239
|
+
else if (message.type === "log")
|
|
3240
|
+
this.callbacks.logs(row.run.dir, message.records);
|
|
3241
|
+
else if (message.type === "error")
|
|
3242
|
+
active2.error = message.message;
|
|
3243
|
+
else if (message.type === "done") {}
|
|
3244
|
+
};
|
|
3245
|
+
worker.exited.then(async (code) => {
|
|
3246
|
+
if (code !== 0 && !active2.error)
|
|
3247
|
+
active2.error = (await stderr).slice(-4000) || `Workflow executor exited ${code}.`;
|
|
3248
|
+
complete();
|
|
3249
|
+
});
|
|
3250
|
+
worker.send({ type: "start", run: row.run, directory: this.directory(row.run), dataDir: this.dataDir });
|
|
3251
|
+
} catch (error) {
|
|
3252
|
+
row.summary = { ...row.summary, status: "blocked", nextAttemptAt: new Date(this.clock.now() + 60000).toISOString() };
|
|
3253
|
+
row.error = toCallError(error).message;
|
|
3254
|
+
await this.save(row);
|
|
3255
|
+
}
|
|
3256
|
+
}
|
|
3257
|
+
async openJournal(run) {
|
|
3258
|
+
const path = this.directory(run);
|
|
3259
|
+
if (!(await stat3(join6(path, "events.jsonl"))).size)
|
|
3260
|
+
throw new Error("The workflow journal is missing or empty; execution is blocked.");
|
|
3261
|
+
return RunJournal.openExisting(path, this.clock);
|
|
3262
|
+
}
|
|
3263
|
+
async recoverCatalog() {
|
|
3264
|
+
let complete = true;
|
|
3265
|
+
const namespaces = await readdir4(this.dataDir, { withFileTypes: true });
|
|
3266
|
+
for (const namespace of namespaces) {
|
|
3267
|
+
if (!namespace.isDirectory() || !/^[a-z0-9][a-z0-9.-]*$/.test(namespace.name))
|
|
3268
|
+
continue;
|
|
3269
|
+
const base = join6(this.dataDir, namespace.name, "workflows");
|
|
3270
|
+
const entries = await readdir4(base, { withFileTypes: true }).catch((error) => {
|
|
3271
|
+
if (error.code !== "ENOENT") {
|
|
3272
|
+
complete = false;
|
|
3273
|
+
if (!this.unrecovered.has(base))
|
|
3274
|
+
console.error("Workflow discovery blocked:", base, toCallError(error).message);
|
|
3275
|
+
this.unrecovered.add(base);
|
|
3276
|
+
}
|
|
3277
|
+
return [];
|
|
3278
|
+
});
|
|
3279
|
+
for (const entry of entries) {
|
|
3280
|
+
if (!entry.isDirectory() || !z7.string().uuid().safeParse(entry.name).success || this.rows.has(entry.name))
|
|
3281
|
+
continue;
|
|
3282
|
+
const directory = join6(base, entry.name);
|
|
3283
|
+
try {
|
|
3284
|
+
const history = await stat3(join6(directory, "events.jsonl")).catch((error) => {
|
|
3285
|
+
if (error.code !== "ENOENT")
|
|
3286
|
+
throw error;
|
|
3287
|
+
return null;
|
|
3288
|
+
});
|
|
3289
|
+
if (!history?.size)
|
|
3290
|
+
continue;
|
|
3291
|
+
const journal = await RunJournal.openExisting(directory, this.clock);
|
|
3292
|
+
try {
|
|
3293
|
+
const state = journal.state;
|
|
3294
|
+
const run = state.metadata;
|
|
3295
|
+
if (!run || run.id !== entry.name || run.namespace !== namespace.name)
|
|
3296
|
+
throw new Error("Workflow journal location is invalid.");
|
|
3297
|
+
const row = { run, summary: summarize(run, state), seen: Date.parse(state.updatedAt), terminal: state.endedAt ? Date.parse(state.endedAt) : null, error: null };
|
|
3298
|
+
await this.catalog.put(row);
|
|
3299
|
+
this.rows.set(run.id, row);
|
|
3300
|
+
this.unrecovered.delete(directory);
|
|
3301
|
+
} finally {
|
|
3302
|
+
await journal.close();
|
|
3303
|
+
}
|
|
3304
|
+
} catch (error) {
|
|
3305
|
+
complete = false;
|
|
3306
|
+
if (!this.unrecovered.has(directory))
|
|
3307
|
+
console.error("Workflow discovery blocked:", directory, toCallError(error).message);
|
|
3308
|
+
this.unrecovered.add(directory);
|
|
3309
|
+
}
|
|
3310
|
+
}
|
|
3311
|
+
}
|
|
3312
|
+
return complete;
|
|
3313
|
+
}
|
|
3314
|
+
async bytes(reserveId) {
|
|
3315
|
+
const tree = async (path) => {
|
|
3316
|
+
const entries = await readdir4(path, { withFileTypes: true }).catch((error) => {
|
|
3317
|
+
if (error.code !== "ENOENT")
|
|
3318
|
+
throw error;
|
|
3319
|
+
return [];
|
|
3320
|
+
});
|
|
3321
|
+
let size = 0;
|
|
3322
|
+
for (const entry of entries) {
|
|
3323
|
+
const name = join6(path, entry.name);
|
|
3324
|
+
if (entry.isDirectory())
|
|
3325
|
+
size += await tree(name);
|
|
3326
|
+
else
|
|
3327
|
+
size += await stat3(name).then((info) => info.size, (error) => {
|
|
3328
|
+
if (error.code !== "ENOENT")
|
|
3329
|
+
throw error;
|
|
3330
|
+
return 0;
|
|
3331
|
+
});
|
|
3332
|
+
}
|
|
3333
|
+
return size;
|
|
3334
|
+
};
|
|
3335
|
+
let bytes = 0;
|
|
3336
|
+
for (const row of this.rows.values()) {
|
|
3337
|
+
const size = await tree(this.directory(row.run));
|
|
3338
|
+
const reserved = openStatuses.has(row.summary.status) || row.summary.status === "blocked" || row.run.id === reserveId;
|
|
3339
|
+
bytes += reserved ? Math.max(size, runReserve) : size;
|
|
3340
|
+
}
|
|
3341
|
+
return bytes;
|
|
3342
|
+
}
|
|
3343
|
+
async close() {
|
|
3344
|
+
if (this.closing)
|
|
3345
|
+
return;
|
|
3346
|
+
this.closing = true;
|
|
3347
|
+
this.timer?.();
|
|
3348
|
+
this.timer = null;
|
|
3349
|
+
const active2 = [...this.active.values()];
|
|
3350
|
+
for (const value of active2)
|
|
3351
|
+
value.worker.send({ type: "suspend" });
|
|
3352
|
+
let timer;
|
|
3353
|
+
try {
|
|
3354
|
+
await Promise.race([Promise.all(active2.map((value) => value.worker.exited)), new Promise((resolve2) => {
|
|
3355
|
+
timer = setTimeout(resolve2, 1000);
|
|
3356
|
+
})]);
|
|
3357
|
+
} finally {
|
|
3358
|
+
if (timer)
|
|
3359
|
+
clearTimeout(timer);
|
|
3360
|
+
}
|
|
3361
|
+
for (const value of active2)
|
|
3362
|
+
if (value.worker.exitCode === null)
|
|
3363
|
+
value.worker.kill("SIGKILL");
|
|
3364
|
+
await Promise.all(active2.map((value) => value.worker.exited));
|
|
3365
|
+
await this.queue;
|
|
3366
|
+
await this.catalog.close();
|
|
3367
|
+
}
|
|
3368
|
+
}
|
|
3369
|
+
async function stderrTail(stream) {
|
|
3370
|
+
const decoder = new TextDecoder;
|
|
3371
|
+
let tail = "";
|
|
3372
|
+
for await (const chunk of stream)
|
|
3373
|
+
tail = (tail + decoder.decode(chunk, { stream: true })).slice(-4000);
|
|
3374
|
+
return (tail + decoder.decode()).slice(-4000);
|
|
3375
|
+
}
|
|
3376
|
+
|
|
3377
|
+
// src/workflow/testing.ts
|
|
3378
|
+
async function testWorkflowRuntime(directory, options = {}) {
|
|
3379
|
+
const dataDir = await mkdtemp(join7(tmpdir(), "repogo-workflow-test-"));
|
|
3380
|
+
let runtime;
|
|
3381
|
+
const host = async (_plugin, method, params) => {
|
|
3382
|
+
if (!options.host)
|
|
3383
|
+
throw new MethodError("FORBIDDEN", `This test does not serve ${method}.`);
|
|
3384
|
+
return options.host(method, params);
|
|
3385
|
+
};
|
|
3386
|
+
const service = await WorkflowService.open(dataDir, {
|
|
3387
|
+
host,
|
|
3388
|
+
invalidate: () => {},
|
|
3389
|
+
notify: () => {},
|
|
3390
|
+
logs: () => {}
|
|
3391
|
+
});
|
|
3392
|
+
try {
|
|
3393
|
+
runtime = await Runtime.open({ dataDir, root: resolve2(options.root ?? directory), host, workflows: (request) => service.request(request) });
|
|
3394
|
+
await runtime.loadDir(resolve2(directory));
|
|
3395
|
+
service.start();
|
|
3396
|
+
return {
|
|
3397
|
+
call: async (method, input = {}, call = {}) => (await runtime.call(method, input, call)).result,
|
|
3398
|
+
manifest: () => runtime.manifest(),
|
|
3399
|
+
wait: (method, id, timeout = 1e4) => new Promise((resolve3, reject) => {
|
|
3400
|
+
const finish = () => {
|
|
3401
|
+
clearTimeout(timer);
|
|
3402
|
+
remove();
|
|
3403
|
+
runtime.call(`${method}.get`, { id }).then((value) => resolve3(value.result), reject);
|
|
3404
|
+
};
|
|
3405
|
+
const remove = service.observe((summary) => {
|
|
3406
|
+
if (summary.id === id && (terminalStatuses.has(summary.status) || summary.status === "blocked"))
|
|
3407
|
+
finish();
|
|
3408
|
+
});
|
|
3409
|
+
const timer = setTimeout(() => {
|
|
3410
|
+
remove();
|
|
3411
|
+
reject(new Error(`Workflow ${id} did not finish within ${timeout}ms.`));
|
|
3412
|
+
}, timeout);
|
|
3413
|
+
runtime.call(`${method}.get`, { id }).then((value) => {
|
|
3414
|
+
const summary = value.result;
|
|
3415
|
+
if (terminalStatuses.has(summary.status) || summary.status === "blocked")
|
|
3416
|
+
finish();
|
|
3417
|
+
}, (error) => {
|
|
3418
|
+
clearTimeout(timer);
|
|
3419
|
+
remove();
|
|
3420
|
+
reject(error);
|
|
3421
|
+
});
|
|
3422
|
+
}),
|
|
3423
|
+
close: async () => {
|
|
3424
|
+
await service.close();
|
|
3425
|
+
runtime.close();
|
|
3426
|
+
await rm4(dataDir, { recursive: true, force: true });
|
|
3427
|
+
}
|
|
3428
|
+
};
|
|
3429
|
+
} catch (error) {
|
|
3430
|
+
await service.close();
|
|
3431
|
+
runtime?.close();
|
|
3432
|
+
await rm4(dataDir, { recursive: true, force: true });
|
|
3433
|
+
throw error;
|
|
3434
|
+
}
|
|
3435
|
+
}
|
|
3436
|
+
|
|
3437
|
+
// src/testing.ts
|
|
3438
|
+
async function testRuntime(plugins, options = {}) {
|
|
3439
|
+
const dataDir = await mkdtemp2(join8(tmpdir2(), "repogo-plugin-test-"));
|
|
3440
|
+
const project = join8(dataDir, "project");
|
|
3441
|
+
await mkdir6(project);
|
|
3442
|
+
const host = options.mcp || options.host ? testHost(options.mcp, options.host) : undefined;
|
|
3443
|
+
const runtime = await Runtime.open({ dataDir: join8(dataDir, "data"), root: options.root ?? project, host });
|
|
3444
|
+
const notices = [];
|
|
3445
|
+
runtime.onNotify((notice) => notices.push(notice));
|
|
3446
|
+
const logs = [];
|
|
3447
|
+
captureConsole((entry) => logs.push(entry), { passThrough: true });
|
|
3448
|
+
try {
|
|
3449
|
+
for (const [name, plugin] of Object.entries(plugins))
|
|
3450
|
+
await runtime.add(name, plugin);
|
|
3451
|
+
} catch (error) {
|
|
3452
|
+
releaseConsole();
|
|
3453
|
+
runtime.close();
|
|
3454
|
+
await rm5(dataDir, { recursive: true, force: true });
|
|
3455
|
+
throw error;
|
|
3456
|
+
}
|
|
3457
|
+
return {
|
|
3458
|
+
call: async (method, input = {}, call = {}) => (await runtime.call(method, input, call)).result,
|
|
3459
|
+
setCrons: (crons) => runtime.setCrons(crons),
|
|
3460
|
+
tickCrons: async (now) => {
|
|
3461
|
+
await Promise.all(runtime.tickCrons(now));
|
|
3462
|
+
},
|
|
3463
|
+
manifest: () => runtime.manifest(),
|
|
3464
|
+
notices,
|
|
3465
|
+
logs,
|
|
3466
|
+
close: async () => {
|
|
3467
|
+
releaseConsole();
|
|
3468
|
+
runtime.close();
|
|
3469
|
+
await rm5(dataDir, { recursive: true, force: true });
|
|
3470
|
+
}
|
|
3471
|
+
};
|
|
3472
|
+
}
|
|
3473
|
+
function testHost(mcp, host) {
|
|
3474
|
+
return async (_plugin, method, params) => {
|
|
3475
|
+
if (method !== "mcp.call") {
|
|
3476
|
+
if (!host)
|
|
3477
|
+
throw new MethodError("FORBIDDEN", `This test runtime doesn't serve ${method}.`);
|
|
3478
|
+
return host(method, params);
|
|
3479
|
+
}
|
|
3480
|
+
if (!mcp)
|
|
3481
|
+
throw new MethodError("FORBIDDEN", "This test runtime has no MCP servers.");
|
|
3482
|
+
try {
|
|
3483
|
+
return { result: await mcp(params) };
|
|
3484
|
+
} catch (error) {
|
|
3485
|
+
return { error: { code: "TOOL_ERROR", message: error instanceof Error ? error.message : String(error) } };
|
|
3486
|
+
}
|
|
3487
|
+
};
|
|
3488
|
+
}
|
|
3489
|
+
export {
|
|
3490
|
+
testWorkflowRuntime,
|
|
3491
|
+
testRuntime
|
|
3492
|
+
};
|