@repogo/plugins 0.1.35 → 0.1.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +243 -3
- package/dist/cache.d.ts +27 -0
- package/dist/context.d.ts +67 -2
- package/dist/define.d.ts +31 -0
- package/dist/index.d.ts +9 -3
- package/dist/index.js +123 -7
- package/dist/runtime.d.ts +16 -1
- package/dist/self.d.ts +1 -0
- package/dist/testing.d.ts +4 -0
- package/dist/testing.js +1562 -49
- package/dist/tracking.d.ts +5 -0
- package/dist/uploads.d.ts +24 -0
- package/dist/workflow/attempts.d.ts +25 -0
- package/dist/workflow/availability.d.ts +8 -0
- package/dist/workflow/catalog.d.ts +21 -0
- package/dist/workflow/clock.d.ts +6 -0
- package/dist/workflow/contracts.d.ts +121 -0
- package/dist/workflow/definition.d.ts +35 -0
- package/dist/workflow/errors.d.ts +19 -0
- package/dist/workflow/events.d.ts +132 -0
- package/dist/workflow/interrupted.d.ts +3 -0
- package/dist/workflow/journal.d.ts +28 -0
- package/dist/workflow/lease.d.ts +2 -0
- package/dist/workflow/lifetime.d.ts +27 -0
- package/dist/workflow/policy.d.ts +36 -0
- package/dist/workflow/register.d.ts +11 -0
- package/dist/workflow/retry.d.ts +14 -0
- package/dist/workflow/runner.d.ts +29 -0
- package/dist/workflow/scope.d.ts +4 -0
- package/dist/workflow/service.d.ts +48 -0
- package/dist/workflow/sleep.d.ts +6 -0
- package/dist/workflow/state.d.ts +41 -0
- package/dist/workflow/storage.d.ts +22 -0
- package/dist/workflow/summary.d.ts +80 -0
- package/dist/workflow/testing.d.ts +12 -0
- package/dist/workflow-executor.js +22917 -0
- package/package.json +3 -3
- package/dist/index-83w1fk10.js +0 -28
package/dist/testing.js
CHANGED
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
import {
|
|
3
3
|
MethodError,
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
attemptScope,
|
|
5
|
+
callFiles,
|
|
6
|
+
exports_src,
|
|
7
|
+
parseDuration,
|
|
8
|
+
toCallError,
|
|
9
|
+
workflowContext,
|
|
10
|
+
workflowScope
|
|
11
|
+
} from "./index.js";
|
|
6
12
|
|
|
7
13
|
// src/testing.ts
|
|
8
|
-
import { mkdir as
|
|
9
|
-
import { tmpdir } from "os";
|
|
10
|
-
import { join as
|
|
14
|
+
import { mkdir as mkdir6, mkdtemp as mkdtemp2, rm as rm5 } from "fs/promises";
|
|
15
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
16
|
+
import { join as join8 } from "path";
|
|
11
17
|
|
|
12
18
|
// src/logs.ts
|
|
13
19
|
import { AsyncLocalStorage } from "async_hooks";
|
|
@@ -87,7 +93,7 @@ function carryable(value) {
|
|
|
87
93
|
import { AsyncLocalStorage as AsyncLocalStorage3 } from "async_hooks";
|
|
88
94
|
import { mkdir as mkdir3, readdir as readdir3 } from "fs/promises";
|
|
89
95
|
import { join as join3 } from "path";
|
|
90
|
-
import { z as
|
|
96
|
+
import { z as z5 } from "zod";
|
|
91
97
|
|
|
92
98
|
// src/buckets.ts
|
|
93
99
|
import { watch } from "fs";
|
|
@@ -141,6 +147,9 @@ async function tracked(readOnly, announce, work) {
|
|
|
141
147
|
access.done.value = true;
|
|
142
148
|
}
|
|
143
149
|
}
|
|
150
|
+
function isolated(work) {
|
|
151
|
+
return current.run({ reads: new Set, writes: new Set, readOnly: true, done: { value: false }, announce: () => {} }, work);
|
|
152
|
+
}
|
|
144
153
|
function recordRead(tag) {
|
|
145
154
|
current.getStore()?.reads.add(tag);
|
|
146
155
|
}
|
|
@@ -501,6 +510,93 @@ function checkName(name) {
|
|
|
501
510
|
}
|
|
502
511
|
}
|
|
503
512
|
|
|
513
|
+
// src/cache.ts
|
|
514
|
+
var maxBytes = 1e6;
|
|
515
|
+
var keepMs = 30 * 86400000;
|
|
516
|
+
|
|
517
|
+
class AnswerCache {
|
|
518
|
+
database;
|
|
519
|
+
namespace;
|
|
520
|
+
announce;
|
|
521
|
+
inflight = new Map;
|
|
522
|
+
constructor(database, namespace, announce) {
|
|
523
|
+
this.database = database;
|
|
524
|
+
this.namespace = namespace;
|
|
525
|
+
this.announce = announce;
|
|
526
|
+
}
|
|
527
|
+
async setup() {
|
|
528
|
+
await this.database.sql.unsafe(`create table if not exists cached_answers (
|
|
529
|
+
key text primary key,
|
|
530
|
+
value text not null,
|
|
531
|
+
checked_at integer not null
|
|
532
|
+
)`);
|
|
533
|
+
}
|
|
534
|
+
async read(key, options, fetch) {
|
|
535
|
+
const tag = `${this.namespace}.cached.${Bun.hash(key).toString(36)}`;
|
|
536
|
+
recordRead(tag);
|
|
537
|
+
const [row] = await this.database.sql.unsafe(`select value, checked_at from cached_answers where key = $1`, [key]);
|
|
538
|
+
if (!row)
|
|
539
|
+
return this.refresh(key, tag, options, fetch);
|
|
540
|
+
if (Date.now() - row.checked_at >= options.fresh) {
|
|
541
|
+
this.refresh(key, tag, options, fetch).catch((error) => record("error", [error]));
|
|
542
|
+
}
|
|
543
|
+
return JSON.parse(row.value);
|
|
544
|
+
}
|
|
545
|
+
async idle() {
|
|
546
|
+
while (this.inflight.size)
|
|
547
|
+
await Promise.allSettled([...this.inflight.values()]);
|
|
548
|
+
}
|
|
549
|
+
refresh(key, tag, options, fetch) {
|
|
550
|
+
const running = this.inflight.get(key);
|
|
551
|
+
if (running)
|
|
552
|
+
return running;
|
|
553
|
+
const started = Date.now();
|
|
554
|
+
const work = isolated(async () => withTimeout(Promise.resolve(fetch()), options.timeout)).then(async (value) => {
|
|
555
|
+
await this.save(key, tag, value ?? null, started);
|
|
556
|
+
return value ?? null;
|
|
557
|
+
}, async (error) => {
|
|
558
|
+
await this.database.transaction(() => this.database.sql.unsafe(`update cached_answers set checked_at = $1 where key = $2`, [Date.now(), key]));
|
|
559
|
+
throw error;
|
|
560
|
+
}).finally(() => this.inflight.delete(key));
|
|
561
|
+
this.inflight.set(key, work);
|
|
562
|
+
return work;
|
|
563
|
+
}
|
|
564
|
+
async save(key, tag, value, started) {
|
|
565
|
+
const text = JSON.stringify(value);
|
|
566
|
+
if (text.length > maxBytes)
|
|
567
|
+
return;
|
|
568
|
+
const changed = await this.database.transaction(async () => {
|
|
569
|
+
const [before] = await this.database.sql.unsafe(`select value, checked_at from cached_answers where key = $1`, [key]);
|
|
570
|
+
if (before && before.checked_at > started)
|
|
571
|
+
return false;
|
|
572
|
+
const now = Date.now();
|
|
573
|
+
await this.database.sql.unsafe(`insert into cached_answers (key, value, checked_at) values ($1, $2, $3)
|
|
574
|
+
on conflict (key) do update set value = excluded.value, checked_at = excluded.checked_at`, [key, text, now]);
|
|
575
|
+
await this.database.sql.unsafe(`delete from cached_answers where checked_at < $1`, [now - keepMs]);
|
|
576
|
+
return before?.value !== text;
|
|
577
|
+
});
|
|
578
|
+
if (changed)
|
|
579
|
+
this.announce([tag]);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
function cacheKey(method, root, input) {
|
|
583
|
+
return JSON.stringify([method, root, sorted(input)]);
|
|
584
|
+
}
|
|
585
|
+
function sorted(value) {
|
|
586
|
+
if (Array.isArray(value))
|
|
587
|
+
return value.map(sorted);
|
|
588
|
+
if (value === null || typeof value !== "object")
|
|
589
|
+
return value;
|
|
590
|
+
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sorted(value[key])]));
|
|
591
|
+
}
|
|
592
|
+
function withTimeout(work, ms) {
|
|
593
|
+
let timer;
|
|
594
|
+
const timeout = new Promise((_, reject) => {
|
|
595
|
+
timer = setTimeout(() => reject(new MethodError("INTERNAL", `The answer took longer than ${Math.round(ms / 1000)} s.`)), ms);
|
|
596
|
+
});
|
|
597
|
+
return Promise.race([work, timeout]).finally(() => clearTimeout(timer));
|
|
598
|
+
}
|
|
599
|
+
|
|
504
600
|
// src/collections.ts
|
|
505
601
|
class Collection {
|
|
506
602
|
database;
|
|
@@ -799,8 +895,8 @@ class Database {
|
|
|
799
895
|
}
|
|
800
896
|
static async open(path) {
|
|
801
897
|
const sql = new SQL(`sqlite://${path}`);
|
|
802
|
-
await sql.unsafe("pragma journal_mode = wal");
|
|
803
898
|
await sql.unsafe("pragma busy_timeout = 5000");
|
|
899
|
+
await sql.unsafe("pragma journal_mode = wal");
|
|
804
900
|
return new Database(sql);
|
|
805
901
|
}
|
|
806
902
|
transaction(work) {
|
|
@@ -824,23 +920,6 @@ class Database {
|
|
|
824
920
|
}
|
|
825
921
|
}
|
|
826
922
|
|
|
827
|
-
// src/duration.ts
|
|
828
|
-
var unitMs = { s: 1000, m: 60000, h: 3600000, d: 86400000 };
|
|
829
|
-
function parseDuration(duration) {
|
|
830
|
-
if (typeof duration === "number") {
|
|
831
|
-
if (!Number.isFinite(duration) || duration <= 0)
|
|
832
|
-
throw new Error(`a duration is a positive number of ms: ${duration}`);
|
|
833
|
-
return duration;
|
|
834
|
-
}
|
|
835
|
-
const match = /^(\d+(?:\.\d+)?)([smhd])$/.exec(duration);
|
|
836
|
-
if (!match)
|
|
837
|
-
throw new Error(`a duration is like "30s", "5m", "1h" or "7d": ${duration}`);
|
|
838
|
-
const ms = Number(match[1]) * unitMs[match[2]];
|
|
839
|
-
if (ms <= 0)
|
|
840
|
-
throw new Error(`a duration is longer than zero: ${duration}`);
|
|
841
|
-
return ms;
|
|
842
|
-
}
|
|
843
|
-
|
|
844
923
|
// src/events.ts
|
|
845
924
|
class EventBus {
|
|
846
925
|
listeners = new Map;
|
|
@@ -1376,6 +1455,108 @@ function toKey(row) {
|
|
|
1376
1455
|
};
|
|
1377
1456
|
}
|
|
1378
1457
|
|
|
1458
|
+
// src/workflow/register.ts
|
|
1459
|
+
import { z as z4 } from "zod";
|
|
1460
|
+
|
|
1461
|
+
// src/workflow/contracts.ts
|
|
1462
|
+
import { z as z3 } from "zod";
|
|
1463
|
+
import { isAbsolute as isAbsolute2 } from "path";
|
|
1464
|
+
|
|
1465
|
+
// src/workflow/policy.ts
|
|
1466
|
+
import { z as z2 } from "zod";
|
|
1467
|
+
var duration = z2.custom((value) => typeof value === "number" || typeof value === "string");
|
|
1468
|
+
var StepOptionsSchema = z2.object({
|
|
1469
|
+
kind: z2.enum(["query", "mutation"]).optional(),
|
|
1470
|
+
attempts: z2.number().optional(),
|
|
1471
|
+
timeout: duration.optional(),
|
|
1472
|
+
backoff: z2.object({ initial: duration.optional(), factor: z2.number().optional(), max: duration.optional() }).strict().optional()
|
|
1473
|
+
}).strict();
|
|
1474
|
+
var maxDurationMs = 365 * 86400000;
|
|
1475
|
+
var DurationMsSchema = z2.number().int().positive().max(maxDurationMs);
|
|
1476
|
+
var StepPolicySchema = z2.object({
|
|
1477
|
+
kind: z2.enum(["query", "mutation"]),
|
|
1478
|
+
attempts: z2.number().int().min(1).max(100),
|
|
1479
|
+
timeoutMs: DurationMsSchema,
|
|
1480
|
+
backoff: z2.object({ initialMs: DurationMsSchema, factor: z2.number().min(1).max(1000), maxMs: DurationMsSchema }).strict()
|
|
1481
|
+
}).strict().refine((value) => value.kind === "query" || value.attempts === 1, "Only query steps may retry.").refine((value) => value.backoff.initialMs <= value.backoff.maxMs, "Initial backoff must not exceed its maximum.");
|
|
1482
|
+
function durationMs(value) {
|
|
1483
|
+
return DurationMsSchema.parse(Math.ceil(parseDuration(value)));
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
// src/workflow/contracts.ts
|
|
1487
|
+
var RunSummarySchema = z3.object({
|
|
1488
|
+
id: z3.string(),
|
|
1489
|
+
workflow: z3.string(),
|
|
1490
|
+
status: z3.enum(["pending", "running", "waiting", "attention", "completed", "failed", "cancelled", "expired", "unknown", "blocked"]),
|
|
1491
|
+
currentStep: z3.string().nullable(),
|
|
1492
|
+
attempt: z3.number().int(),
|
|
1493
|
+
createdAt: z3.string(),
|
|
1494
|
+
updatedAt: z3.string(),
|
|
1495
|
+
nextAttemptAt: z3.string().nullable(),
|
|
1496
|
+
deadlineAt: z3.string().nullable()
|
|
1497
|
+
});
|
|
1498
|
+
var RunMetadataSchema = z3.object({
|
|
1499
|
+
id: z3.string().uuid(),
|
|
1500
|
+
dir: z3.string().refine(isAbsolute2),
|
|
1501
|
+
root: z3.string().refine(isAbsolute2),
|
|
1502
|
+
namespace: z3.string().regex(/^[a-z0-9][a-z0-9.-]*$/),
|
|
1503
|
+
plugin: z3.string().regex(/^[a-z0-9][a-z0-9-]*$/),
|
|
1504
|
+
caller: z3.enum(["card", "agent", "cron", "code", "http"]),
|
|
1505
|
+
workflow: z3.string().regex(/^[a-z][a-zA-Z0-9_-]*$/),
|
|
1506
|
+
input: z3.json(),
|
|
1507
|
+
key: z3.string().nullable(),
|
|
1508
|
+
timeoutMs: DurationMsSchema,
|
|
1509
|
+
maxOfflineMs: DurationMsSchema,
|
|
1510
|
+
retentionMs: DurationMsSchema,
|
|
1511
|
+
createdAt: z3.iso.datetime()
|
|
1512
|
+
}).strict();
|
|
1513
|
+
var RunLocationSchema = RunMetadataSchema.omit({ input: true, timeoutMs: true, workflow: true, createdAt: true });
|
|
1514
|
+
|
|
1515
|
+
// src/workflow/register.ts
|
|
1516
|
+
function workflowTag(namespace) {
|
|
1517
|
+
return `${namespace}.workflows`;
|
|
1518
|
+
}
|
|
1519
|
+
function workflowMethods(plugin, namespace, name, definition, ctx, bridge, directory) {
|
|
1520
|
+
const invoke = (operation, input) => {
|
|
1521
|
+
const dir = directory();
|
|
1522
|
+
if (!bridge || !dir)
|
|
1523
|
+
throw new MethodError("INTERNAL", "Durable workflows run through the plugins service with source files.");
|
|
1524
|
+
return bridge({ operation, dir, plugin, namespace, workflow: name, root: ctx.root, caller: ctx.caller, input, settings: workflowSettings(definition) });
|
|
1525
|
+
};
|
|
1526
|
+
const id = z4.object({ id: z4.string().uuid() }).strict();
|
|
1527
|
+
const spec = (method, kind, input, description, confirm, output) => ({
|
|
1528
|
+
name: `${plugin}.${name}.${method}`,
|
|
1529
|
+
plugin,
|
|
1530
|
+
kind,
|
|
1531
|
+
input,
|
|
1532
|
+
description,
|
|
1533
|
+
confirm,
|
|
1534
|
+
output,
|
|
1535
|
+
run: async (input2, raw) => {
|
|
1536
|
+
if (kind === "query")
|
|
1537
|
+
recordRead(workflowTag(namespace));
|
|
1538
|
+
const request = method === "start" ? { ...input2, input: raw.input } : input2;
|
|
1539
|
+
return invoke(method, request);
|
|
1540
|
+
}
|
|
1541
|
+
});
|
|
1542
|
+
return [
|
|
1543
|
+
spec("list", "query", z4.object({ limit: z4.number().int().min(1).max(100).default(20), status: z4.string().optional(), cursor: z4.string().optional() }).strict(), "List workflow runs", undefined, z4.object({ items: z4.array(RunSummarySchema), cursor: z4.string().nullable() })),
|
|
1544
|
+
spec("get", "query", id, "Get one workflow run"),
|
|
1545
|
+
spec("start", "mutation", z4.object({ input: definition.input, key: z4.string().min(1).max(200).optional() }).strict(), definition.description ?? "Start a durable workflow", definition.confirm, RunSummarySchema),
|
|
1546
|
+
spec("retry", "mutation", id, "Retry a safely failed query step", undefined, RunSummarySchema),
|
|
1547
|
+
spec("restart", "mutation", id, "Start a run that needs attention over from the beginning, on the current code", undefined, RunSummarySchema),
|
|
1548
|
+
spec("cancel", "mutation", id, "Request cancellation of a workflow", undefined, RunSummarySchema),
|
|
1549
|
+
spec("cancelAll", "mutation", z4.object({ status: z4.string().optional() }).strict(), "Cancel every unfinished run, or only those in one status", undefined, z4.object({ cancelled: z4.number().int() }))
|
|
1550
|
+
];
|
|
1551
|
+
}
|
|
1552
|
+
function workflowSettings(definition) {
|
|
1553
|
+
return {
|
|
1554
|
+
timeoutMs: durationMs(definition.timeout ?? "30d"),
|
|
1555
|
+
maxOfflineMs: durationMs(definition.maxOffline ?? "14d"),
|
|
1556
|
+
retentionMs: durationMs(definition.retention ?? "30d")
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1379
1560
|
// src/runtime.ts
|
|
1380
1561
|
var sampleMs = Number(process.env.REPOGO_PLUGINS_SAMPLE_MS ?? 5000);
|
|
1381
1562
|
var sampleLimit = 200;
|
|
@@ -1390,19 +1571,23 @@ class Runtime {
|
|
|
1390
1571
|
dataDir;
|
|
1391
1572
|
root;
|
|
1392
1573
|
host;
|
|
1574
|
+
workflows;
|
|
1393
1575
|
events = new EventBus;
|
|
1394
1576
|
methods = new Map;
|
|
1395
1577
|
plugins = new Map;
|
|
1396
1578
|
crons = [];
|
|
1579
|
+
contexts = new Map;
|
|
1397
1580
|
databases = new Map;
|
|
1581
|
+
answerCaches = new Map;
|
|
1398
1582
|
stores = new Map;
|
|
1399
1583
|
invalidationListeners = new Set;
|
|
1400
1584
|
noticeListeners = new Set;
|
|
1401
1585
|
pluginsDir = null;
|
|
1402
|
-
constructor(dataDir, root, host) {
|
|
1586
|
+
constructor(dataDir, root, host, workflows) {
|
|
1403
1587
|
this.dataDir = dataDir;
|
|
1404
1588
|
this.root = root;
|
|
1405
1589
|
this.host = host;
|
|
1590
|
+
this.workflows = workflows;
|
|
1406
1591
|
}
|
|
1407
1592
|
callRoot = new AsyncLocalStorage3;
|
|
1408
1593
|
callSignal = new AsyncLocalStorage3;
|
|
@@ -1411,7 +1596,7 @@ class Runtime {
|
|
|
1411
1596
|
cronsTag = `crons.${crypto.randomUUID()}`;
|
|
1412
1597
|
static async open(options) {
|
|
1413
1598
|
await mkdir3(options.dataDir, { recursive: true });
|
|
1414
|
-
return new Runtime(options.dataDir, options.root ?? process.cwd(), options.host);
|
|
1599
|
+
return new Runtime(options.dataDir, options.root ?? process.cwd(), options.host, options.workflows);
|
|
1415
1600
|
}
|
|
1416
1601
|
async loadDir(pluginsDir, only = null) {
|
|
1417
1602
|
this.pluginsDir = pluginsDir;
|
|
@@ -1439,7 +1624,8 @@ class Runtime {
|
|
|
1439
1624
|
if (!stores.includes(store))
|
|
1440
1625
|
throw new Error(`plugin ${name} lists ${store} as private, but keeps no collection, kv, stream or bucket by that name`);
|
|
1441
1626
|
}
|
|
1442
|
-
const
|
|
1627
|
+
const hasCache = Object.values(plugin.methods ?? {}).some((method) => method.cache);
|
|
1628
|
+
const hasData = hasCache || Object.keys(plugin.workflows ?? {}).length > 0 || declared.length > 0 || [plugin.buckets, plugin.kv, plugin.streams].some((stores2) => Object.keys(stores2 ?? {}).length > 0);
|
|
1443
1629
|
const namespace = plugin.namespace ?? "";
|
|
1444
1630
|
if (hasData && !namespace)
|
|
1445
1631
|
throw new Error(`plugin ${name} keeps data, so it needs a namespace: "<project>.${name}" for one project's, "${name}" to share it`);
|
|
@@ -1489,7 +1675,7 @@ class Runtime {
|
|
|
1489
1675
|
return runtime.callRoot.getStore() ?? runtime.root;
|
|
1490
1676
|
},
|
|
1491
1677
|
get signal() {
|
|
1492
|
-
return runtime.callSignal.getStore() ?? never;
|
|
1678
|
+
return attemptScope.getStore()?.signal ?? runtime.callSignal.getStore() ?? never;
|
|
1493
1679
|
},
|
|
1494
1680
|
collections,
|
|
1495
1681
|
buckets,
|
|
@@ -1498,6 +1684,8 @@ class Runtime {
|
|
|
1498
1684
|
call: async (method, input) => (await this.call(method.includes(".") ? method : `${name}.${method}`, input)).result,
|
|
1499
1685
|
emit: (event, payload) => this.events.emit(`${name}.${event}`, payload),
|
|
1500
1686
|
host: async (method, params) => {
|
|
1687
|
+
if (workflowScope.getStore())
|
|
1688
|
+
assertWritable("host methods");
|
|
1501
1689
|
if (!runtime.host)
|
|
1502
1690
|
throw new MethodError("INTERNAL", "ctx.host runs only under the RepoGo host.");
|
|
1503
1691
|
return runtime.host(name, method, params ?? {});
|
|
@@ -1522,31 +1710,67 @@ class Runtime {
|
|
|
1522
1710
|
return this.manifest().plugins;
|
|
1523
1711
|
},
|
|
1524
1712
|
notify: (notice) => {
|
|
1713
|
+
assertWritable("notifications");
|
|
1525
1714
|
const sent = { ...notice, plugin: name, root: runtime.callRoot.getStore() ?? runtime.root };
|
|
1526
1715
|
for (const listener of runtime.noticeListeners)
|
|
1527
1716
|
listener(sent);
|
|
1528
1717
|
},
|
|
1718
|
+
activity: async (activity) => {
|
|
1719
|
+
assertWritable("activities");
|
|
1720
|
+
if (!runtime.host)
|
|
1721
|
+
throw new MethodError("INTERNAL", "ctx.activity runs only under the RepoGo host.");
|
|
1722
|
+
await runtime.host(name, "activities.set", activityParams(activity, ctx.root));
|
|
1723
|
+
},
|
|
1529
1724
|
agent: (prompt, options2) => this.agent(name, ctx, prompt, options2 ?? {}),
|
|
1530
1725
|
log: (...args) => console.log(...args)
|
|
1531
1726
|
};
|
|
1727
|
+
const cache = hasCache ? await this.answerCache(namespace, file) : null;
|
|
1532
1728
|
for (const [methodName, method] of Object.entries(plugin.methods ?? {})) {
|
|
1729
|
+
const fullName = `${name}.${methodName}`;
|
|
1730
|
+
const options2 = method.cache;
|
|
1533
1731
|
this.register({
|
|
1534
|
-
name:
|
|
1732
|
+
name: fullName,
|
|
1535
1733
|
plugin: name,
|
|
1536
1734
|
kind: method.kind,
|
|
1537
1735
|
description: method.description,
|
|
1538
1736
|
confirm: method.confirm,
|
|
1737
|
+
http: method.http,
|
|
1539
1738
|
input: method.input,
|
|
1540
1739
|
output: method.output,
|
|
1541
|
-
run: async (input) => method.run(input, ctx)
|
|
1740
|
+
run: options2 && cache ? async (input) => cache.read(cacheKey(fullName, options2.shared ? null : ctx.root, input), options2, () => method.run(input, ctx)) : async (input) => method.run(input, ctx)
|
|
1542
1741
|
});
|
|
1543
1742
|
}
|
|
1743
|
+
this.contexts.set(name, ctx);
|
|
1744
|
+
for (const [workflowName, definition] of Object.entries(plugin.workflows ?? {})) {
|
|
1745
|
+
if (!/^[a-z][a-zA-Z0-9_-]*$/.test(workflowName))
|
|
1746
|
+
throw new Error(`Invalid workflow name ${workflowName}.`);
|
|
1747
|
+
workflowSettings(definition);
|
|
1748
|
+
for (const method of workflowMethods(name, namespace, workflowName, definition, ctx, this.workflows, () => this.pluginsDir))
|
|
1749
|
+
this.register(method);
|
|
1750
|
+
}
|
|
1544
1751
|
for (const [event, handler] of Object.entries(plugin.on ?? {})) {
|
|
1545
1752
|
this.events.on(`${name}.${event}`, (e) => logContext.run(this.logContextFor(name), () => handler(e.payload, ctx)));
|
|
1546
1753
|
}
|
|
1547
1754
|
this.plugins.set(name, plugin);
|
|
1548
1755
|
}
|
|
1756
|
+
async executeWorkflow(plugin, name, input, context) {
|
|
1757
|
+
const definition = this.plugins.get(plugin)?.workflows?.[name];
|
|
1758
|
+
if (!definition)
|
|
1759
|
+
throw new MethodError("NOT_FOUND", `No workflow ${plugin}.${name}.`);
|
|
1760
|
+
const ctx = this.contexts.get(plugin);
|
|
1761
|
+
return workflowScope.run(true, () => this.callCaller.run(context.caller, () => this.callRoot.run(context.root, () => this.callSignal.run(context.signal, () => runWith({ plugin, method: `${plugin}.${name}`, caller: context.caller, callId: context.runId }, async () => {
|
|
1762
|
+
const result = await definition.run(definition.input.parse(input), workflowContext(ctx, context));
|
|
1763
|
+
return definition.output ? definition.output.parse(result) : result;
|
|
1764
|
+
})))));
|
|
1765
|
+
}
|
|
1549
1766
|
async call(name, input = {}, options = {}) {
|
|
1767
|
+
if (options.files) {
|
|
1768
|
+
const { files, ...rest } = options;
|
|
1769
|
+
const fields = input ?? {};
|
|
1770
|
+
if (Object.hasOwn(fields, "files"))
|
|
1771
|
+
throw new MethodError("VALIDATION", "files comes from the upload; leave it out of the input.");
|
|
1772
|
+
return callFiles.run(files, () => this.call(name, { ...fields, files }, rest));
|
|
1773
|
+
}
|
|
1550
1774
|
if (options.root && options.root !== this.callRoot.getStore()) {
|
|
1551
1775
|
const { root, ...rest } = options;
|
|
1552
1776
|
return this.callRoot.run(root, () => this.call(name, input, rest));
|
|
@@ -1568,7 +1792,7 @@ class Runtime {
|
|
|
1568
1792
|
}
|
|
1569
1793
|
const raw = input ?? {};
|
|
1570
1794
|
const parsed = method.input.parse(raw);
|
|
1571
|
-
if (method.confirm && options.confirmed === false)
|
|
1795
|
+
if (method.confirm && (options.confirmed === false || workflowScope.getStore()))
|
|
1572
1796
|
throw new MethodError("CONFIRM", fillConfirm(method.confirm, parsed));
|
|
1573
1797
|
if (method.kind === "stream")
|
|
1574
1798
|
return { result: await this.sample(name, input), reads: [], invalidates: [] };
|
|
@@ -1750,6 +1974,7 @@ class Runtime {
|
|
|
1750
1974
|
};
|
|
1751
1975
|
}
|
|
1752
1976
|
async agent(plugin, ctx, prompt, options) {
|
|
1977
|
+
assertWritable("agents");
|
|
1753
1978
|
if (!this.host)
|
|
1754
1979
|
throw new MethodError("INTERNAL", "ctx.agent runs only under the RepoGo host.");
|
|
1755
1980
|
if (!options.agent)
|
|
@@ -1785,8 +2010,9 @@ class Runtime {
|
|
|
1785
2010
|
kind: method.kind,
|
|
1786
2011
|
description: method.description ?? null,
|
|
1787
2012
|
confirm: method.confirm ?? null,
|
|
1788
|
-
|
|
1789
|
-
|
|
2013
|
+
http: method.http ?? null,
|
|
2014
|
+
input: z5.toJSONSchema(method.input, { io: "input", unrepresentable: "any" }),
|
|
2015
|
+
output: method.output ? z5.toJSONSchema(method.output, { io: "output", unrepresentable: "any" }) : null
|
|
1790
2016
|
})),
|
|
1791
2017
|
crons: this.crons.filter((job) => job.plugin === name).map(cronState),
|
|
1792
2018
|
storage: {
|
|
@@ -1823,8 +2049,17 @@ class Runtime {
|
|
|
1823
2049
|
this.databases.set(namespace, database);
|
|
1824
2050
|
return database;
|
|
1825
2051
|
}
|
|
2052
|
+
async answerCache(namespace, database) {
|
|
2053
|
+
let cache = this.answerCaches.get(namespace);
|
|
2054
|
+
if (cache)
|
|
2055
|
+
return cache;
|
|
2056
|
+
cache = new AnswerCache(database, namespace, (tags) => this.notifyInvalidated(tags));
|
|
2057
|
+
await cache.setup();
|
|
2058
|
+
this.answerCaches.set(namespace, cache);
|
|
2059
|
+
return cache;
|
|
2060
|
+
}
|
|
1826
2061
|
async store(plugin, kind, name, schema, make, extra) {
|
|
1827
|
-
const signature = JSON.stringify([
|
|
2062
|
+
const signature = JSON.stringify([z5.toJSONSchema(schema, { unrepresentable: "any" }), extra ?? null]);
|
|
1828
2063
|
const made = make();
|
|
1829
2064
|
const found = this.stores.get(made.tag);
|
|
1830
2065
|
if (found && found.schema !== signature) {
|
|
@@ -1862,8 +2097,8 @@ class Runtime {
|
|
|
1862
2097
|
}
|
|
1863
2098
|
registerCrud(plugin, name, collection, schema) {
|
|
1864
2099
|
const prefix = `${plugin}.${name}`;
|
|
1865
|
-
const id =
|
|
1866
|
-
const record2 = schema.extend({ id:
|
|
2100
|
+
const id = z5.object({ id: z5.string() });
|
|
2101
|
+
const record2 = schema.extend({ id: z5.string(), createdAt: z5.string(), updatedAt: z5.string() });
|
|
1867
2102
|
const crud = (method, kind, input, output, description, run, confirm) => this.register({
|
|
1868
2103
|
name: `${prefix}.${method}`,
|
|
1869
2104
|
plugin,
|
|
@@ -1874,16 +2109,16 @@ class Runtime {
|
|
|
1874
2109
|
output,
|
|
1875
2110
|
run: async (value, raw) => run(value, raw)
|
|
1876
2111
|
});
|
|
1877
|
-
crud("list", "query",
|
|
1878
|
-
limit:
|
|
1879
|
-
offset:
|
|
1880
|
-
orderBy:
|
|
1881
|
-
order:
|
|
1882
|
-
}),
|
|
2112
|
+
crud("list", "query", z5.object({
|
|
2113
|
+
limit: z5.number().int().positive().max(500).optional(),
|
|
2114
|
+
offset: z5.number().int().min(0).optional(),
|
|
2115
|
+
orderBy: z5.string().optional(),
|
|
2116
|
+
order: z5.enum(["asc", "desc"]).optional()
|
|
2117
|
+
}), z5.array(record2), `List ${name} records, newest first unless orderBy says`, (input) => collection.list(input));
|
|
1883
2118
|
crud("get", "query", id, record2.nullable(), `Get one ${name} record by id, or null`, (input) => collection.get(input.id));
|
|
1884
2119
|
crud("create", "mutation", schema, record2, `Create a ${name} record`, (input) => collection.create(input));
|
|
1885
2120
|
crud("update", "mutation", id.extend(schema.partial().shape), record2, `Update fields on a ${name} record`, ({ id: id2, ...fields }, raw) => collection.update(id2, Object.fromEntries(Object.entries(fields).filter(([key]) => Object.hasOwn(raw ?? {}, key)))));
|
|
1886
|
-
crud("delete", "mutation", id,
|
|
2121
|
+
crud("delete", "mutation", id, z5.null(), `Delete a ${name} record by id`, async (input) => {
|
|
1887
2122
|
await collection.delete(input.id);
|
|
1888
2123
|
return null;
|
|
1889
2124
|
}, "Delete this record?");
|
|
@@ -1918,17 +2153,1294 @@ function mcpFailure(server, tool, error) {
|
|
|
1918
2153
|
}
|
|
1919
2154
|
return new MethodError("INTERNAL", error.message);
|
|
1920
2155
|
}
|
|
2156
|
+
function activityParams(activity, project) {
|
|
2157
|
+
const { compactLabel, timer, ...rest } = activity;
|
|
2158
|
+
return {
|
|
2159
|
+
...rest,
|
|
2160
|
+
compact_label: compactLabel,
|
|
2161
|
+
timer: timer && {
|
|
2162
|
+
kind: timer.kind,
|
|
2163
|
+
started_at: timer.kind === "elapsed" ? timer.startedAt : undefined,
|
|
2164
|
+
ends_at: timer.kind === "countdown" ? timer.endsAt : undefined
|
|
2165
|
+
},
|
|
2166
|
+
project
|
|
2167
|
+
};
|
|
2168
|
+
}
|
|
1921
2169
|
function isAsyncIterable(value) {
|
|
1922
2170
|
return typeof value?.[Symbol.asyncIterator] === "function";
|
|
1923
2171
|
}
|
|
1924
2172
|
|
|
2173
|
+
// src/self.ts
|
|
2174
|
+
Bun.plugin({
|
|
2175
|
+
name: "@repogo/plugins",
|
|
2176
|
+
setup(build) {
|
|
2177
|
+
build.module("@repogo/plugins", () => ({ exports: exports_src, loader: "object" }));
|
|
2178
|
+
}
|
|
2179
|
+
});
|
|
2180
|
+
|
|
2181
|
+
// src/workflow/testing.ts
|
|
2182
|
+
import { mkdtemp, rm as rm4 } from "fs/promises";
|
|
2183
|
+
import { tmpdir } from "os";
|
|
2184
|
+
import { join as join7, resolve as resolve2 } from "path";
|
|
2185
|
+
|
|
2186
|
+
// src/workflow/service.ts
|
|
2187
|
+
import { isDeepStrictEqual as isDeepStrictEqual2 } from "util";
|
|
2188
|
+
import { readdir as readdir4, rm as rm3, stat as stat3 } from "fs/promises";
|
|
2189
|
+
import { join as join6 } from "path";
|
|
2190
|
+
import { fileURLToPath } from "url";
|
|
2191
|
+
import { z as z7 } from "zod";
|
|
2192
|
+
|
|
2193
|
+
// src/workflow/catalog.ts
|
|
2194
|
+
var {SQL: SQL3 } = globalThis.Bun;
|
|
2195
|
+
import { mkdir as mkdir4 } from "fs/promises";
|
|
2196
|
+
import { join as join4 } from "path";
|
|
2197
|
+
|
|
2198
|
+
// src/workflow/lease.ts
|
|
2199
|
+
var {SQL: SQL2 } = globalThis.Bun;
|
|
2200
|
+
|
|
2201
|
+
// src/workflow/events.ts
|
|
2202
|
+
import { z as z6 } from "zod";
|
|
2203
|
+
var ID = z6.string().min(1).max(200);
|
|
2204
|
+
var sleepId = (position) => `@sleep:${position}`;
|
|
2205
|
+
var SleepID = z6.string().regex(/^@sleep:[1-9]\d*$/);
|
|
2206
|
+
var StepID = ID.refine((value) => !value.startsWith("@sleep:"), "Sleep IDs are reserved for the workflow core.");
|
|
2207
|
+
var Attempt = z6.number().int().positive();
|
|
2208
|
+
var Failure = z6.object({ message: z6.string(), code: z6.string().optional() }).strict();
|
|
2209
|
+
var Step = { stepId: StepID, attempt: Attempt };
|
|
2210
|
+
var EventInputSchema = z6.discriminatedUnion("type", [
|
|
2211
|
+
z6.object({ type: z6.literal("run.started"), workflow: ID, input: z6.json(), timeoutMs: DurationMsSchema, location: RunLocationSchema.optional() }).strict(),
|
|
2212
|
+
z6.object({ type: z6.literal("sleep.scheduled"), sleepId: SleepID, durationMs: DurationMsSchema }).strict(),
|
|
2213
|
+
z6.object({ type: z6.literal("sleep.completed"), sleepId: SleepID }).strict(),
|
|
2214
|
+
z6.object({ type: z6.literal("step.started"), ...Step, policy: StepPolicySchema }).strict(),
|
|
2215
|
+
z6.object({ type: z6.literal("step.completed"), ...Step, result: z6.json() }).strict(),
|
|
2216
|
+
z6.object({ type: z6.literal("step.failed"), ...Step, error: Failure }).strict(),
|
|
2217
|
+
z6.object({ type: z6.literal("step.unknown"), ...Step, error: Failure }).strict(),
|
|
2218
|
+
z6.object({ type: z6.literal("step.retryScheduled"), ...Step, error: Failure, delayMs: z6.number().int().min(0).max(maxDurationMs) }).strict(),
|
|
2219
|
+
z6.object({ type: z6.literal("run.retryRequested"), stepId: StepID }).strict(),
|
|
2220
|
+
z6.object({ type: z6.literal("run.completed"), result: z6.json() }).strict(),
|
|
2221
|
+
z6.object({ type: z6.literal("run.failed"), error: Failure }).strict(),
|
|
2222
|
+
z6.object({ type: z6.literal("run.cancelRequested") }).strict(),
|
|
2223
|
+
z6.object({ type: z6.literal("run.cancelled") }).strict(),
|
|
2224
|
+
z6.object({ type: z6.literal("run.expired"), reason: z6.string().min(1) }).strict(),
|
|
2225
|
+
z6.object({ type: z6.literal("run.unknown"), error: Failure }).strict(),
|
|
2226
|
+
z6.object({ type: z6.literal("run.attention"), error: Failure }).strict(),
|
|
2227
|
+
z6.object({ type: z6.literal("run.restarted") }).strict()
|
|
2228
|
+
]);
|
|
2229
|
+
var Envelope = z6.object({ version: z6.literal(1), seq: z6.number().int().positive(), at: z6.iso.datetime() }).strict();
|
|
2230
|
+
|
|
2231
|
+
class JournalError extends Error {
|
|
2232
|
+
code;
|
|
2233
|
+
constructor(code, message) {
|
|
2234
|
+
super(message);
|
|
2235
|
+
this.code = code;
|
|
2236
|
+
this.name = "JournalError";
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
function eventInput(value) {
|
|
2240
|
+
const parsed = EventInputSchema.safeParse(value);
|
|
2241
|
+
if (!parsed.success)
|
|
2242
|
+
throw new JournalError("INVALID_EVENT", parsed.error.message);
|
|
2243
|
+
return JSON.parse(JSON.stringify(parsed.data));
|
|
2244
|
+
}
|
|
2245
|
+
function storedEvent(value) {
|
|
2246
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
2247
|
+
throw new Error("Expected an event object.");
|
|
2248
|
+
const { version, seq, at, ...input } = value;
|
|
2249
|
+
return { ...eventInput(input), ...Envelope.parse({ version, seq, at }) };
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
// src/workflow/lease.ts
|
|
2253
|
+
async function acquireOwner(path) {
|
|
2254
|
+
const owner = new SQL2({ adapter: "sqlite", filename: path });
|
|
2255
|
+
try {
|
|
2256
|
+
await owner.unsafe("pragma busy_timeout = 0");
|
|
2257
|
+
await owner.unsafe("begin exclusive");
|
|
2258
|
+
} catch (error) {
|
|
2259
|
+
await owner.close();
|
|
2260
|
+
if (error instanceof SQL2.SQLiteError && [5, 6].includes(error.errno & 255))
|
|
2261
|
+
throw new JournalError("LOCKED", "Another executor owns this storage.");
|
|
2262
|
+
throw error;
|
|
2263
|
+
}
|
|
2264
|
+
let closed = false;
|
|
2265
|
+
return async () => {
|
|
2266
|
+
if (closed)
|
|
2267
|
+
return;
|
|
2268
|
+
closed = true;
|
|
2269
|
+
try {
|
|
2270
|
+
await owner.unsafe("rollback");
|
|
2271
|
+
} finally {
|
|
2272
|
+
await owner.close();
|
|
2273
|
+
}
|
|
2274
|
+
};
|
|
2275
|
+
}
|
|
2276
|
+
|
|
2277
|
+
// src/workflow/catalog.ts
|
|
2278
|
+
class WorkflowCatalog {
|
|
2279
|
+
sql;
|
|
2280
|
+
release;
|
|
2281
|
+
constructor(sql, release) {
|
|
2282
|
+
this.sql = sql;
|
|
2283
|
+
this.release = release;
|
|
2284
|
+
}
|
|
2285
|
+
static async open(directory) {
|
|
2286
|
+
await mkdir4(directory, { recursive: true, mode: 448 });
|
|
2287
|
+
const release = await acquireOwner(join4(directory, "owner.db"));
|
|
2288
|
+
let sql = null;
|
|
2289
|
+
try {
|
|
2290
|
+
sql = new SQL3({ adapter: "sqlite", filename: join4(directory, "catalog.db") });
|
|
2291
|
+
await sql.unsafe("pragma journal_mode = wal");
|
|
2292
|
+
await sql.unsafe("pragma synchronous = full");
|
|
2293
|
+
await sql.unsafe("create table if not exists runs (id text primary key, metadata text not null, summary text not null, seen real not null, terminal real, error text)");
|
|
2294
|
+
await sql.unsafe("create table if not exists availability (id integer primary key check(id = 1), seen real not null, offline_since real)");
|
|
2295
|
+
return new WorkflowCatalog(sql, release);
|
|
2296
|
+
} catch (error) {
|
|
2297
|
+
await sql?.close();
|
|
2298
|
+
await release();
|
|
2299
|
+
throw error;
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
2302
|
+
async all() {
|
|
2303
|
+
const rows = await this.sql.unsafe("select metadata, summary, seen, terminal, error from runs order by id");
|
|
2304
|
+
return rows.map((row) => ({ run: JSON.parse(row.metadata), summary: JSON.parse(row.summary), seen: row.seen, terminal: row.terminal, error: row.error }));
|
|
2305
|
+
}
|
|
2306
|
+
async put(value) {
|
|
2307
|
+
await this.sql.unsafe("insert into runs (id,metadata,summary,seen,terminal,error) values (?,?,?,?,?,?) on conflict(id) do update set metadata=excluded.metadata,summary=excluded.summary,seen=excluded.seen,terminal=excluded.terminal,error=excluded.error", [
|
|
2308
|
+
value.run.id,
|
|
2309
|
+
JSON.stringify(value.run),
|
|
2310
|
+
JSON.stringify(value.summary),
|
|
2311
|
+
value.seen,
|
|
2312
|
+
value.terminal,
|
|
2313
|
+
value.error
|
|
2314
|
+
]);
|
|
2315
|
+
}
|
|
2316
|
+
async availability() {
|
|
2317
|
+
const [row] = await this.sql.unsafe("select seen, offline_since from availability where id = 1");
|
|
2318
|
+
return row ? { seen: row.seen, offlineSince: row.offline_since } : null;
|
|
2319
|
+
}
|
|
2320
|
+
async heartbeat(value) {
|
|
2321
|
+
await this.sql.unsafe("insert into availability (id,seen,offline_since) values (1,?,?) on conflict(id) do update set seen=excluded.seen,offline_since=excluded.offline_since", [value.seen, value.offlineSince]);
|
|
2322
|
+
}
|
|
2323
|
+
async remove(id) {
|
|
2324
|
+
await this.sql.unsafe("delete from runs where id = ?", [id]);
|
|
2325
|
+
}
|
|
2326
|
+
async close() {
|
|
2327
|
+
try {
|
|
2328
|
+
await this.sql.close();
|
|
2329
|
+
} finally {
|
|
2330
|
+
await this.release();
|
|
2331
|
+
}
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
// src/workflow/availability.ts
|
|
2336
|
+
import { networkInterfaces } from "os";
|
|
2337
|
+
var networkAvailable = () => {
|
|
2338
|
+
try {
|
|
2339
|
+
return Object.values(networkInterfaces()).flat().some((address) => address && !address.internal && (address.family === "IPv4" ? !address.address.startsWith("169.254.") : !address.address.startsWith("fe80:")));
|
|
2340
|
+
} catch {
|
|
2341
|
+
return null;
|
|
2342
|
+
}
|
|
2343
|
+
};
|
|
2344
|
+
function nextAvailability(saved, now, online) {
|
|
2345
|
+
const offlineSince = online === true ? null : online === false ? saved?.offlineSince ?? now : saved?.offlineSince ?? null;
|
|
2346
|
+
return { seen: now, offlineSince };
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
// src/workflow/clock.ts
|
|
2350
|
+
var systemClock = {
|
|
2351
|
+
now: () => Date.now(),
|
|
2352
|
+
schedule(at, callback) {
|
|
2353
|
+
let timer;
|
|
2354
|
+
let cancelled = false;
|
|
2355
|
+
const tick = () => {
|
|
2356
|
+
if (cancelled)
|
|
2357
|
+
return;
|
|
2358
|
+
const remaining = at - Date.now();
|
|
2359
|
+
if (remaining <= 0)
|
|
2360
|
+
callback();
|
|
2361
|
+
else
|
|
2362
|
+
timer = setTimeout(tick, Math.min(remaining, 2147483647));
|
|
2363
|
+
};
|
|
2364
|
+
timer = setTimeout(tick, Math.min(Math.max(0, at - Date.now()), 2147483647));
|
|
2365
|
+
return () => {
|
|
2366
|
+
cancelled = true;
|
|
2367
|
+
if (timer !== undefined)
|
|
2368
|
+
clearTimeout(timer);
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
};
|
|
2372
|
+
function deadlineFrom(at, milliseconds) {
|
|
2373
|
+
return new Date(Date.parse(at) + milliseconds).toISOString();
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
// src/workflow/interrupted.ts
|
|
2377
|
+
async function settleInterrupted(journal, code) {
|
|
2378
|
+
for (const [stepId, step] of Object.entries(journal.state.steps)) {
|
|
2379
|
+
if (step.status !== "running")
|
|
2380
|
+
continue;
|
|
2381
|
+
await journal.append({
|
|
2382
|
+
type: step.policy.kind === "mutation" ? "step.unknown" : "step.failed",
|
|
2383
|
+
stepId,
|
|
2384
|
+
attempt: step.attempt,
|
|
2385
|
+
error: { code, message: "The workflow stopped before this attempt's outcome was recorded." }
|
|
2386
|
+
});
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
// src/workflow/state.ts
|
|
2391
|
+
import { isDeepStrictEqual } from "util";
|
|
2392
|
+
function initialState() {
|
|
2393
|
+
return {
|
|
2394
|
+
version: 2,
|
|
2395
|
+
lastSeq: 0,
|
|
2396
|
+
status: "pending",
|
|
2397
|
+
workflow: null,
|
|
2398
|
+
input: null,
|
|
2399
|
+
currentStep: null,
|
|
2400
|
+
operationOrder: [],
|
|
2401
|
+
steps: {},
|
|
2402
|
+
sleeps: {},
|
|
2403
|
+
result: null,
|
|
2404
|
+
error: null,
|
|
2405
|
+
expiryReason: null,
|
|
2406
|
+
deadlineAt: null,
|
|
2407
|
+
timeoutMs: null,
|
|
2408
|
+
cancelRequested: false,
|
|
2409
|
+
updatedAt: null,
|
|
2410
|
+
endedAt: null,
|
|
2411
|
+
metadata: null
|
|
2412
|
+
};
|
|
2413
|
+
}
|
|
2414
|
+
function applyEvent(state, event) {
|
|
2415
|
+
if (event.seq !== state.lastSeq + 1)
|
|
2416
|
+
refuse("Event sequence is not contiguous.");
|
|
2417
|
+
const next = { ...state, lastSeq: event.seq, updatedAt: event.at };
|
|
2418
|
+
if (event.type === "run.started") {
|
|
2419
|
+
if (state.status !== "pending")
|
|
2420
|
+
refuse("The run already started.");
|
|
2421
|
+
return { ...next, metadata: event.location ? { ...event.location, workflow: event.workflow, input: event.input, timeoutMs: event.timeoutMs, createdAt: event.at } : null, status: "running", workflow: event.workflow, input: event.input, timeoutMs: event.timeoutMs, deadlineAt: deadlineFrom(event.at, event.timeoutMs) };
|
|
2422
|
+
}
|
|
2423
|
+
if (event.type === "run.retryRequested") {
|
|
2424
|
+
const step2 = Object.hasOwn(state.steps, event.stepId) ? state.steps[event.stepId] : null;
|
|
2425
|
+
if (state.status !== "failed" || state.cancelRequested || !step2 || step2.status !== "failed" || step2.policy.kind !== "query" || !["STEP", "TIMEOUT", "INTERRUPTED"].includes(step2.error?.code ?? "") || Object.values(state.sleeps).some((sleep) => sleep.status !== "completed") || Object.entries(state.steps).some(([id, value]) => id !== event.stepId && value.status !== "completed"))
|
|
2426
|
+
refuse("Only a safely failed query step can be retried.");
|
|
2427
|
+
if (state.deadlineAt && Date.parse(event.at) >= Date.parse(state.deadlineAt))
|
|
2428
|
+
refuse("The workflow lifetime has ended.");
|
|
2429
|
+
const limit2 = step2.attempt + step2.policy.attempts;
|
|
2430
|
+
if (limit2 > 1e4)
|
|
2431
|
+
refuse("The run's attempt limit is exhausted.");
|
|
2432
|
+
return { ...next, endedAt: null, status: "waiting", error: null, currentStep: event.stepId, steps: {
|
|
2433
|
+
...state.steps,
|
|
2434
|
+
[event.stepId]: { ...step2, status: "waiting", limit: limit2, nextAttemptAt: event.at }
|
|
2435
|
+
} };
|
|
2436
|
+
}
|
|
2437
|
+
if (event.type === "run.restarted") {
|
|
2438
|
+
if (state.status !== "attention")
|
|
2439
|
+
refuse("Only a run that needs attention can restart.");
|
|
2440
|
+
const deadlineAt = state.timeoutMs === null ? state.deadlineAt : deadlineFrom(event.at, state.timeoutMs);
|
|
2441
|
+
return { ...next, status: "running", error: null, currentStep: null, operationOrder: [], steps: {}, sleeps: {}, deadlineAt };
|
|
2442
|
+
}
|
|
2443
|
+
if (state.status === "attention") {
|
|
2444
|
+
if (event.type === "run.cancelRequested")
|
|
2445
|
+
return { ...next, cancelRequested: true };
|
|
2446
|
+
if (event.type === "run.cancelled")
|
|
2447
|
+
return { ...next, endedAt: event.at, status: "cancelled" };
|
|
2448
|
+
if (event.type === "run.expired")
|
|
2449
|
+
return { ...next, endedAt: event.at, status: "expired", expiryReason: event.reason };
|
|
2450
|
+
refuse("The run needs attention: restart or cancel it.");
|
|
2451
|
+
}
|
|
2452
|
+
if (state.status !== "running" && state.status !== "waiting")
|
|
2453
|
+
refuse("The run is not active.");
|
|
2454
|
+
if (event.type === "run.attention") {
|
|
2455
|
+
if (Object.values(state.steps).some((step2) => step2.status === "running"))
|
|
2456
|
+
refuse("A step is running.");
|
|
2457
|
+
return { ...next, status: "attention", error: event.error };
|
|
2458
|
+
}
|
|
2459
|
+
if (event.type === "run.cancelRequested")
|
|
2460
|
+
return { ...next, cancelRequested: true };
|
|
2461
|
+
if (event.type === "run.completed") {
|
|
2462
|
+
if (state.cancelRequested)
|
|
2463
|
+
refuse("Cancellation prevents run completion.");
|
|
2464
|
+
if (Object.values(state.sleeps).some((sleep) => sleep.status !== "completed") || Object.values(state.steps).some((step2) => step2.status !== "completed"))
|
|
2465
|
+
refuse("An unfinished or failed step prevents run completion.");
|
|
2466
|
+
return { ...next, endedAt: event.at, status: "completed", result: event.result };
|
|
2467
|
+
}
|
|
2468
|
+
if (event.type === "run.failed" || event.type === "run.unknown") {
|
|
2469
|
+
return { ...next, endedAt: event.at, status: event.type === "run.failed" ? "failed" : "unknown", error: event.error };
|
|
2470
|
+
}
|
|
2471
|
+
if (event.type === "run.cancelled")
|
|
2472
|
+
return { ...next, endedAt: event.at, status: "cancelled" };
|
|
2473
|
+
if (event.type === "run.expired")
|
|
2474
|
+
return { ...next, endedAt: event.at, status: "expired", expiryReason: event.reason };
|
|
2475
|
+
if (event.type === "sleep.scheduled") {
|
|
2476
|
+
if (state.cancelRequested)
|
|
2477
|
+
refuse("Cancellation prevents starting sleeps.");
|
|
2478
|
+
if (event.sleepId !== sleepId(state.operationOrder.length + 1) || Object.hasOwn(state.sleeps, event.sleepId))
|
|
2479
|
+
refuse("Sleep identity must match its operation position.");
|
|
2480
|
+
if (Object.values(state.steps).some((step2) => step2.status !== "completed") || Object.values(state.sleeps).some((sleep2) => sleep2.status !== "completed"))
|
|
2481
|
+
refuse("Another operation is unfinished.");
|
|
2482
|
+
const sleep = { status: "waiting", durationMs: event.durationMs, wakeAt: deadlineFrom(event.at, event.durationMs) };
|
|
2483
|
+
return { ...next, status: "waiting", currentStep: event.sleepId, operationOrder: [...state.operationOrder, event.sleepId], sleeps: { ...state.sleeps, [event.sleepId]: sleep } };
|
|
2484
|
+
}
|
|
2485
|
+
if (event.type === "sleep.completed") {
|
|
2486
|
+
const sleep = Object.hasOwn(state.sleeps, event.sleepId) ? state.sleeps[event.sleepId] : null;
|
|
2487
|
+
if (!sleep || sleep.status !== "waiting")
|
|
2488
|
+
refuse("Only a waiting sleep can complete.");
|
|
2489
|
+
if (state.cancelRequested)
|
|
2490
|
+
refuse("Cancellation prevents completing sleeps.");
|
|
2491
|
+
if (Date.parse(event.at) < Date.parse(sleep.wakeAt))
|
|
2492
|
+
throw new JournalError("NOT_DUE", "The sleep is not due.");
|
|
2493
|
+
return { ...next, status: "running", currentStep: event.sleepId, sleeps: { ...state.sleeps, [event.sleepId]: { ...sleep, status: "completed" } } };
|
|
2494
|
+
}
|
|
2495
|
+
const previous = Object.hasOwn(state.steps, event.stepId) ? state.steps[event.stepId] : undefined;
|
|
2496
|
+
let step;
|
|
2497
|
+
if (event.type === "step.started") {
|
|
2498
|
+
if (state.cancelRequested)
|
|
2499
|
+
refuse("Cancellation prevents starting steps.");
|
|
2500
|
+
if (Object.values(state.sleeps).some((sleep) => sleep.status !== "completed"))
|
|
2501
|
+
refuse("A sleep is unfinished.");
|
|
2502
|
+
if (Object.values(state.steps).some((step2) => step2.status === "failed"))
|
|
2503
|
+
refuse("A failed step prevents starting steps.");
|
|
2504
|
+
if (previous) {
|
|
2505
|
+
if (!isDeepStrictEqual(previous.policy, event.policy))
|
|
2506
|
+
refuse("Retry policy differs from its recorded policy.");
|
|
2507
|
+
if (previous.status !== "waiting" || event.attempt !== previous.attempt + 1)
|
|
2508
|
+
refuse("Only a scheduled retry can start the next attempt.");
|
|
2509
|
+
if (Date.parse(event.at) < Date.parse(previous.nextAttemptAt))
|
|
2510
|
+
throw new JournalError("NOT_DUE", "The retry is not due.");
|
|
2511
|
+
} else if (event.attempt !== 1) {
|
|
2512
|
+
refuse("A new step starts at attempt one.");
|
|
2513
|
+
}
|
|
2514
|
+
if (event.attempt > (previous?.limit ?? event.policy.attempts))
|
|
2515
|
+
refuse("The attempt budget is exhausted.");
|
|
2516
|
+
if (Object.entries(state.steps).some(([id, value]) => id !== event.stepId && active(value)))
|
|
2517
|
+
refuse("Another step is unfinished.");
|
|
2518
|
+
step = { status: "running", attempt: event.attempt, limit: previous?.limit ?? event.policy.attempts, result: null, error: null, nextAttemptAt: null, policy: event.policy };
|
|
2519
|
+
} else {
|
|
2520
|
+
if (!previous || previous.status !== "running" || previous.attempt !== event.attempt)
|
|
2521
|
+
refuse("The event does not match a running attempt.");
|
|
2522
|
+
if (event.type === "step.completed") {
|
|
2523
|
+
step = { ...previous, status: "completed", result: event.result, error: null, nextAttemptAt: null };
|
|
2524
|
+
} else if (event.type === "step.failed" || event.type === "step.unknown") {
|
|
2525
|
+
step = { ...previous, status: event.type === "step.failed" ? "failed" : "unknown", error: event.error };
|
|
2526
|
+
} else {
|
|
2527
|
+
if (previous.policy.kind !== "query" || previous.attempt >= previous.limit)
|
|
2528
|
+
refuse("The step cannot retry.");
|
|
2529
|
+
if (state.cancelRequested)
|
|
2530
|
+
refuse("Cancellation prevents retries.");
|
|
2531
|
+
const nextAttemptAt = deadlineFrom(event.at, event.delayMs);
|
|
2532
|
+
step = { ...previous, status: "waiting", error: event.error, nextAttemptAt };
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
return {
|
|
2536
|
+
...next,
|
|
2537
|
+
status: step.status === "waiting" ? "waiting" : "running",
|
|
2538
|
+
currentStep: event.stepId,
|
|
2539
|
+
operationOrder: previous ? state.operationOrder : [...state.operationOrder, event.stepId],
|
|
2540
|
+
steps: { ...state.steps, [event.stepId]: step }
|
|
2541
|
+
};
|
|
2542
|
+
}
|
|
2543
|
+
function active(step) {
|
|
2544
|
+
return step.status === "running" || step.status === "waiting" || step.status === "unknown";
|
|
2545
|
+
}
|
|
2546
|
+
function refuse(message) {
|
|
2547
|
+
throw new JournalError("INVALID_EVENT", message);
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
// src/workflow/storage.ts
|
|
2551
|
+
import { constants } from "fs";
|
|
2552
|
+
import { mkdir as mkdir5, open, rename as rename2 } from "fs/promises";
|
|
2553
|
+
import { dirname, join as join5, resolve } from "path";
|
|
2554
|
+
var maxRecordBytes = 1024 * 1024;
|
|
2555
|
+
var maxJournalBytes = 64 * 1024 * 1024;
|
|
2556
|
+
|
|
2557
|
+
class FileJournalStorage {
|
|
2558
|
+
directory;
|
|
2559
|
+
events;
|
|
2560
|
+
release;
|
|
2561
|
+
constructor(directory, events, release) {
|
|
2562
|
+
this.directory = directory;
|
|
2563
|
+
this.events = events;
|
|
2564
|
+
this.release = release;
|
|
2565
|
+
}
|
|
2566
|
+
static async open(directory, create = true) {
|
|
2567
|
+
directory = resolve(directory);
|
|
2568
|
+
const created = create ? await mkdir5(directory, { recursive: true, mode: 448 }) : undefined;
|
|
2569
|
+
const ancestor = created ? dirname(resolve(created)) : dirname(directory);
|
|
2570
|
+
for (let parent = dirname(directory);; parent = dirname(parent)) {
|
|
2571
|
+
await syncDirectory(parent);
|
|
2572
|
+
if (parent === ancestor)
|
|
2573
|
+
break;
|
|
2574
|
+
}
|
|
2575
|
+
const release = await acquireOwner(join5(directory, "owner.db"));
|
|
2576
|
+
let events;
|
|
2577
|
+
try {
|
|
2578
|
+
events = await open(join5(directory, "events.jsonl"), constants.O_RDWR | constants.O_APPEND | (create ? constants.O_CREAT : 0), 384);
|
|
2579
|
+
await events.sync();
|
|
2580
|
+
await syncDirectory(directory);
|
|
2581
|
+
return new FileJournalStorage(directory, events, release);
|
|
2582
|
+
} catch (error) {
|
|
2583
|
+
try {
|
|
2584
|
+
await events?.close();
|
|
2585
|
+
} finally {
|
|
2586
|
+
await release();
|
|
2587
|
+
}
|
|
2588
|
+
throw error;
|
|
2589
|
+
}
|
|
2590
|
+
}
|
|
2591
|
+
async readEvents() {
|
|
2592
|
+
if ((await this.events.stat()).size > maxJournalBytes)
|
|
2593
|
+
throw new JournalError("LIMIT", "The run journal exceeds its size limit.");
|
|
2594
|
+
return this.events.readFile();
|
|
2595
|
+
}
|
|
2596
|
+
async append(record2) {
|
|
2597
|
+
await this.events.writeFile(record2);
|
|
2598
|
+
await this.events.sync();
|
|
2599
|
+
}
|
|
2600
|
+
async truncateEvents(bytes) {
|
|
2601
|
+
await this.events.truncate(bytes);
|
|
2602
|
+
await this.events.sync();
|
|
2603
|
+
}
|
|
2604
|
+
async replaceSnapshot(snapshot) {
|
|
2605
|
+
const temporary = join5(this.directory, "state.json.tmp");
|
|
2606
|
+
const file = await open(temporary, "w", 384);
|
|
2607
|
+
try {
|
|
2608
|
+
await file.writeFile(snapshot);
|
|
2609
|
+
await file.sync();
|
|
2610
|
+
} finally {
|
|
2611
|
+
await file.close();
|
|
2612
|
+
}
|
|
2613
|
+
await rename2(temporary, join5(this.directory, "state.json"));
|
|
2614
|
+
await syncDirectory(this.directory);
|
|
2615
|
+
}
|
|
2616
|
+
async close() {
|
|
2617
|
+
try {
|
|
2618
|
+
await this.events.close();
|
|
2619
|
+
} finally {
|
|
2620
|
+
await this.release();
|
|
2621
|
+
}
|
|
2622
|
+
}
|
|
2623
|
+
}
|
|
2624
|
+
async function syncDirectory(path) {
|
|
2625
|
+
const directory = await open(path, "r");
|
|
2626
|
+
try {
|
|
2627
|
+
await directory.sync();
|
|
2628
|
+
} finally {
|
|
2629
|
+
await directory.close();
|
|
2630
|
+
}
|
|
2631
|
+
}
|
|
2632
|
+
|
|
2633
|
+
// src/workflow/journal.ts
|
|
2634
|
+
class RunJournal {
|
|
2635
|
+
storage;
|
|
2636
|
+
clock;
|
|
2637
|
+
value = initialState();
|
|
2638
|
+
listeners = new Set;
|
|
2639
|
+
bytes = 0;
|
|
2640
|
+
queue = Promise.resolve();
|
|
2641
|
+
closing = null;
|
|
2642
|
+
poisoned = false;
|
|
2643
|
+
snapshotFailure = null;
|
|
2644
|
+
constructor(storage, clock) {
|
|
2645
|
+
this.storage = storage;
|
|
2646
|
+
this.clock = clock;
|
|
2647
|
+
}
|
|
2648
|
+
static async open(directory, clock = systemClock) {
|
|
2649
|
+
return RunJournal.recover(await FileJournalStorage.open(directory), clock);
|
|
2650
|
+
}
|
|
2651
|
+
static async openExisting(directory, clock = systemClock) {
|
|
2652
|
+
return RunJournal.recover(await FileJournalStorage.open(directory, false), clock);
|
|
2653
|
+
}
|
|
2654
|
+
static async recover(storage, clock) {
|
|
2655
|
+
const journal = new RunJournal(storage, clock);
|
|
2656
|
+
try {
|
|
2657
|
+
const bytes = Buffer.from(await storage.readEvents());
|
|
2658
|
+
if (bytes.length > maxJournalBytes)
|
|
2659
|
+
throw new JournalError("LIMIT", "The run journal exceeds its size limit.");
|
|
2660
|
+
const complete = bytes.lastIndexOf(10) + 1;
|
|
2661
|
+
let offset = 0;
|
|
2662
|
+
while (offset < complete) {
|
|
2663
|
+
const end = bytes.indexOf(10, offset);
|
|
2664
|
+
if (end - offset + 1 > maxRecordBytes)
|
|
2665
|
+
throw new JournalError("CORRUPT_JOURNAL", "A committed record exceeds its size limit.");
|
|
2666
|
+
try {
|
|
2667
|
+
const line = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(offset, end));
|
|
2668
|
+
journal.value = applyEvent(journal.value, storedEvent(JSON.parse(line)));
|
|
2669
|
+
} catch (error) {
|
|
2670
|
+
throw new JournalError("CORRUPT_JOURNAL", `Invalid committed event at byte ${offset}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2671
|
+
}
|
|
2672
|
+
offset = end + 1;
|
|
2673
|
+
}
|
|
2674
|
+
if (complete !== bytes.length)
|
|
2675
|
+
await storage.truncateEvents(complete);
|
|
2676
|
+
journal.bytes = complete;
|
|
2677
|
+
await journal.checkpoint();
|
|
2678
|
+
return journal;
|
|
2679
|
+
} catch (error) {
|
|
2680
|
+
await storage.close();
|
|
2681
|
+
throw error;
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
onChange(listener) {
|
|
2685
|
+
this.listeners.add(listener);
|
|
2686
|
+
return () => {
|
|
2687
|
+
this.listeners.delete(listener);
|
|
2688
|
+
};
|
|
2689
|
+
}
|
|
2690
|
+
get state() {
|
|
2691
|
+
return structuredClone(this.value);
|
|
2692
|
+
}
|
|
2693
|
+
get snapshotError() {
|
|
2694
|
+
return this.snapshotFailure;
|
|
2695
|
+
}
|
|
2696
|
+
append(input) {
|
|
2697
|
+
if (this.closing)
|
|
2698
|
+
return Promise.reject(new JournalError("CLOSED", "The journal is closing or closed."));
|
|
2699
|
+
let captured;
|
|
2700
|
+
try {
|
|
2701
|
+
captured = structuredClone(eventInput(input));
|
|
2702
|
+
} catch (error) {
|
|
2703
|
+
return Promise.reject(error);
|
|
2704
|
+
}
|
|
2705
|
+
const writing = this.queue.then(async () => {
|
|
2706
|
+
if (this.poisoned)
|
|
2707
|
+
throw new JournalError("POISONED", "A journal write failed. Close and recover before continuing.");
|
|
2708
|
+
const event = { ...captured, version: 1, seq: this.value.lastSeq + 1, at: new Date(this.clock.now()).toISOString() };
|
|
2709
|
+
const next = applyEvent(this.value, event);
|
|
2710
|
+
const record2 = Buffer.from(JSON.stringify(event) + `
|
|
2711
|
+
`);
|
|
2712
|
+
const reserve = event.type === "step.started" || event.type === "sleep.scheduled" ? 3 * maxRecordBytes : 0;
|
|
2713
|
+
if (record2.length > maxRecordBytes || this.bytes + record2.length + reserve > maxJournalBytes) {
|
|
2714
|
+
throw new JournalError("LIMIT", "The event or run journal exceeds its size limit.");
|
|
2715
|
+
}
|
|
2716
|
+
try {
|
|
2717
|
+
await this.storage.append(record2);
|
|
2718
|
+
} catch (error) {
|
|
2719
|
+
this.poisoned = true;
|
|
2720
|
+
throw error;
|
|
2721
|
+
}
|
|
2722
|
+
this.value = next;
|
|
2723
|
+
this.bytes += record2.length;
|
|
2724
|
+
await this.checkpoint();
|
|
2725
|
+
for (const listener of this.listeners) {
|
|
2726
|
+
try {
|
|
2727
|
+
listener();
|
|
2728
|
+
} catch (error) {
|
|
2729
|
+
console.error("Workflow observer failed:", error);
|
|
2730
|
+
}
|
|
2731
|
+
}
|
|
2732
|
+
return structuredClone(event);
|
|
2733
|
+
});
|
|
2734
|
+
this.queue = writing.catch(() => {});
|
|
2735
|
+
return writing;
|
|
2736
|
+
}
|
|
2737
|
+
close() {
|
|
2738
|
+
this.closing ??= this.queue.then(async () => {
|
|
2739
|
+
try {
|
|
2740
|
+
if (!this.poisoned && this.snapshotFailure) {
|
|
2741
|
+
await this.checkpoint();
|
|
2742
|
+
if (this.snapshotFailure)
|
|
2743
|
+
throw this.snapshotFailure;
|
|
2744
|
+
}
|
|
2745
|
+
} finally {
|
|
2746
|
+
await this.storage.close();
|
|
2747
|
+
}
|
|
2748
|
+
});
|
|
2749
|
+
return this.closing;
|
|
2750
|
+
}
|
|
2751
|
+
async checkpoint() {
|
|
2752
|
+
try {
|
|
2753
|
+
await this.storage.replaceSnapshot(JSON.stringify(this.value) + `
|
|
2754
|
+
`);
|
|
2755
|
+
this.snapshotFailure = null;
|
|
2756
|
+
} catch (error) {
|
|
2757
|
+
this.snapshotFailure = error;
|
|
2758
|
+
}
|
|
2759
|
+
}
|
|
2760
|
+
}
|
|
2761
|
+
|
|
2762
|
+
// src/workflow/summary.ts
|
|
2763
|
+
function summarize(run, state) {
|
|
2764
|
+
const step = state.currentStep && Object.hasOwn(state.steps, state.currentStep) ? state.steps[state.currentStep] : null;
|
|
2765
|
+
const sleep = state.currentStep && Object.hasOwn(state.sleeps, state.currentStep) ? state.sleeps[state.currentStep] : null;
|
|
2766
|
+
return {
|
|
2767
|
+
id: run.id,
|
|
2768
|
+
workflow: run.workflow,
|
|
2769
|
+
status: state.status,
|
|
2770
|
+
currentStep: state.currentStep,
|
|
2771
|
+
attempt: step?.attempt ?? 0,
|
|
2772
|
+
createdAt: run.createdAt,
|
|
2773
|
+
updatedAt: state.updatedAt ?? run.createdAt,
|
|
2774
|
+
nextAttemptAt: !activeStatuses.has(state.status) ? null : sleep?.status === "waiting" ? sleep.wakeAt : step?.nextAttemptAt ?? null,
|
|
2775
|
+
deadlineAt: state.deadlineAt
|
|
2776
|
+
};
|
|
2777
|
+
}
|
|
2778
|
+
var terminalStatuses = new Set(["completed", "failed", "cancelled", "expired", "unknown"]);
|
|
2779
|
+
var activeStatuses = new Set(["pending", "running", "waiting"]);
|
|
2780
|
+
var openStatuses = new Set([...activeStatuses, "attention"]);
|
|
2781
|
+
function describe(state) {
|
|
2782
|
+
const bounded = (value) => JSON.stringify(value).length <= 16 * 1024 ? value : null;
|
|
2783
|
+
const result = bounded(state.result);
|
|
2784
|
+
return {
|
|
2785
|
+
error: briefError(state.error),
|
|
2786
|
+
input: bounded(state.input),
|
|
2787
|
+
result,
|
|
2788
|
+
resultTruncated: result === null && state.result !== null,
|
|
2789
|
+
inputTruncated: bounded(state.input) === null && state.input !== null,
|
|
2790
|
+
expiryReason: state.expiryReason,
|
|
2791
|
+
cancelRequested: state.cancelRequested,
|
|
2792
|
+
steps: state.operationOrder.slice(-20).map((id) => {
|
|
2793
|
+
const sleep = Object.hasOwn(state.sleeps, id) ? state.sleeps[id] : null;
|
|
2794
|
+
if (sleep)
|
|
2795
|
+
return { id, kind: "sleep", status: sleep.status, attempt: 0, attempts: 0, error: null, nextAttemptAt: activeStatuses.has(state.status) && sleep.status === "waiting" ? sleep.wakeAt : null, durationMs: sleep.durationMs };
|
|
2796
|
+
const step = state.steps[id];
|
|
2797
|
+
return { id, kind: "step", durationMs: null, status: step.status, attempt: step.attempt, attempts: step.limit, error: briefError(step.error), nextAttemptAt: activeStatuses.has(state.status) ? step.nextAttemptAt : null };
|
|
2798
|
+
})
|
|
2799
|
+
};
|
|
2800
|
+
}
|
|
2801
|
+
function briefError(error) {
|
|
2802
|
+
return error ? { ...error, message: error.message.slice(0, 4000) } : null;
|
|
2803
|
+
}
|
|
2804
|
+
function unavailableDetail(input, error) {
|
|
2805
|
+
return describe({ ...initialState(), input, error: error ? { message: error } : null });
|
|
2806
|
+
}
|
|
2807
|
+
|
|
2808
|
+
// src/workflow/service.ts
|
|
2809
|
+
var scopeMatches = (run, request) => run.dir === request.dir && run.plugin === request.plugin && run.namespace === request.namespace && run.workflow === request.workflow && run.root === request.root;
|
|
2810
|
+
var runReserve = 130 * 1024 * 1024;
|
|
2811
|
+
var storageFull = "The workflow storage budget is fully reserved by unfinished runs and retained history. Finish or cancel runs, or wait for retention cleanup.";
|
|
2812
|
+
var executor = fileURLToPath(new URL(import.meta.url.endsWith(".ts") ? "./executor.ts" : "./workflow-executor.js", import.meta.url));
|
|
2813
|
+
|
|
2814
|
+
class WorkflowService {
|
|
2815
|
+
dataDir;
|
|
2816
|
+
catalog;
|
|
2817
|
+
callbacks;
|
|
2818
|
+
clock;
|
|
2819
|
+
storageLimit;
|
|
2820
|
+
network;
|
|
2821
|
+
queue = Promise.resolve();
|
|
2822
|
+
details = new Map;
|
|
2823
|
+
rows = new Map;
|
|
2824
|
+
unrecovered = new Set;
|
|
2825
|
+
maintenanceErrors = new Map;
|
|
2826
|
+
cancelling = new Set;
|
|
2827
|
+
active = new Map;
|
|
2828
|
+
observers = new Set;
|
|
2829
|
+
timer = null;
|
|
2830
|
+
closing = false;
|
|
2831
|
+
started = false;
|
|
2832
|
+
constructor(dataDir, catalog, callbacks, clock, storageLimit, network) {
|
|
2833
|
+
this.dataDir = dataDir;
|
|
2834
|
+
this.catalog = catalog;
|
|
2835
|
+
this.callbacks = callbacks;
|
|
2836
|
+
this.clock = clock;
|
|
2837
|
+
this.storageLimit = storageLimit;
|
|
2838
|
+
this.network = network;
|
|
2839
|
+
}
|
|
2840
|
+
static async open(dataDir, callbacks, options = {}) {
|
|
2841
|
+
const service = new WorkflowService(dataDir, await WorkflowCatalog.open(join6(dataDir, ".workflows")), callbacks, options.clock ?? systemClock, options.storageLimit ?? 512 * 1024 * 1024, options.network ?? networkAvailable);
|
|
2842
|
+
try {
|
|
2843
|
+
for (const row of await service.catalog.all()) {
|
|
2844
|
+
row.run = RunMetadataSchema.parse(row.run);
|
|
2845
|
+
service.rows.set(row.run.id, row);
|
|
2846
|
+
}
|
|
2847
|
+
await service.recoverCatalog();
|
|
2848
|
+
return service;
|
|
2849
|
+
} catch (error) {
|
|
2850
|
+
await service.catalog.close();
|
|
2851
|
+
throw error;
|
|
2852
|
+
}
|
|
2853
|
+
}
|
|
2854
|
+
directory(run) {
|
|
2855
|
+
return join6(this.dataDir, run.namespace, "workflows", run.id);
|
|
2856
|
+
}
|
|
2857
|
+
serial(work) {
|
|
2858
|
+
const result = this.queue.then(work);
|
|
2859
|
+
this.queue = result.catch(() => {});
|
|
2860
|
+
return result;
|
|
2861
|
+
}
|
|
2862
|
+
observe(listener) {
|
|
2863
|
+
this.observers.add(listener);
|
|
2864
|
+
return () => {
|
|
2865
|
+
this.observers.delete(listener);
|
|
2866
|
+
};
|
|
2867
|
+
}
|
|
2868
|
+
start() {
|
|
2869
|
+
if (!this.closing) {
|
|
2870
|
+
this.started = true;
|
|
2871
|
+
this.wake();
|
|
2872
|
+
}
|
|
2873
|
+
}
|
|
2874
|
+
wake() {
|
|
2875
|
+
this.timer?.();
|
|
2876
|
+
this.timer = null;
|
|
2877
|
+
if (!this.started || this.closing)
|
|
2878
|
+
return;
|
|
2879
|
+
let due = this.clock.now() + (this.rows.size ? 5000 : 60000);
|
|
2880
|
+
for (const row of this.rows.values()) {
|
|
2881
|
+
if (!activeStatuses.has(row.summary.status) && row.summary.status !== "blocked")
|
|
2882
|
+
continue;
|
|
2883
|
+
due = Math.min(due, this.clock.now() + row.run.maxOfflineMs / 2);
|
|
2884
|
+
if (this.active.has(row.run.id) || this.active.size >= 4)
|
|
2885
|
+
continue;
|
|
2886
|
+
due = Math.min(due, row.summary.nextAttemptAt ? Date.parse(row.summary.nextAttemptAt) : this.clock.now());
|
|
2887
|
+
if (row.summary.status !== "blocked" && row.summary.deadlineAt)
|
|
2888
|
+
due = Math.min(due, Date.parse(row.summary.deadlineAt));
|
|
2889
|
+
}
|
|
2890
|
+
this.timer = this.clock.schedule(due, () => {
|
|
2891
|
+
this.tick().finally(() => this.wake()).catch(() => {});
|
|
2892
|
+
});
|
|
2893
|
+
}
|
|
2894
|
+
async save(row) {
|
|
2895
|
+
this.rows.set(row.run.id, row);
|
|
2896
|
+
await this.catalog.put(row);
|
|
2897
|
+
this.callbacks.invalidate([workflowTag(row.run.namespace)]);
|
|
2898
|
+
for (const observer of this.observers) {
|
|
2899
|
+
try {
|
|
2900
|
+
observer(row.summary);
|
|
2901
|
+
} catch (error) {
|
|
2902
|
+
console.error(error);
|
|
2903
|
+
}
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2906
|
+
async request(request) {
|
|
2907
|
+
if (this.closing)
|
|
2908
|
+
throw new MethodError("INTERNAL", "The workflow service is closing.");
|
|
2909
|
+
if (request.operation === "list") {
|
|
2910
|
+
const rows = [...this.rows.values()].filter((row2) => scopeMatches(row2.run, request) && (!request.input.status || row2.summary.status === request.input.status)).sort((a, b) => b.run.createdAt.localeCompare(a.run.createdAt) || b.run.id.localeCompare(a.run.id));
|
|
2911
|
+
const cursor = request.input.cursor;
|
|
2912
|
+
const start = cursor ? rows.findIndex((row2) => row2.run.id === cursor) + 1 : 0;
|
|
2913
|
+
if (cursor && start === 0)
|
|
2914
|
+
throw new MethodError("VALIDATION", "Invalid workflow cursor.");
|
|
2915
|
+
const count = Math.min(100, Math.max(1, Number(request.input.limit ?? 20)));
|
|
2916
|
+
const page = rows.slice(start, start + count);
|
|
2917
|
+
return { items: page.map((row2) => row2.summary), cursor: start + count < rows.length ? page.at(-1).run.id : null };
|
|
2918
|
+
}
|
|
2919
|
+
if (request.operation === "start")
|
|
2920
|
+
return this.serial(() => this.create(request));
|
|
2921
|
+
if (request.operation === "cancelAll")
|
|
2922
|
+
return this.cancelAll(request);
|
|
2923
|
+
const row = this.rows.get(String(request.input.id));
|
|
2924
|
+
if (!row || !scopeMatches(row.run, request))
|
|
2925
|
+
throw new MethodError("NOT_FOUND", "No workflow run in this project.");
|
|
2926
|
+
if (request.operation === "retry")
|
|
2927
|
+
return this.serial(async () => {
|
|
2928
|
+
if (await this.bytes(row.run.id) > this.storageLimit)
|
|
2929
|
+
throw new MethodError("VALIDATION", storageFull);
|
|
2930
|
+
const active2 = this.active.get(row.run.id);
|
|
2931
|
+
if (active2) {
|
|
2932
|
+
await active2.ended;
|
|
2933
|
+
this.active.delete(row.run.id);
|
|
2934
|
+
}
|
|
2935
|
+
const journal = await this.openJournal(row.run);
|
|
2936
|
+
try {
|
|
2937
|
+
const stepId = journal.state.currentStep;
|
|
2938
|
+
if (!stepId)
|
|
2939
|
+
throw new MethodError("VALIDATION", "This run has no failed query to retry.");
|
|
2940
|
+
try {
|
|
2941
|
+
await journal.append({ type: "run.retryRequested", stepId });
|
|
2942
|
+
} catch (error) {
|
|
2943
|
+
throw new MethodError("VALIDATION", toCallError(error).message);
|
|
2944
|
+
}
|
|
2945
|
+
row.summary = summarize(row.run, journal.state);
|
|
2946
|
+
row.terminal = null;
|
|
2947
|
+
row.error = null;
|
|
2948
|
+
row.seen = this.clock.now();
|
|
2949
|
+
await this.save(row);
|
|
2950
|
+
this.wake();
|
|
2951
|
+
return row.summary;
|
|
2952
|
+
} finally {
|
|
2953
|
+
await journal.close();
|
|
2954
|
+
}
|
|
2955
|
+
});
|
|
2956
|
+
if (request.operation === "restart")
|
|
2957
|
+
return this.serial(async () => {
|
|
2958
|
+
const journal = await this.openJournal(row.run);
|
|
2959
|
+
try {
|
|
2960
|
+
if (journal.state.status !== "attention")
|
|
2961
|
+
throw new MethodError("VALIDATION", "Only a run that needs attention can restart.");
|
|
2962
|
+
await journal.append({ type: "run.restarted" });
|
|
2963
|
+
row.summary = summarize(row.run, journal.state);
|
|
2964
|
+
row.error = null;
|
|
2965
|
+
row.seen = this.clock.now();
|
|
2966
|
+
await this.save(row);
|
|
2967
|
+
this.wake();
|
|
2968
|
+
return row.summary;
|
|
2969
|
+
} finally {
|
|
2970
|
+
await journal.close();
|
|
2971
|
+
}
|
|
2972
|
+
});
|
|
2973
|
+
if (request.operation === "cancel")
|
|
2974
|
+
return this.cancel(row);
|
|
2975
|
+
return this.serial(async () => {
|
|
2976
|
+
const fallback = unavailableDetail(row.run.input, row.error);
|
|
2977
|
+
if (this.active.has(row.run.id))
|
|
2978
|
+
return { ...row.summary, ...this.details.get(row.run.id) ?? fallback };
|
|
2979
|
+
let journal;
|
|
2980
|
+
try {
|
|
2981
|
+
journal = await this.openJournal(row.run);
|
|
2982
|
+
} catch (error) {
|
|
2983
|
+
if (row.summary.status === "blocked" || row.error)
|
|
2984
|
+
return { ...row.summary, ...fallback };
|
|
2985
|
+
throw error;
|
|
2986
|
+
}
|
|
2987
|
+
try {
|
|
2988
|
+
const state = journal.state;
|
|
2989
|
+
const summary = row.summary.status === "blocked" ? row.summary : summarize(row.run, state);
|
|
2990
|
+
return { ...summary, ...describe(state), error: briefError(state.error ?? (row.error ? { message: row.error } : null)) };
|
|
2991
|
+
} finally {
|
|
2992
|
+
await journal.close();
|
|
2993
|
+
}
|
|
2994
|
+
});
|
|
2995
|
+
}
|
|
2996
|
+
async cancelAll(request) {
|
|
2997
|
+
const status = request.input.status;
|
|
2998
|
+
const rows = [...this.rows.values()].filter((row) => scopeMatches(row.run, request) && (openStatuses.has(row.summary.status) || row.summary.status === "blocked") && (!status || row.summary.status === status));
|
|
2999
|
+
for (const row of rows)
|
|
3000
|
+
await this.cancel(row);
|
|
3001
|
+
return { cancelled: rows.length };
|
|
3002
|
+
}
|
|
3003
|
+
cancel(row) {
|
|
3004
|
+
const active2 = this.active.get(row.run.id);
|
|
3005
|
+
if (active2 && active2.worker.exitCode === null)
|
|
3006
|
+
return this.cancelActive(row, active2);
|
|
3007
|
+
return this.serial(async () => {
|
|
3008
|
+
if (active2) {
|
|
3009
|
+
await active2.ended;
|
|
3010
|
+
this.active.delete(row.run.id);
|
|
3011
|
+
}
|
|
3012
|
+
return this.cancelIdle(row);
|
|
3013
|
+
});
|
|
3014
|
+
}
|
|
3015
|
+
async cancelIdle(row) {
|
|
3016
|
+
const journal = await this.openJournal(row.run);
|
|
3017
|
+
try {
|
|
3018
|
+
if (openStatuses.has(journal.state.status)) {
|
|
3019
|
+
await journal.append({ type: "run.cancelRequested" });
|
|
3020
|
+
await settleInterrupted(journal, "CANCELLED");
|
|
3021
|
+
await journal.append({ type: "run.cancelled" });
|
|
3022
|
+
}
|
|
3023
|
+
row.summary = summarize(row.run, journal.state);
|
|
3024
|
+
if (terminalStatuses.has(row.summary.status))
|
|
3025
|
+
row.terminal ??= Date.parse(row.summary.updatedAt);
|
|
3026
|
+
await this.save(row);
|
|
3027
|
+
return row.summary;
|
|
3028
|
+
} finally {
|
|
3029
|
+
await journal.close();
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
async cancelActive(row, active2) {
|
|
3033
|
+
this.cancelling.add(row.run.id);
|
|
3034
|
+
const ack = Promise.withResolvers();
|
|
3035
|
+
active2.acknowledge = ack.resolve;
|
|
3036
|
+
let timer;
|
|
3037
|
+
try {
|
|
3038
|
+
active2.worker.send({ type: "cancel" });
|
|
3039
|
+
const accepted = await Promise.race([ack.promise, active2.worker.exited.then(() => null), new Promise((resolve2) => {
|
|
3040
|
+
timer = setTimeout(() => {
|
|
3041
|
+
active2.worker.kill("SIGKILL");
|
|
3042
|
+
resolve2(null);
|
|
3043
|
+
}, 1000);
|
|
3044
|
+
})]);
|
|
3045
|
+
if (accepted)
|
|
3046
|
+
return accepted;
|
|
3047
|
+
await active2.worker.exited;
|
|
3048
|
+
return await this.serial(async () => {
|
|
3049
|
+
this.active.delete(row.run.id);
|
|
3050
|
+
return this.cancelIdle(row);
|
|
3051
|
+
});
|
|
3052
|
+
} finally {
|
|
3053
|
+
if (timer)
|
|
3054
|
+
clearTimeout(timer);
|
|
3055
|
+
delete active2.acknowledge;
|
|
3056
|
+
this.cancelling.delete(row.run.id);
|
|
3057
|
+
this.wake();
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3060
|
+
async create(request) {
|
|
3061
|
+
const input = z7.json().parse(request.input.input);
|
|
3062
|
+
const key = request.input.key ? String(request.input.key) : null;
|
|
3063
|
+
const found = key ? [...this.rows.values()].find((row2) => scopeMatches(row2.run, request) && row2.run.key === key) : undefined;
|
|
3064
|
+
if (found) {
|
|
3065
|
+
if (!isDeepStrictEqual2(found.run.input, input))
|
|
3066
|
+
throw new MethodError("VALIDATION", "This workflow key already has different input.");
|
|
3067
|
+
return found.summary;
|
|
3068
|
+
}
|
|
3069
|
+
if (!request.settings)
|
|
3070
|
+
throw new MethodError("VALIDATION", "Workflow settings are required.");
|
|
3071
|
+
if (await this.bytes() + runReserve > this.storageLimit)
|
|
3072
|
+
throw new MethodError("VALIDATION", storageFull);
|
|
3073
|
+
let run = RunMetadataSchema.parse({ dir: request.dir, root: request.root, namespace: request.namespace, plugin: request.plugin, workflow: request.workflow, caller: request.caller ?? "code", input, key, id: crypto.randomUUID(), createdAt: new Date(this.clock.now()).toISOString(), ...request.settings });
|
|
3074
|
+
const journal = await RunJournal.open(this.directory(run), this.clock);
|
|
3075
|
+
let summary;
|
|
3076
|
+
try {
|
|
3077
|
+
const { workflow, input: savedInput, timeoutMs, createdAt, ...location } = run;
|
|
3078
|
+
await journal.append({ type: "run.started", workflow, input: savedInput, timeoutMs, location });
|
|
3079
|
+
run = journal.state.metadata;
|
|
3080
|
+
summary = summarize(run, journal.state);
|
|
3081
|
+
} finally {
|
|
3082
|
+
await journal.close();
|
|
3083
|
+
}
|
|
3084
|
+
const row = { run, summary, seen: this.clock.now(), terminal: null, error: null };
|
|
3085
|
+
await this.save(row);
|
|
3086
|
+
if (!await this.catalog.availability())
|
|
3087
|
+
await this.catalog.heartbeat(nextAvailability(null, this.clock.now(), this.network()));
|
|
3088
|
+
this.wake();
|
|
3089
|
+
return summary;
|
|
3090
|
+
}
|
|
3091
|
+
tick() {
|
|
3092
|
+
return this.serial(async () => {
|
|
3093
|
+
if (this.closing)
|
|
3094
|
+
return;
|
|
3095
|
+
const now = this.clock.now();
|
|
3096
|
+
if (!this.rows.size) {
|
|
3097
|
+
await this.recoverCatalog();
|
|
3098
|
+
return;
|
|
3099
|
+
}
|
|
3100
|
+
const availability = await this.catalog.availability();
|
|
3101
|
+
const next = nextAvailability(availability, now, this.network());
|
|
3102
|
+
const offlineSince = availability?.offlineSince ?? next.offlineSince;
|
|
3103
|
+
for (const row of [...this.rows.values()]) {
|
|
3104
|
+
try {
|
|
3105
|
+
const networkExpired = offlineSince !== null && now - Math.max(offlineSince, Date.parse(row.run.createdAt)) >= row.run.maxOfflineMs;
|
|
3106
|
+
const unavailable = availability !== null && now - Math.max(availability.seen, Date.parse(row.run.createdAt)) >= row.run.maxOfflineMs;
|
|
3107
|
+
const deadlineReached = row.summary.deadlineAt !== null && now >= Date.parse(row.summary.deadlineAt);
|
|
3108
|
+
if (this.active.has(row.run.id)) {
|
|
3109
|
+
if (!networkExpired && !unavailable && !deadlineReached) {
|
|
3110
|
+
row.seen = now;
|
|
3111
|
+
continue;
|
|
3112
|
+
}
|
|
3113
|
+
const active2 = this.active.get(row.run.id);
|
|
3114
|
+
active2.worker.kill("SIGKILL");
|
|
3115
|
+
await active2.worker.exited;
|
|
3116
|
+
this.active.delete(row.run.id);
|
|
3117
|
+
}
|
|
3118
|
+
if (row.terminal !== null && now - row.terminal < row.run.retentionMs)
|
|
3119
|
+
continue;
|
|
3120
|
+
if (availability && !networkExpired && !unavailable && !deadlineReached && row.summary.nextAttemptAt && Date.parse(row.summary.nextAttemptAt) > now) {
|
|
3121
|
+
row.seen = now;
|
|
3122
|
+
continue;
|
|
3123
|
+
}
|
|
3124
|
+
const blockedUntil = row.summary.status === "blocked" ? row.summary.nextAttemptAt : null;
|
|
3125
|
+
try {
|
|
3126
|
+
const journal = await this.openJournal(row.run);
|
|
3127
|
+
try {
|
|
3128
|
+
const state = journal.state;
|
|
3129
|
+
const offline = networkExpired || unavailable || !availability && now - row.seen >= row.run.maxOfflineMs;
|
|
3130
|
+
const expired = state.deadlineAt !== null && now >= Date.parse(state.deadlineAt);
|
|
3131
|
+
if (openStatuses.has(state.status) && (offline || expired)) {
|
|
3132
|
+
await settleInterrupted(journal, "EXPIRED");
|
|
3133
|
+
await journal.append({ type: "run.expired", reason: offline ? "Environment unavailable beyond the workflow offline limit." : "Workflow deadline reached." });
|
|
3134
|
+
}
|
|
3135
|
+
row.summary = summarize(row.run, journal.state);
|
|
3136
|
+
if (activeStatuses.has(row.summary.status) && blockedUntil && Date.parse(blockedUntil) > now)
|
|
3137
|
+
row.summary = { ...row.summary, status: "blocked", nextAttemptAt: blockedUntil };
|
|
3138
|
+
row.seen = now;
|
|
3139
|
+
if (journal.state.endedAt)
|
|
3140
|
+
row.terminal ??= Date.parse(journal.state.endedAt);
|
|
3141
|
+
await this.save(row);
|
|
3142
|
+
} finally {
|
|
3143
|
+
await journal.close();
|
|
3144
|
+
}
|
|
3145
|
+
} catch (error) {
|
|
3146
|
+
row.error = toCallError(error).message;
|
|
3147
|
+
if (!terminalStatuses.has(row.summary.status)) {
|
|
3148
|
+
const expired = now - row.seen >= row.run.maxOfflineMs || row.summary.deadlineAt !== null && now >= Date.parse(row.summary.deadlineAt);
|
|
3149
|
+
row.summary = { ...row.summary, status: expired ? "expired" : "blocked", nextAttemptAt: expired ? null : new Date(now + 60000).toISOString() };
|
|
3150
|
+
if (expired)
|
|
3151
|
+
row.terminal ??= now;
|
|
3152
|
+
}
|
|
3153
|
+
await this.save(row);
|
|
3154
|
+
}
|
|
3155
|
+
if (row.terminal !== null && now - row.terminal >= row.run.retentionMs) {
|
|
3156
|
+
await rm3(this.directory(row.run), { recursive: true, force: true });
|
|
3157
|
+
await this.catalog.remove(row.run.id);
|
|
3158
|
+
this.rows.delete(row.run.id);
|
|
3159
|
+
this.details.delete(row.run.id);
|
|
3160
|
+
this.maintenanceErrors.delete(row.run.id);
|
|
3161
|
+
this.callbacks.invalidate([workflowTag(row.run.namespace)]);
|
|
3162
|
+
continue;
|
|
3163
|
+
}
|
|
3164
|
+
if (!this.cancelling.has(row.run.id) && activeStatuses.has(row.summary.status) && this.active.size < 4 && (!row.summary.nextAttemptAt || now >= Date.parse(row.summary.nextAttemptAt)))
|
|
3165
|
+
await this.launch(row);
|
|
3166
|
+
this.maintenanceErrors.delete(row.run.id);
|
|
3167
|
+
} catch (error) {
|
|
3168
|
+
row.error = toCallError(error).message;
|
|
3169
|
+
if (this.maintenanceErrors.get(row.run.id) !== row.error)
|
|
3170
|
+
console.error("Workflow maintenance failed:", row.run.id, row.error);
|
|
3171
|
+
this.maintenanceErrors.set(row.run.id, row.error);
|
|
3172
|
+
}
|
|
3173
|
+
}
|
|
3174
|
+
await this.catalog.heartbeat(next);
|
|
3175
|
+
await this.recoverCatalog();
|
|
3176
|
+
});
|
|
3177
|
+
}
|
|
3178
|
+
async launch(row) {
|
|
3179
|
+
try {
|
|
3180
|
+
const worker = Bun.spawn([process.execPath, "--no-env-file", executor], {
|
|
3181
|
+
cwd: row.run.dir,
|
|
3182
|
+
stdin: "ignore",
|
|
3183
|
+
stdout: "ignore",
|
|
3184
|
+
stderr: "pipe",
|
|
3185
|
+
ipc: (message) => onMessage(message),
|
|
3186
|
+
serialization: "advanced"
|
|
3187
|
+
});
|
|
3188
|
+
const stderr = stderrTail(worker.stderr);
|
|
3189
|
+
let finish;
|
|
3190
|
+
const ended = new Promise((resolve2) => {
|
|
3191
|
+
finish = resolve2;
|
|
3192
|
+
});
|
|
3193
|
+
const active2 = { worker, ended, finish, error: null };
|
|
3194
|
+
this.active.set(row.run.id, active2);
|
|
3195
|
+
let finished = false;
|
|
3196
|
+
const complete = () => {
|
|
3197
|
+
if (finished)
|
|
3198
|
+
return;
|
|
3199
|
+
finished = true;
|
|
3200
|
+
active2.finish();
|
|
3201
|
+
if (this.closing)
|
|
3202
|
+
return;
|
|
3203
|
+
this.serial(async () => {
|
|
3204
|
+
if (this.active.get(row.run.id) !== active2)
|
|
3205
|
+
return;
|
|
3206
|
+
this.active.delete(row.run.id);
|
|
3207
|
+
row.error = active2.error;
|
|
3208
|
+
if (active2.error && !terminalStatuses.has(row.summary.status)) {
|
|
3209
|
+
row.summary = { ...row.summary, status: "blocked", nextAttemptAt: new Date(this.clock.now() + 60000).toISOString() };
|
|
3210
|
+
await this.save(row);
|
|
3211
|
+
}
|
|
3212
|
+
}).then(() => this.tick()).then(() => this.wake()).catch(() => {});
|
|
3213
|
+
};
|
|
3214
|
+
const onMessage = (message) => {
|
|
3215
|
+
if (this.closing)
|
|
3216
|
+
return;
|
|
3217
|
+
if (message.type === "host") {
|
|
3218
|
+
this.callbacks.host(message.plugin, message.method, message.params).then((result) => worker.send({ type: "hosted", id: message.id, result }), (error) => worker.send({ type: "hosted", id: message.id, error: toCallError(error) })).catch(() => {});
|
|
3219
|
+
} else if (message.type === "state") {
|
|
3220
|
+
const summary = RunSummarySchema.safeParse(message.summary);
|
|
3221
|
+
if (!summary.success || summary.data.id !== row.run.id || summary.data.workflow !== row.run.workflow || summary.data.createdAt !== row.run.createdAt) {
|
|
3222
|
+
active2.error = "The workflow executor reported an invalid run identity.";
|
|
3223
|
+
return;
|
|
3224
|
+
}
|
|
3225
|
+
if (message.detail.cancelRequested)
|
|
3226
|
+
active2.acknowledge?.(message.summary);
|
|
3227
|
+
this.serial(async () => {
|
|
3228
|
+
if (this.active.get(row.run.id) !== active2)
|
|
3229
|
+
return;
|
|
3230
|
+
row.summary = message.summary;
|
|
3231
|
+
this.details.set(row.run.id, message.detail);
|
|
3232
|
+
row.seen = this.clock.now();
|
|
3233
|
+
await this.save(row);
|
|
3234
|
+
});
|
|
3235
|
+
} else if (message.type === "invalidated")
|
|
3236
|
+
this.callbacks.invalidate(message.tags);
|
|
3237
|
+
else if (message.type === "notice")
|
|
3238
|
+
this.callbacks.notify(message.notice);
|
|
3239
|
+
else if (message.type === "log")
|
|
3240
|
+
this.callbacks.logs(row.run.dir, message.records);
|
|
3241
|
+
else if (message.type === "error")
|
|
3242
|
+
active2.error = message.message;
|
|
3243
|
+
else if (message.type === "done") {}
|
|
3244
|
+
};
|
|
3245
|
+
worker.exited.then(async (code) => {
|
|
3246
|
+
if (code !== 0 && !active2.error)
|
|
3247
|
+
active2.error = (await stderr).slice(-4000) || `Workflow executor exited ${code}.`;
|
|
3248
|
+
complete();
|
|
3249
|
+
});
|
|
3250
|
+
worker.send({ type: "start", run: row.run, directory: this.directory(row.run), dataDir: this.dataDir });
|
|
3251
|
+
} catch (error) {
|
|
3252
|
+
row.summary = { ...row.summary, status: "blocked", nextAttemptAt: new Date(this.clock.now() + 60000).toISOString() };
|
|
3253
|
+
row.error = toCallError(error).message;
|
|
3254
|
+
await this.save(row);
|
|
3255
|
+
}
|
|
3256
|
+
}
|
|
3257
|
+
async openJournal(run) {
|
|
3258
|
+
const path = this.directory(run);
|
|
3259
|
+
if (!(await stat3(join6(path, "events.jsonl"))).size)
|
|
3260
|
+
throw new Error("The workflow journal is missing or empty; execution is blocked.");
|
|
3261
|
+
return RunJournal.openExisting(path, this.clock);
|
|
3262
|
+
}
|
|
3263
|
+
async recoverCatalog() {
|
|
3264
|
+
let complete = true;
|
|
3265
|
+
const namespaces = await readdir4(this.dataDir, { withFileTypes: true });
|
|
3266
|
+
for (const namespace of namespaces) {
|
|
3267
|
+
if (!namespace.isDirectory() || !/^[a-z0-9][a-z0-9.-]*$/.test(namespace.name))
|
|
3268
|
+
continue;
|
|
3269
|
+
const base = join6(this.dataDir, namespace.name, "workflows");
|
|
3270
|
+
const entries = await readdir4(base, { withFileTypes: true }).catch((error) => {
|
|
3271
|
+
if (error.code !== "ENOENT") {
|
|
3272
|
+
complete = false;
|
|
3273
|
+
if (!this.unrecovered.has(base))
|
|
3274
|
+
console.error("Workflow discovery blocked:", base, toCallError(error).message);
|
|
3275
|
+
this.unrecovered.add(base);
|
|
3276
|
+
}
|
|
3277
|
+
return [];
|
|
3278
|
+
});
|
|
3279
|
+
for (const entry of entries) {
|
|
3280
|
+
if (!entry.isDirectory() || !z7.string().uuid().safeParse(entry.name).success || this.rows.has(entry.name))
|
|
3281
|
+
continue;
|
|
3282
|
+
const directory = join6(base, entry.name);
|
|
3283
|
+
try {
|
|
3284
|
+
const history = await stat3(join6(directory, "events.jsonl")).catch((error) => {
|
|
3285
|
+
if (error.code !== "ENOENT")
|
|
3286
|
+
throw error;
|
|
3287
|
+
return null;
|
|
3288
|
+
});
|
|
3289
|
+
if (!history?.size)
|
|
3290
|
+
continue;
|
|
3291
|
+
const journal = await RunJournal.openExisting(directory, this.clock);
|
|
3292
|
+
try {
|
|
3293
|
+
const state = journal.state;
|
|
3294
|
+
const run = state.metadata;
|
|
3295
|
+
if (!run || run.id !== entry.name || run.namespace !== namespace.name)
|
|
3296
|
+
throw new Error("Workflow journal location is invalid.");
|
|
3297
|
+
const row = { run, summary: summarize(run, state), seen: Date.parse(state.updatedAt), terminal: state.endedAt ? Date.parse(state.endedAt) : null, error: null };
|
|
3298
|
+
await this.catalog.put(row);
|
|
3299
|
+
this.rows.set(run.id, row);
|
|
3300
|
+
this.unrecovered.delete(directory);
|
|
3301
|
+
} finally {
|
|
3302
|
+
await journal.close();
|
|
3303
|
+
}
|
|
3304
|
+
} catch (error) {
|
|
3305
|
+
complete = false;
|
|
3306
|
+
if (!this.unrecovered.has(directory))
|
|
3307
|
+
console.error("Workflow discovery blocked:", directory, toCallError(error).message);
|
|
3308
|
+
this.unrecovered.add(directory);
|
|
3309
|
+
}
|
|
3310
|
+
}
|
|
3311
|
+
}
|
|
3312
|
+
return complete;
|
|
3313
|
+
}
|
|
3314
|
+
async bytes(reserveId) {
|
|
3315
|
+
const tree = async (path) => {
|
|
3316
|
+
const entries = await readdir4(path, { withFileTypes: true }).catch((error) => {
|
|
3317
|
+
if (error.code !== "ENOENT")
|
|
3318
|
+
throw error;
|
|
3319
|
+
return [];
|
|
3320
|
+
});
|
|
3321
|
+
let size = 0;
|
|
3322
|
+
for (const entry of entries) {
|
|
3323
|
+
const name = join6(path, entry.name);
|
|
3324
|
+
if (entry.isDirectory())
|
|
3325
|
+
size += await tree(name);
|
|
3326
|
+
else
|
|
3327
|
+
size += await stat3(name).then((info) => info.size, (error) => {
|
|
3328
|
+
if (error.code !== "ENOENT")
|
|
3329
|
+
throw error;
|
|
3330
|
+
return 0;
|
|
3331
|
+
});
|
|
3332
|
+
}
|
|
3333
|
+
return size;
|
|
3334
|
+
};
|
|
3335
|
+
let bytes = 0;
|
|
3336
|
+
for (const row of this.rows.values()) {
|
|
3337
|
+
const size = await tree(this.directory(row.run));
|
|
3338
|
+
const reserved = openStatuses.has(row.summary.status) || row.summary.status === "blocked" || row.run.id === reserveId;
|
|
3339
|
+
bytes += reserved ? Math.max(size, runReserve) : size;
|
|
3340
|
+
}
|
|
3341
|
+
return bytes;
|
|
3342
|
+
}
|
|
3343
|
+
async close() {
|
|
3344
|
+
if (this.closing)
|
|
3345
|
+
return;
|
|
3346
|
+
this.closing = true;
|
|
3347
|
+
this.timer?.();
|
|
3348
|
+
this.timer = null;
|
|
3349
|
+
const active2 = [...this.active.values()];
|
|
3350
|
+
for (const value of active2)
|
|
3351
|
+
value.worker.send({ type: "suspend" });
|
|
3352
|
+
let timer;
|
|
3353
|
+
try {
|
|
3354
|
+
await Promise.race([Promise.all(active2.map((value) => value.worker.exited)), new Promise((resolve2) => {
|
|
3355
|
+
timer = setTimeout(resolve2, 1000);
|
|
3356
|
+
})]);
|
|
3357
|
+
} finally {
|
|
3358
|
+
if (timer)
|
|
3359
|
+
clearTimeout(timer);
|
|
3360
|
+
}
|
|
3361
|
+
for (const value of active2)
|
|
3362
|
+
if (value.worker.exitCode === null)
|
|
3363
|
+
value.worker.kill("SIGKILL");
|
|
3364
|
+
await Promise.all(active2.map((value) => value.worker.exited));
|
|
3365
|
+
await this.queue;
|
|
3366
|
+
await this.catalog.close();
|
|
3367
|
+
}
|
|
3368
|
+
}
|
|
3369
|
+
async function stderrTail(stream) {
|
|
3370
|
+
const decoder = new TextDecoder;
|
|
3371
|
+
let tail = "";
|
|
3372
|
+
for await (const chunk of stream)
|
|
3373
|
+
tail = (tail + decoder.decode(chunk, { stream: true })).slice(-4000);
|
|
3374
|
+
return (tail + decoder.decode()).slice(-4000);
|
|
3375
|
+
}
|
|
3376
|
+
|
|
3377
|
+
// src/workflow/testing.ts
|
|
3378
|
+
async function testWorkflowRuntime(directory, options = {}) {
|
|
3379
|
+
const dataDir = await mkdtemp(join7(tmpdir(), "repogo-workflow-test-"));
|
|
3380
|
+
let runtime;
|
|
3381
|
+
const host = async (_plugin, method, params) => {
|
|
3382
|
+
if (!options.host)
|
|
3383
|
+
throw new MethodError("FORBIDDEN", `This test does not serve ${method}.`);
|
|
3384
|
+
return options.host(method, params);
|
|
3385
|
+
};
|
|
3386
|
+
const service = await WorkflowService.open(dataDir, {
|
|
3387
|
+
host,
|
|
3388
|
+
invalidate: () => {},
|
|
3389
|
+
notify: () => {},
|
|
3390
|
+
logs: () => {}
|
|
3391
|
+
});
|
|
3392
|
+
try {
|
|
3393
|
+
runtime = await Runtime.open({ dataDir, root: resolve2(options.root ?? directory), host, workflows: (request) => service.request(request) });
|
|
3394
|
+
await runtime.loadDir(resolve2(directory));
|
|
3395
|
+
service.start();
|
|
3396
|
+
return {
|
|
3397
|
+
call: async (method, input = {}, call = {}) => (await runtime.call(method, input, call)).result,
|
|
3398
|
+
manifest: () => runtime.manifest(),
|
|
3399
|
+
wait: (method, id, timeout = 1e4) => new Promise((resolve3, reject) => {
|
|
3400
|
+
const finish = () => {
|
|
3401
|
+
clearTimeout(timer);
|
|
3402
|
+
remove();
|
|
3403
|
+
runtime.call(`${method}.get`, { id }).then((value) => resolve3(value.result), reject);
|
|
3404
|
+
};
|
|
3405
|
+
const remove = service.observe((summary) => {
|
|
3406
|
+
if (summary.id === id && (terminalStatuses.has(summary.status) || summary.status === "blocked"))
|
|
3407
|
+
finish();
|
|
3408
|
+
});
|
|
3409
|
+
const timer = setTimeout(() => {
|
|
3410
|
+
remove();
|
|
3411
|
+
reject(new Error(`Workflow ${id} did not finish within ${timeout}ms.`));
|
|
3412
|
+
}, timeout);
|
|
3413
|
+
runtime.call(`${method}.get`, { id }).then((value) => {
|
|
3414
|
+
const summary = value.result;
|
|
3415
|
+
if (terminalStatuses.has(summary.status) || summary.status === "blocked")
|
|
3416
|
+
finish();
|
|
3417
|
+
}, (error) => {
|
|
3418
|
+
clearTimeout(timer);
|
|
3419
|
+
remove();
|
|
3420
|
+
reject(error);
|
|
3421
|
+
});
|
|
3422
|
+
}),
|
|
3423
|
+
close: async () => {
|
|
3424
|
+
await service.close();
|
|
3425
|
+
runtime.close();
|
|
3426
|
+
await rm4(dataDir, { recursive: true, force: true });
|
|
3427
|
+
}
|
|
3428
|
+
};
|
|
3429
|
+
} catch (error) {
|
|
3430
|
+
await service.close();
|
|
3431
|
+
runtime?.close();
|
|
3432
|
+
await rm4(dataDir, { recursive: true, force: true });
|
|
3433
|
+
throw error;
|
|
3434
|
+
}
|
|
3435
|
+
}
|
|
3436
|
+
|
|
1925
3437
|
// src/testing.ts
|
|
1926
3438
|
async function testRuntime(plugins, options = {}) {
|
|
1927
|
-
const dataDir = await
|
|
1928
|
-
const project =
|
|
1929
|
-
await
|
|
3439
|
+
const dataDir = await mkdtemp2(join8(tmpdir2(), "repogo-plugin-test-"));
|
|
3440
|
+
const project = join8(dataDir, "project");
|
|
3441
|
+
await mkdir6(project);
|
|
1930
3442
|
const host = options.mcp || options.host ? testHost(options.mcp, options.host) : undefined;
|
|
1931
|
-
const runtime = await Runtime.open({ dataDir:
|
|
3443
|
+
const runtime = await Runtime.open({ dataDir: join8(dataDir, "data"), root: options.root ?? project, host });
|
|
1932
3444
|
const notices = [];
|
|
1933
3445
|
runtime.onNotify((notice) => notices.push(notice));
|
|
1934
3446
|
const logs = [];
|
|
@@ -1939,7 +3451,7 @@ async function testRuntime(plugins, options = {}) {
|
|
|
1939
3451
|
} catch (error) {
|
|
1940
3452
|
releaseConsole();
|
|
1941
3453
|
runtime.close();
|
|
1942
|
-
await
|
|
3454
|
+
await rm5(dataDir, { recursive: true, force: true });
|
|
1943
3455
|
throw error;
|
|
1944
3456
|
}
|
|
1945
3457
|
return {
|
|
@@ -1954,7 +3466,7 @@ async function testRuntime(plugins, options = {}) {
|
|
|
1954
3466
|
close: async () => {
|
|
1955
3467
|
releaseConsole();
|
|
1956
3468
|
runtime.close();
|
|
1957
|
-
await
|
|
3469
|
+
await rm5(dataDir, { recursive: true, force: true });
|
|
1958
3470
|
}
|
|
1959
3471
|
};
|
|
1960
3472
|
}
|
|
@@ -1975,5 +3487,6 @@ function testHost(mcp, host) {
|
|
|
1975
3487
|
};
|
|
1976
3488
|
}
|
|
1977
3489
|
export {
|
|
3490
|
+
testWorkflowRuntime,
|
|
1978
3491
|
testRuntime
|
|
1979
3492
|
};
|