@voltro/workflow 0.83.1 → 0.84.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +642 -0
- package/dist/{cluster--WvQZFwk.js → cluster-i3frC3Da.js} +142 -133
- package/dist/cluster.d.ts +13 -13
- package/dist/cluster.js +1 -1
- package/dist/clusterTestSuite.js +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +2 -2
- package/package.json +4 -4
|
@@ -2,13 +2,13 @@ import { _ as e, a as t, b as n, c as r, l as i, m as a, n as o, p as s, s as c,
|
|
|
2
2
|
import { Context as d, Duration as f, Effect as p, Layer as m, Option as h, Runtime as g, Schedule as _, Schema as v } from "effect";
|
|
3
3
|
import { eq as y } from "@voltro/database";
|
|
4
4
|
import { SqlClient as b } from "@effect/sql";
|
|
5
|
-
import { ClusterCron as ee, ClusterSchema as te, ClusterWorkflowEngine as x, EntityAddress as ne, EntityId as re, EntityType as ie, MessageStorage as S, RunnerAddress as C, RunnerHealth as ae, RunnerStorage as w, Runners as oe, ShardId as se, Sharding as T, ShardingConfig as
|
|
6
|
-
import { applyUndeclaredCharsetWork as
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
5
|
+
import { ClusterCron as ee, ClusterSchema as te, ClusterWorkflowEngine as x, EntityAddress as ne, EntityId as re, EntityType as ie, MessageStorage as S, RunnerAddress as C, RunnerHealth as ae, RunnerStorage as w, Runners as oe, ShardId as se, Sharding as T, ShardingConfig as E, SqlMessageStorage as D, SqlRunnerStorage as O } from "@effect/cluster";
|
|
6
|
+
import { applyUndeclaredCharsetWork as ce, fnv1a64 as k, readUndeclaredCharsetColumns as le, resolveMigrationLockSchema as A, sqlPoolCapacity as ue, undeclaredCharsetWork as de } from "@voltro/database/sql";
|
|
7
|
+
import { PersistenceError as fe } from "@effect/cluster/ClusterError";
|
|
8
|
+
import { realpath as pe } from "node:fs/promises";
|
|
9
|
+
import { createLogger as me } from "@voltro/logger";
|
|
10
10
|
//#region src/workflowResumeDispatch.ts
|
|
11
|
-
var
|
|
11
|
+
var he = (e) => p.gen(function* () {
|
|
12
12
|
let t = yield* p.runtime(), r = yield* p.promise(() => e.transaction((n) => u(n, e.requestId, (e) => g.runPromise(t)(s(e)))));
|
|
13
13
|
if (r.kind !== "bound") return r;
|
|
14
14
|
let i = yield* p.promise(() => e.transaction((t) => n(t, e.requestId)));
|
|
@@ -25,10 +25,10 @@ var _e = (e) => p.gen(function* () {
|
|
|
25
25
|
request: o,
|
|
26
26
|
journal: c
|
|
27
27
|
};
|
|
28
|
-
}),
|
|
28
|
+
}), j = class extends d.Tag("@voltro/workflow/WorkflowResumeDispatcher")() {}, ge = p.gen(function* () {
|
|
29
29
|
let e = yield* b.SqlClient, t = yield* T.Sharding;
|
|
30
|
-
return
|
|
31
|
-
}),
|
|
30
|
+
return j.of({ dispatch: (n) => he(n).pipe(p.provideService(b.SqlClient, e), p.provideService(T.Sharding, t)) });
|
|
31
|
+
}), _e = async (e) => {
|
|
32
32
|
if (e.workflow.name !== e.command.workflowName || e.control.workflowName !== e.command.workflowName) throw Error("Restart preparation requires the matching workflow");
|
|
33
33
|
let n = await e.transaction(async (n) => {
|
|
34
34
|
let r = await t(n, e.command);
|
|
@@ -54,17 +54,17 @@ var _e = (e) => p.gen(function* () {
|
|
|
54
54
|
kind: "reserved",
|
|
55
55
|
request: n.request,
|
|
56
56
|
reused: !0
|
|
57
|
-
} :
|
|
57
|
+
} : ye({
|
|
58
58
|
...e,
|
|
59
59
|
command: {
|
|
60
60
|
...e.command,
|
|
61
61
|
generation: n.generation
|
|
62
62
|
}
|
|
63
63
|
});
|
|
64
|
-
},
|
|
64
|
+
}, ve = v.Struct({
|
|
65
65
|
requestId: v.NonEmptyString,
|
|
66
66
|
restartToken: v.NullOr(v.NonEmptyString)
|
|
67
|
-
}),
|
|
67
|
+
}), ye = async (e) => {
|
|
68
68
|
let { command: t } = e;
|
|
69
69
|
if (e.workflow.name !== t.workflowName || e.control.workflowName !== t.workflowName) throw Error("Restart preparation requires the matching workflow");
|
|
70
70
|
let n = await e.transaction(async (n) => {
|
|
@@ -87,7 +87,7 @@ var _e = (e) => p.gen(function* () {
|
|
|
87
87
|
return {
|
|
88
88
|
kind: "observe",
|
|
89
89
|
executionId: i.executionId,
|
|
90
|
-
delivery: v.decodeUnknownSync(
|
|
90
|
+
delivery: v.decodeUnknownSync(ve)(i.engineDelivery),
|
|
91
91
|
activities: await e.readActivities(n)
|
|
92
92
|
};
|
|
93
93
|
});
|
|
@@ -130,16 +130,16 @@ var _e = (e) => p.gen(function* () {
|
|
|
130
130
|
now: e.now
|
|
131
131
|
});
|
|
132
132
|
});
|
|
133
|
-
},
|
|
133
|
+
}, M = (e, t) => `${e}/${t}`, N = (e, t, n) => {
|
|
134
134
|
let r = re.make(t), i = d.get(e.annotations, te.ShardGroup)(r);
|
|
135
135
|
return new ne.EntityAddress({
|
|
136
136
|
entityType: ie.EntityType.make(`Workflow/${e.name}`),
|
|
137
137
|
entityId: r,
|
|
138
138
|
shardId: n.getShardId(r, i)
|
|
139
139
|
});
|
|
140
|
-
},
|
|
140
|
+
}, be = (e, t) => p.gen(function* () {
|
|
141
141
|
let n = yield* T.Sharding, r = yield* S.MessageStorage, i = yield* r.requestIdForPrimaryKey({
|
|
142
|
-
address:
|
|
142
|
+
address: N(e, t, n),
|
|
143
143
|
tag: "run",
|
|
144
144
|
id: ""
|
|
145
145
|
});
|
|
@@ -154,23 +154,23 @@ var _e = (e) => p.gen(function* () {
|
|
|
154
154
|
replyId: String(o.id),
|
|
155
155
|
exit: o.exit
|
|
156
156
|
};
|
|
157
|
-
}),
|
|
157
|
+
}), P = v.decodeUnknownOption(v.Exit({
|
|
158
158
|
success: v.Unknown,
|
|
159
159
|
failure: v.Unknown,
|
|
160
160
|
defect: v.Unknown
|
|
161
|
-
})),
|
|
162
|
-
let t = (e) => e && typeof e == "object" && "_tag" in e && typeof e._tag == "string" ? e._tag : void 0, n =
|
|
161
|
+
})), F = (e) => {
|
|
162
|
+
let t = (e) => e && typeof e == "object" && "_tag" in e && typeof e._tag == "string" ? e._tag : void 0, n = P(e);
|
|
163
163
|
if (h.isNone(n)) return "unrecognized-reply";
|
|
164
164
|
if (n.value._tag === "Failure") return "failed";
|
|
165
165
|
let r = n.value.value, i = t(r);
|
|
166
166
|
if (i === "Suspended") return "suspended";
|
|
167
167
|
if (i === "Complete") {
|
|
168
|
-
let e = r.exit, t =
|
|
168
|
+
let e = r.exit, t = P(e);
|
|
169
169
|
return h.isNone(t) ? "unrecognized-reply" : t.value._tag === "Success" ? "succeeded" : "failed";
|
|
170
170
|
}
|
|
171
171
|
return "unrecognized-reply";
|
|
172
|
-
},
|
|
173
|
-
let r = yield* T.Sharding, i = yield* S.MessageStorage, a =
|
|
172
|
+
}, xe = (e) => I(e.workflow, e.executionId, e.failedActivities), Se = (e) => I(e.workflow, e.executionId, e.resetActivities), I = (e, t, n) => p.gen(function* () {
|
|
173
|
+
let r = yield* T.Sharding, i = yield* S.MessageStorage, a = N(e, t, r), o = yield* i.requestIdForPrimaryKey({
|
|
174
174
|
address: a,
|
|
175
175
|
tag: "run",
|
|
176
176
|
id: ""
|
|
@@ -188,7 +188,7 @@ var _e = (e) => p.gen(function* () {
|
|
|
188
188
|
activitiesReset: 0,
|
|
189
189
|
reason: "no-reply"
|
|
190
190
|
};
|
|
191
|
-
let c =
|
|
191
|
+
let c = F(s.exit);
|
|
192
192
|
if (c !== "failed") return {
|
|
193
193
|
redriven: !1,
|
|
194
194
|
runReplyCleared: !1,
|
|
@@ -200,7 +200,7 @@ var _e = (e) => p.gen(function* () {
|
|
|
200
200
|
let t = yield* i.requestIdForPrimaryKey({
|
|
201
201
|
address: a,
|
|
202
202
|
tag: "activity",
|
|
203
|
-
id:
|
|
203
|
+
id: M(e.name, e.attempt)
|
|
204
204
|
}).pipe(p.orDie);
|
|
205
205
|
if (h.isNone(t)) return {
|
|
206
206
|
redriven: !1,
|
|
@@ -232,12 +232,12 @@ var _e = (e) => p.gen(function* () {
|
|
|
232
232
|
activitiesReset: u,
|
|
233
233
|
reason: "run-reset-unconfirmed"
|
|
234
234
|
};
|
|
235
|
-
}),
|
|
235
|
+
}), Ce = v.parseJson(v.Record({
|
|
236
236
|
key: v.String,
|
|
237
237
|
value: v.String
|
|
238
|
-
})),
|
|
238
|
+
})), we = (e) => p.gen(function* () {
|
|
239
239
|
if (!/^[1-9]\d*$/.test(e.delivery.requestId) || e.executionId.length === 0 || e.activities.some((e) => e.name.length === 0 || !Number.isSafeInteger(e.attempt) || e.attempt < 1)) return yield* p.die(/* @__PURE__ */ Error("Restart observation requires an exact delivery and valid activity attempts"));
|
|
240
|
-
let t = (yield* b.SqlClient).withoutTransforms(), n = yield* S.MessageStorage, r = yield* T.Sharding, i =
|
|
240
|
+
let t = (yield* b.SqlClient).withoutTransforms(), n = yield* S.MessageStorage, r = yield* T.Sharding, i = N(e.workflow, e.executionId, r), a = (n, r) => p.gen(function* () {
|
|
241
241
|
let [i] = yield* t`SELECT m.entity_type, m.entity_id, m.tag, m.headers, r.id AS reply_id, r.payload
|
|
242
242
|
FROM cluster_messages m
|
|
243
243
|
LEFT JOIN cluster_replies r ON r.id = m.last_reply_id AND r.request_id = m.id AND r.kind = 0
|
|
@@ -248,7 +248,7 @@ var _e = (e) => p.gen(function* () {
|
|
|
248
248
|
kind: "unavailable",
|
|
249
249
|
reason: "no-journal"
|
|
250
250
|
};
|
|
251
|
-
if (((yield* p.sync(() => v.decodeUnknownSync(
|
|
251
|
+
if (((yield* p.sync(() => v.decodeUnknownSync(Ce)(o.headers)))["voltro-workflow-restart"] ?? null) !== e.delivery.restartToken) return {
|
|
252
252
|
kind: "unavailable",
|
|
253
253
|
reason: "delivery-changed"
|
|
254
254
|
};
|
|
@@ -256,7 +256,7 @@ var _e = (e) => p.gen(function* () {
|
|
|
256
256
|
kind: "unavailable",
|
|
257
257
|
reason: "no-reply"
|
|
258
258
|
};
|
|
259
|
-
let s =
|
|
259
|
+
let s = F(yield* p.sync(() => JSON.parse(o.payload)));
|
|
260
260
|
if (s !== "failed") return {
|
|
261
261
|
kind: "unavailable",
|
|
262
262
|
reason: s
|
|
@@ -266,7 +266,7 @@ var _e = (e) => p.gen(function* () {
|
|
|
266
266
|
let e = yield* n.requestIdForPrimaryKey({
|
|
267
267
|
address: i,
|
|
268
268
|
tag: "activity",
|
|
269
|
-
id:
|
|
269
|
+
id: M(t.name, t.attempt)
|
|
270
270
|
});
|
|
271
271
|
if (h.isNone(e)) return {
|
|
272
272
|
kind: "unavailable",
|
|
@@ -286,59 +286,59 @@ var _e = (e) => p.gen(function* () {
|
|
|
286
286
|
replyId: t
|
|
287
287
|
}))
|
|
288
288
|
};
|
|
289
|
-
}).pipe(p.provideService(b.SafeIntegers, !0)),
|
|
289
|
+
}).pipe(p.provideService(b.SafeIntegers, !0)), Te = (e) => p.gen(function* () {
|
|
290
290
|
let t = yield* i(e);
|
|
291
291
|
return t.kind === "waiting-capacity" ? t : (yield* (yield* T.Sharding).pollStorage, {
|
|
292
292
|
kind: "attempted",
|
|
293
293
|
journal: t
|
|
294
294
|
});
|
|
295
|
-
}),
|
|
296
|
-
let e = yield* b.SqlClient, t = yield* T.Sharding, n = yield* S.MessageStorage, r = (r) =>
|
|
297
|
-
return
|
|
295
|
+
}), L = class extends d.Tag("@voltro/workflow/WorkflowRestartDispatcher")() {}, Ee = p.gen(function* () {
|
|
296
|
+
let e = yield* b.SqlClient, t = yield* T.Sharding, n = yield* S.MessageStorage, r = (r) => we(r).pipe(p.provideService(b.SqlClient, e), p.provideService(T.Sharding, t), p.provideService(S.MessageStorage, n));
|
|
297
|
+
return L.of({
|
|
298
298
|
prepare: (e) => p.tryPromise({
|
|
299
|
-
try: (t) =>
|
|
299
|
+
try: (t) => _e({
|
|
300
300
|
...e,
|
|
301
301
|
observe: (e) => p.runPromise(r(e), { signal: t })
|
|
302
302
|
}),
|
|
303
303
|
catch: (e) => e
|
|
304
304
|
}),
|
|
305
305
|
observe: r,
|
|
306
|
-
dispatch: (n) =>
|
|
306
|
+
dispatch: (n) => Te(n).pipe(p.provideService(b.SqlClient, e), p.provideService(T.Sharding, t))
|
|
307
307
|
});
|
|
308
|
-
}),
|
|
309
|
-
let e =
|
|
310
|
-
return e === void 0 ?
|
|
311
|
-
},
|
|
312
|
-
let e = `voltro_cluster_migration_${
|
|
308
|
+
}), R = 6322741009312601n, z = () => {
|
|
309
|
+
let e = A();
|
|
310
|
+
return e === void 0 ? R : k(`voltro_cluster_migration_lock:${e}`) & 9223372036854775807n;
|
|
311
|
+
}, B = () => {
|
|
312
|
+
let e = `voltro_cluster_migration_${R.toString(36)}`, t = A();
|
|
313
313
|
if (t === void 0) return e;
|
|
314
314
|
let n = `${e}_${t}`;
|
|
315
|
-
return /^[A-Za-z0-9_]+$/.test(t) && n.length <= 64 ? n : `${e}_h${
|
|
316
|
-
},
|
|
315
|
+
return /^[A-Za-z0-9_]+$/.test(t) && n.length <= 64 ? n : `${e}_h${k(`voltro_cluster_migration_lock:${t}`).toString(16)}`;
|
|
316
|
+
}, V = () => {
|
|
317
317
|
let e = Number(process.env.VOLTRO_CLUSTER_MIGRATION_LOCK_TIMEOUT_MS);
|
|
318
318
|
return Number.isFinite(e) && e > 0 ? e : 6e4;
|
|
319
|
-
},
|
|
319
|
+
}, De = 250, H = (e) => e[0]?.[0], Oe = (e, t) => e.onDialectOrElse({
|
|
320
320
|
sqlite: () => p.void,
|
|
321
|
-
mysql: () => t.executeValues(`SELECT GET_LOCK('${
|
|
322
|
-
mssql: () => t.executeValues(`DECLARE @res int; EXEC @res = sp_getapplock @Resource = '${
|
|
321
|
+
mysql: () => t.executeValues(`SELECT GET_LOCK('${B()}', ${Math.ceil(V() / 1e3)})`, []).pipe(p.flatMap((e) => H(e) === 1 || H(e) === "1" ? p.void : p.die(/* @__PURE__ */ Error(`could not acquire cluster migration lock '${B()}' (GET_LOCK timed out)`)))),
|
|
322
|
+
mssql: () => t.executeValues(`DECLARE @res int; EXEC @res = sp_getapplock @Resource = '${B()}', @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = ${V()}; SELECT @res`, []).pipe(p.flatMap((e) => Number(H(e) ?? -999) >= 0 ? p.void : p.die(/* @__PURE__ */ Error(`could not acquire cluster migration lock '${B()}' (sp_getapplock ${String(H(e))})`)))),
|
|
323
323
|
orElse: () => {
|
|
324
|
-
let e =
|
|
325
|
-
let t =
|
|
326
|
-
return t === !0 || t === "t" || t === 1 ? p.void : Date.now() - n >=
|
|
324
|
+
let e = z().toString(), n = Date.now(), r = t.executeValues(`SELECT pg_try_advisory_lock(${e}::bigint)`, []).pipe(p.flatMap((e) => {
|
|
325
|
+
let t = H(e);
|
|
326
|
+
return t === !0 || t === "t" || t === 1 ? p.void : Date.now() - n >= V() ? p.die(/* @__PURE__ */ Error(`could not acquire the cluster migration advisory lock within ${Math.round(V() / 1e3)}s. Another runner is migrating the cluster schema, or a prior boot wedged while holding it. (Raise VOLTRO_CLUSTER_MIGRATION_LOCK_TIMEOUT_MS, or inspect pg_locks for the holder.)`)) : p.sleep(`${De} millis`).pipe(p.zipRight(r));
|
|
327
327
|
}));
|
|
328
328
|
return r;
|
|
329
329
|
}
|
|
330
|
-
}),
|
|
330
|
+
}), ke = (e, t) => e.onDialectOrElse({
|
|
331
331
|
sqlite: () => p.void,
|
|
332
|
-
mysql: () => p.asVoid(t.executeValues(`SELECT RELEASE_LOCK('${
|
|
333
|
-
mssql: () => p.asVoid(t.executeValues(`EXEC sp_releaseapplock @Resource = '${
|
|
334
|
-
orElse: () => p.asVoid(t.executeValues(`SELECT pg_advisory_unlock(${
|
|
335
|
-
}),
|
|
332
|
+
mysql: () => p.asVoid(t.executeValues(`SELECT RELEASE_LOCK('${B()}')`, [])),
|
|
333
|
+
mssql: () => p.asVoid(t.executeValues(`EXEC sp_releaseapplock @Resource = '${B()}', @LockOwner = 'Session'`, [])),
|
|
334
|
+
orElse: () => p.asVoid(t.executeValues(`SELECT pg_advisory_unlock(${z().toString()}::bigint)`, []))
|
|
335
|
+
}), Ae = (e, t) => e.onDialectOrElse({
|
|
336
336
|
sqlite: () => !0,
|
|
337
337
|
orElse: () => !1
|
|
338
338
|
}) ? t : p.scoped(p.gen(function* () {
|
|
339
339
|
let n = yield* e.reserve;
|
|
340
|
-
return yield* p.acquireRelease(
|
|
341
|
-
})),
|
|
340
|
+
return yield* p.acquireRelease(Oe(e, n), () => ke(e, n).pipe(p.orDie)), yield* t;
|
|
341
|
+
})), je = (e) => e === "sql" ? ["cluster_messages", "cluster_runners"] : ["cluster_messages"], Me = (e, t) => e.onDialectOrElse({
|
|
342
342
|
pg: () => e`SELECT (to_regclass(${t}) IS NOT NULL) AS present`,
|
|
343
343
|
mysql: () => e`SELECT (COUNT(*) > 0) AS present FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ${t}`,
|
|
344
344
|
mssql: () => e`SELECT CASE WHEN OBJECT_ID(${t}, 'U') IS NULL THEN 0 ELSE 1 END AS present`,
|
|
@@ -346,63 +346,99 @@ var _e = (e) => p.gen(function* () {
|
|
|
346
346
|
}).pipe(p.map((e) => {
|
|
347
347
|
let t = e[0]?.present;
|
|
348
348
|
return t === !0 || t === 1 || t === "1" || t === "t" || typeof t == "bigint" && t === 1n;
|
|
349
|
-
}), p.catchAll(() => p.succeed(!1))),
|
|
349
|
+
}), p.catchAll(() => p.succeed(!1))), Ne = (e, t) => p.forEach(je(t), (t) => Me(e, t), { concurrency: 1 }).pipe(p.map((e) => e.every(Boolean))), U = (e, t) => p.scoped(p.asVoid(m.build(t.pipe(m.provide(E.layerDefaults), m.provide(m.succeed(b.SqlClient, e)))))), W = (e, t) => U(e, D.layer).pipe(p.zipRight(t === "sql" ? U(e, O.layer) : p.void)), G = (e) => p.map(e.unsafe("SELECT table_name AS t FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name LIKE 'cluster\\_%'"), (e) => new Set(e.map((e) => String(e.t ?? e.T)))), Pe = (e, t, n) => n === void 0 || !e.onDialectOrElse({
|
|
350
350
|
mysql: () => !0,
|
|
351
351
|
orElse: () => !1
|
|
352
|
-
}) ?
|
|
353
|
-
let r = yield*
|
|
354
|
-
yield*
|
|
355
|
-
let i = new Set([...yield*
|
|
356
|
-
i.size !== 0 && (yield*
|
|
357
|
-
}),
|
|
358
|
-
let t = yield* b.SqlClient, n =
|
|
359
|
-
return r !== void 0 && r < n ? yield* p.dieMessage(`the workflow runtime needs ${n} database connections at once and the pool has ${r}: ` + (e.runnerStorage === "sql" ? "on a first boot it holds its migration lock and its runner lock while the migration runs on a third, and once running it keeps the runner lock and needs one more to work. " : "on a first boot it holds its migration lock while the migration runs on a second. ") + `Set DB_MAX_CONNECTIONS (or PG_MAX_CONNECTIONS) to at least ${n}` + (e.runnerStorage === "sql" ? ", or VOLTRO_WORKFLOW_RUNNER_STORAGE=memory for a single-process runtime that needs 2 (no cross-pod shard handoff)." : ".")) : ((yield*
|
|
360
|
-
})),
|
|
352
|
+
}) ? W(e, t) : p.gen(function* () {
|
|
353
|
+
let r = yield* G(e);
|
|
354
|
+
yield* W(e, t);
|
|
355
|
+
let i = new Set([...yield* G(e)].filter((e) => !r.has(e)));
|
|
356
|
+
i.size !== 0 && (yield* ce(e, de(yield* le(e), /* @__PURE__ */ new Set(), n, i)));
|
|
357
|
+
}), Fe = (e) => e === "sql" ? 3 : 2, K = (e) => m.effect(b.SqlClient, p.gen(function* () {
|
|
358
|
+
let t = yield* b.SqlClient, n = Fe(e.runnerStorage), r = ue(t);
|
|
359
|
+
return r !== void 0 && r < n ? yield* p.dieMessage(`the workflow runtime needs ${n} database connections at once and the pool has ${r}: ` + (e.runnerStorage === "sql" ? "on a first boot it holds its migration lock and its runner lock while the migration runs on a third, and once running it keeps the runner lock and needs one more to work. " : "on a first boot it holds its migration lock while the migration runs on a second. ") + `Set DB_MAX_CONNECTIONS (or PG_MAX_CONNECTIONS) to at least ${n}` + (e.runnerStorage === "sql" ? ", or VOLTRO_WORKFLOW_RUNNER_STORAGE=memory for a single-process runtime that needs 2 (no cross-pod shard handoff)." : ".")) : ((yield* Ne(t, e.runnerStorage)) || (yield* Ae(t, Pe(t, e.runnerStorage, e.charset))), t);
|
|
360
|
+
})), Ie = (e, t) => p.gen(function* () {
|
|
361
|
+
let n = f.decode(t.interval), r = f.toMillis(t.giveUpAfter), i, a, o = !1, s = !1;
|
|
362
|
+
yield* t.probe.pipe(p.timeout(n), p.isSuccess, p.flatMap((e) => p.sync(() => {
|
|
363
|
+
let n = Date.now();
|
|
364
|
+
if (e) {
|
|
365
|
+
if (i = void 0, !o || (a ??= n, n - a < r)) return;
|
|
366
|
+
t.log.info("workflow runner: its storage answers again — taking part in shard assignment again"), a = void 0, o = !1, s = !1;
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
a = void 0, i ??= n, !o && n - i >= r && (o = !0, t.log.warn(`workflow runner: its connection pool has not served a query for ${Math.round((n - i) / 1e3)}s — released its shards and stopped its heartbeat, so another runner takes them`, { fix: "look for what holds this process's database connections (`voltro_db_operations_in_flight` against DB_MAX_CONNECTIONS)" }));
|
|
370
|
+
}))).pipe(p.repeat(_.spaced(n)), p.forkScoped);
|
|
371
|
+
let c = () => new fe({ cause: /* @__PURE__ */ Error("workflow runner gave its shards up: its connection pool does not serve queries") });
|
|
372
|
+
return w.RunnerStorage.of({
|
|
373
|
+
...e,
|
|
374
|
+
register: (t, n) => p.suspend(() => o ? p.fail(c()) : e.register(t, n)),
|
|
375
|
+
acquire: (t, n) => p.suspend(() => o ? p.succeed([]) : e.acquire(t, n)),
|
|
376
|
+
refresh: (t, n) => p.suspend(() => {
|
|
377
|
+
if (!o) return e.refresh(t, n);
|
|
378
|
+
let r = s ? p.void : e.releaseAll(t).pipe(p.tap(() => p.sync(() => {
|
|
379
|
+
s = !0;
|
|
380
|
+
})), p.ignore);
|
|
381
|
+
return p.zipRight(r, p.fail(c()));
|
|
382
|
+
})
|
|
383
|
+
});
|
|
384
|
+
}), Le = (e) => p.map(p.makeSemaphore(1), (t) => ({
|
|
385
|
+
...e,
|
|
386
|
+
release: (n, r) => t.withPermits(1)(p.suspend(() => e.release(n, r)))
|
|
387
|
+
})), Re = m.unwrapEffect(p.map(b.SqlClient, (e) => e.onDialectOrElse({
|
|
388
|
+
sqlite: () => m.effect(w.RunnerStorage, p.flatMap(w.RunnerStorage, Le)).pipe(m.provide(O.layer)),
|
|
389
|
+
orElse: () => O.layer
|
|
390
|
+
}))), ze = (e) => m.scoped(w.RunnerStorage, p.gen(function* () {
|
|
391
|
+
let t = yield* b.SqlClient;
|
|
392
|
+
return yield* Ie(yield* w.RunnerStorage, {
|
|
393
|
+
...e,
|
|
394
|
+
probe: t`SELECT 1`
|
|
395
|
+
});
|
|
396
|
+
})).pipe(m.provide(m.orDie(Re))), Be = (e) => T.layer.pipe(m.provideMerge(oe.layerNoop), m.provideMerge(D.layer), m.provide([e.runnerStorage === "memory" ? w.layerMemory : ze(e.guard), ae.layerNoop]), m.provide(E.layerFromEnv(e.shardingConfig))), Ve = (e) => p.gen(function* () {
|
|
361
397
|
let t = yield* p.tryPromise({
|
|
362
|
-
try: () =>
|
|
363
|
-
catch:
|
|
398
|
+
try: () => pe(e),
|
|
399
|
+
catch: q
|
|
364
400
|
}), n = yield* p.acquireRelease(p.tryPromise({
|
|
365
401
|
try: async () => {
|
|
366
402
|
let { DatabaseSync: e } = await import("node:sqlite");
|
|
367
403
|
return new e(`${t}.voltro-workflow-lock`);
|
|
368
404
|
},
|
|
369
|
-
catch:
|
|
405
|
+
catch: q
|
|
370
406
|
}), (e) => p.sync(() => e.close()));
|
|
371
407
|
yield* p.try({
|
|
372
408
|
try: () => {
|
|
373
409
|
n.exec("PRAGMA busy_timeout=0"), n.exec("CREATE TABLE IF NOT EXISTS workflow_mutex(id INTEGER PRIMARY KEY)"), n.exec("BEGIN IMMEDIATE");
|
|
374
410
|
},
|
|
375
411
|
catch: (e) => {
|
|
376
|
-
let n =
|
|
412
|
+
let n = q(e), r = n.errcode;
|
|
377
413
|
return r !== void 0 && ((r & 255) == 5 || (r & 255) == 6) ? Error("SQLite workflow journal is already owned by another engine. Stop that engine before starting another, or use a SQL dialect with shared runner coordination for replicas.", { cause: n }) : Error(`Could not acquire SQLite workflow ownership for ${t}`, { cause: n });
|
|
378
414
|
}
|
|
379
415
|
});
|
|
380
|
-
}),
|
|
416
|
+
}), q = (e) => e instanceof Error ? e : Error(String(e)), J = (e) => {
|
|
381
417
|
if (e instanceof Date) return Number.isNaN(e.getTime()) ? null : e;
|
|
382
418
|
if (typeof e == "string" || typeof e == "number") {
|
|
383
419
|
let t = typeof e == "string" && /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(e) ? `${e.replace(" ", "T")}Z` : e, n = new Date(t);
|
|
384
420
|
return Number.isNaN(n.getTime()) ? null : n;
|
|
385
421
|
}
|
|
386
422
|
return null;
|
|
387
|
-
},
|
|
423
|
+
}, Y = (e) => e === !0 || e === 1 || e === "1" || e instanceof Uint8Array && e[0] === 1, He = (e, t) => {
|
|
388
424
|
let n = e.lastIndexOf(":");
|
|
389
425
|
if (n < 0) return;
|
|
390
426
|
let r = e.slice(0, n), i = Number(e.slice(n + 1)), a = t.indexOf(r);
|
|
391
427
|
if (!(a < 0 || !Number.isInteger(i))) return (a + 1) * 1e6 + i;
|
|
392
|
-
},
|
|
428
|
+
}, X = (e) => p.gen(function* () {
|
|
393
429
|
let t = yield* b.SqlClient, n = e.shardGroups ?? ["default"], r = e.dialectId ?? "", i = r.startsWith("postgres") || r === "pg", a = (e) => e.pipe(p.catchAll(() => p.succeed([]))), o = t.onDialectOrElse({
|
|
394
430
|
pg: () => t.literal("NOW()"),
|
|
395
431
|
mysql: () => t.literal("NOW()"),
|
|
396
432
|
mssql: () => t.literal("GETDATE()"),
|
|
397
433
|
orElse: () => t.literal("CURRENT_TIMESTAMP")
|
|
398
434
|
}), s = (e, t) => {
|
|
399
|
-
let n =
|
|
435
|
+
let n = J(e.last_read), r = J(e.server_now), i = n !== null && r !== null ? Math.max(0, r.getTime() - n.getTime()) : null;
|
|
400
436
|
return {
|
|
401
437
|
tag: typeof e.tag == "string" ? e.tag : null,
|
|
402
438
|
entityType: String(e.entity_type ?? ""),
|
|
403
439
|
entityId: String(e.entity_id ?? ""),
|
|
404
440
|
shardId: String(e.shard_id ?? ""),
|
|
405
|
-
processed:
|
|
441
|
+
processed: Y(e.processed),
|
|
406
442
|
lastRead: i === null ? null : new Date(t - i)
|
|
407
443
|
};
|
|
408
444
|
}, c = (e) => t.onDialectOrElse({
|
|
@@ -447,12 +483,12 @@ var _e = (e) => p.gen(function* () {
|
|
|
447
483
|
mode: "row",
|
|
448
484
|
holder: r,
|
|
449
485
|
own: r === e.ownAddress || (e.hasShard?.(i) ?? !1),
|
|
450
|
-
since:
|
|
486
|
+
since: J(n.acquired_at)
|
|
451
487
|
};
|
|
452
488
|
}))), h = (r) => {
|
|
453
489
|
let i = /* @__PURE__ */ new Map();
|
|
454
490
|
for (let e of r) {
|
|
455
|
-
let t =
|
|
491
|
+
let t = He(e, n);
|
|
456
492
|
t !== void 0 && i.set(t, e);
|
|
457
493
|
}
|
|
458
494
|
return i.size === 0 ? p.succeed([]) : p.gen(function* () {
|
|
@@ -470,14 +506,14 @@ var _e = (e) => p.gen(function* () {
|
|
|
470
506
|
mode: "advisory",
|
|
471
507
|
holder: s ? `this runner (pid ${a})` : `pid ${a} from ${o}`,
|
|
472
508
|
own: s,
|
|
473
|
-
since:
|
|
509
|
+
since: J(n.state_change),
|
|
474
510
|
detail: {
|
|
475
511
|
pid: a,
|
|
476
512
|
clientAddr: n.client_addr ?? null,
|
|
477
513
|
applicationName: n.application_name ?? null,
|
|
478
514
|
state: n.state ?? null,
|
|
479
515
|
waitEventType: n.wait_event_type ?? null,
|
|
480
|
-
backendStart:
|
|
516
|
+
backendStart: J(n.backend_start)?.toISOString() ?? null
|
|
481
517
|
}
|
|
482
518
|
}];
|
|
483
519
|
});
|
|
@@ -493,8 +529,8 @@ var _e = (e) => p.gen(function* () {
|
|
|
493
529
|
let t = e;
|
|
494
530
|
return {
|
|
495
531
|
address: String(t.address ?? ""),
|
|
496
|
-
lastHeartbeat:
|
|
497
|
-
healthy:
|
|
532
|
+
lastHeartbeat: J(t.last_heartbeat),
|
|
533
|
+
healthy: Y(t.healthy)
|
|
498
534
|
};
|
|
499
535
|
}))) : p.succeed([]),
|
|
500
536
|
discardMessages: (e) => {
|
|
@@ -510,62 +546,38 @@ var _e = (e) => p.gen(function* () {
|
|
|
510
546
|
}).pipe(t.withTransaction, p.catchAll(() => p.succeed(0)));
|
|
511
547
|
}
|
|
512
548
|
};
|
|
513
|
-
}),
|
|
514
|
-
let n = f.decode(t.interval), r = f.toMillis(t.giveUpAfter), i, a, o = !1, s = !1;
|
|
515
|
-
yield* t.probe.pipe(p.timeout(n), p.isSuccess, p.flatMap((e) => p.sync(() => {
|
|
516
|
-
let n = Date.now();
|
|
517
|
-
if (e) {
|
|
518
|
-
if (i = void 0, !o || (a ??= n, n - a < r)) return;
|
|
519
|
-
t.log.info("workflow runner: its storage answers again — taking part in shard assignment again"), a = void 0, o = !1, s = !1;
|
|
520
|
-
return;
|
|
521
|
-
}
|
|
522
|
-
a = void 0, i ??= n, !o && n - i >= r && (o = !0, t.log.warn(`workflow runner: its connection pool has not served a query for ${Math.round((n - i) / 1e3)}s — released its shards and stopped its heartbeat, so another runner takes them`, { fix: "look for what holds this process's database connections (`voltro_db_operations_in_flight` against DB_MAX_CONNECTIONS)" }));
|
|
523
|
-
}))).pipe(p.repeat(_.spaced(n)), p.forkScoped);
|
|
524
|
-
let c = () => new ge({ cause: /* @__PURE__ */ Error("workflow runner gave its shards up: its connection pool does not serve queries") });
|
|
525
|
-
return w.RunnerStorage.of({
|
|
526
|
-
...e,
|
|
527
|
-
register: (t, n) => p.suspend(() => o ? p.fail(c()) : e.register(t, n)),
|
|
528
|
-
acquire: (t, n) => p.suspend(() => o ? p.succeed([]) : e.acquire(t, n)),
|
|
529
|
-
refresh: (t, n) => p.suspend(() => {
|
|
530
|
-
if (!o) return e.refresh(t, n);
|
|
531
|
-
let r = s ? p.void : e.releaseAll(t).pipe(p.tap(() => p.sync(() => {
|
|
532
|
-
s = !0;
|
|
533
|
-
})), p.ignore);
|
|
534
|
-
return p.zipRight(r, p.fail(c()));
|
|
535
|
-
})
|
|
536
|
-
});
|
|
537
|
-
}), X = he({ scope: "voltro:workflow" }), Ve = 34e3, Z = (e) => {
|
|
549
|
+
}), Z = me({ scope: "voltro:workflow" }), Ue = 34e3, Q = (e) => {
|
|
538
550
|
let t = e.runnerListenHost ?? process.env.POD_IP ?? process.env.VOLTRO_WORKFLOW_RUNNER_HOST ?? "localhost";
|
|
539
551
|
return {
|
|
540
552
|
host: t,
|
|
541
553
|
port: e.runnerListenPort ?? 34e3,
|
|
542
554
|
localhostRisk: e.runnerStorage === "sql" && (t === "localhost" || t === "127.0.0.1")
|
|
543
555
|
};
|
|
544
|
-
},
|
|
556
|
+
}, We = (e) => {
|
|
545
557
|
let t = e.sqlRunnerSupported ?? !e.sqliteFamily, n = t ? "sql" : "memory", r = e.override ?? process.env.VOLTRO_WORKFLOW_RUNNER_STORAGE;
|
|
546
558
|
if (r === void 0 || r.trim() === "") return n;
|
|
547
559
|
let i = r.trim().toLowerCase();
|
|
548
560
|
if (i !== "memory" && i !== "sql") throw Error(`VOLTRO_WORKFLOW_RUNNER_STORAGE must be 'memory' or 'sql', got '${r}'.`);
|
|
549
561
|
if (i === "sql" && !t) throw Error("VOLTRO_WORKFLOW_RUNNER_STORAGE='sql' is unsupported for this database connection. Use 'memory' for a local or embedded SQLite-family connection.");
|
|
550
562
|
return i;
|
|
551
|
-
},
|
|
563
|
+
}, $ = (e = process.env.VOLTRO_WORKFLOW_SHARD_LOCK) => {
|
|
552
564
|
if (e === void 0 || e.trim() === "") return "auto";
|
|
553
565
|
let t = e.trim().toLowerCase();
|
|
554
566
|
if (t === "auto" || t === "row" || t === "advisory") return t;
|
|
555
567
|
throw Error(`VOLTRO_WORKFLOW_SHARD_LOCK must be 'auto', 'row', or 'advisory', got '${e}'.`);
|
|
556
|
-
},
|
|
568
|
+
}, Ge = p.gen(function* () {
|
|
557
569
|
let e = yield* (yield* b.SqlClient)`SELECT @@wsrep_on AS wsrep`.pipe(p.catchAll(() => p.succeed([])));
|
|
558
570
|
if (e.length === 0) return !1;
|
|
559
571
|
let t = e[0].wsrep;
|
|
560
572
|
return t === 1 || t === "1" || t === !0 || t === "ON" || t === "on";
|
|
561
|
-
}),
|
|
562
|
-
return e.runnerStorage === "memory" ? !1 : e.dialectId === "sqlite" || e.dialectId === "turso" || e.mode === "row" ? !0 : e.mode === "advisory" || e.dialectId !== "mysql" && e.dialectId !== "mariadb" ? !1 : yield*
|
|
563
|
-
}),
|
|
573
|
+
}), Ke = (e) => p.gen(function* () {
|
|
574
|
+
return e.runnerStorage === "memory" ? !1 : e.dialectId === "sqlite" || e.dialectId === "turso" || e.mode === "row" ? !0 : e.mode === "advisory" || e.dialectId !== "mysql" && e.dialectId !== "mariadb" ? !1 : yield* Ge;
|
|
575
|
+
}), qe = (e) => ({
|
|
564
576
|
coordination: e.disableAdvisory ? "row-based" : "advisory",
|
|
565
577
|
dialect: e.dialectId ?? "unknown",
|
|
566
578
|
mode: e.mode,
|
|
567
579
|
...e.disableAdvisory && e.mode === "auto" ? { reason: e.dialectId === "sqlite" || e.dialectId === "turso" ? "SQLite-family connection uses row locks" : "wsrep/Galera cluster detected" } : {}
|
|
568
|
-
}),
|
|
580
|
+
}), Je = () => {
|
|
569
581
|
let e = (e) => {
|
|
570
582
|
if (e === void 0) return;
|
|
571
583
|
let t = Number(e);
|
|
@@ -576,30 +588,30 @@ var _e = (e) => p.gen(function* () {
|
|
|
576
588
|
...n === void 0 ? {} : { failoverHeartbeatSeconds: n },
|
|
577
589
|
...r === void 0 ? {} : { messagePollSeconds: r }
|
|
578
590
|
};
|
|
579
|
-
},
|
|
580
|
-
let n =
|
|
591
|
+
}, Ye = p.flatMap(T.Sharding, (e) => e.pollStorage), Xe = /* @__PURE__ */ new Set(), Ze = (t) => {
|
|
592
|
+
let n = Q({
|
|
581
593
|
runnerStorage: t.runnerStorage,
|
|
582
594
|
runnerListenHost: t.runnerListenHost,
|
|
583
595
|
runnerListenPort: t.runnerListenPort
|
|
584
596
|
}), r = n.host, i = n.port;
|
|
585
|
-
n.localhostRisk &&
|
|
597
|
+
n.localhostRisk && !Xe.has(r) && (Xe.add(r), Z.warn("runner host is not routable from other pods with SQL cluster storage — a workflow cannot resume on another pod after a reschedule. Inject POD_IP via the K8s downward API (fieldRef: status.podIP) or set runnerListenHost / VOLTRO_WORKFLOW_RUNNER_HOST.", { runnerHost: r }));
|
|
586
598
|
let a = t.dialectId === "sqlite" ? m.scoped(b.SqlClient, p.gen(function* () {
|
|
587
599
|
let e = yield* b.SqlClient, t = (yield* e.unsafe("PRAGMA database_list")).find((e) => e.name === "main");
|
|
588
|
-
return t === void 0 ? yield* p.die(/* @__PURE__ */ Error("SQLite workflow connection has no main database")) : (t.file !== "" && (yield*
|
|
589
|
-
}).pipe(p.orDie)).pipe(m.provide(t.sqlClientLayer)) : t.sqlClientLayer, s =
|
|
600
|
+
return t === void 0 ? yield* p.die(/* @__PURE__ */ Error("SQLite workflow connection has no main database")) : (t.file !== "" && (yield* Ve(t.file)), e);
|
|
601
|
+
}).pipe(p.orDie)).pipe(m.provide(t.sqlClientLayer)) : t.sqlClientLayer, s = K({
|
|
590
602
|
runnerStorage: t.runnerStorage,
|
|
591
603
|
charset: t.charset
|
|
592
|
-
}).pipe(m.provide(a)), c = t.shardLockMode ??
|
|
604
|
+
}).pipe(m.provide(a)), c = t.shardLockMode ?? $(), l = m.unwrapEffect(Ke({
|
|
593
605
|
dialectId: t.dialectId,
|
|
594
606
|
runnerStorage: t.runnerStorage,
|
|
595
607
|
mode: c
|
|
596
608
|
}).pipe(p.map((e) => {
|
|
597
|
-
t.runnerStorage === "sql" &&
|
|
609
|
+
t.runnerStorage === "sql" && Z.info("shard-lock coordination", qe({
|
|
598
610
|
disableAdvisory: e,
|
|
599
611
|
dialectId: t.dialectId,
|
|
600
612
|
mode: c
|
|
601
613
|
}));
|
|
602
|
-
let n = m.effect(o, p.flatMap(T.Sharding, (n) =>
|
|
614
|
+
let n = m.effect(o, p.flatMap(T.Sharding, (n) => X({
|
|
603
615
|
ownAddress: C.make(r, i).toString(),
|
|
604
616
|
shardLockMode: t.runnerStorage === "sql" ? e ? "row" : "advisory" : "none",
|
|
605
617
|
runnerStorage: t.runnerStorage,
|
|
@@ -612,26 +624,23 @@ var _e = (e) => p.gen(function* () {
|
|
|
612
624
|
...t.failoverLeaseSeconds === void 0 ? {} : { shardLockExpiration: f.seconds(t.failoverLeaseSeconds) },
|
|
613
625
|
...t.failoverHeartbeatSeconds === void 0 ? {} : { shardLockRefreshInterval: f.seconds(t.failoverHeartbeatSeconds) },
|
|
614
626
|
...t.messagePollSeconds === void 0 ? {} : { entityMessagePollInterval: f.seconds(t.messagePollSeconds) }
|
|
615
|
-
}, s = f.seconds(t.failoverHeartbeatSeconds ?? 10), l =
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
627
|
+
}, s = f.seconds(t.failoverHeartbeatSeconds ?? 10), l = Be({
|
|
628
|
+
runnerStorage: t.runnerStorage,
|
|
629
|
+
shardingConfig: a,
|
|
630
|
+
guard: {
|
|
619
631
|
interval: s,
|
|
620
632
|
giveUpAfter: f.times(s, 3),
|
|
621
|
-
log:
|
|
622
|
-
}
|
|
623
|
-
})).pipe(m.provide(m.orDie(D.layer))), ae.layerNoop]), m.provide(ce.layerFromEnv(a))) : le.layer({
|
|
624
|
-
runnerStorage: t.runnerStorage,
|
|
625
|
-
shardingConfig: a
|
|
633
|
+
log: Z
|
|
634
|
+
}
|
|
626
635
|
});
|
|
627
|
-
return m.mergeAll(n, m.effect(
|
|
636
|
+
return m.mergeAll(n, m.effect(j, ge), m.effect(L, Ee)).pipe(m.provideMerge(l));
|
|
628
637
|
}))).pipe(m.provide(s)), u = t.extraShardingLayers ?? [], g = u.length === 0 ? x.layer : m.mergeAll(x.layer, ...u), _ = l.pipe(m.map((t) => d.add(t, T.Sharding, e(d.get(t, T.Sharding)))));
|
|
629
638
|
return g.pipe(m.provideMerge(_));
|
|
630
|
-
},
|
|
639
|
+
}, Qe = (e) => ee.make({
|
|
631
640
|
name: e.name,
|
|
632
641
|
cron: e.cron,
|
|
633
642
|
execute: p.promise(() => e.execute()),
|
|
634
643
|
skipIfOlderThan: e.skipIfOlderThan ?? f.days(1)
|
|
635
644
|
});
|
|
636
645
|
//#endregion
|
|
637
|
-
export {
|
|
646
|
+
export { Se as _, Ye as a, We as c, X as d, K as f, xe as g, be as h, $ as i, Ke as l, M as m, Ue as n, Je as o, L as p, Ge as r, Q as s, Qe as t, Ze as u, j as v };
|