@voltro/workflow 0.84.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1215 -0
- package/THIRD-PARTY-NOTICES.md +5 -151
- package/dist/admission.js +140 -0
- package/dist/admissionDrainer.js +705 -0
- package/dist/admissionGate.js +115 -0
- package/dist/admissionStore.js +347 -0
- package/dist/admissionSubmission.js +297 -0
- package/dist/admissionTables.js +137 -0
- package/dist/admissionTransaction.js +21 -0
- package/dist/awaitEvent.js +82 -0
- package/dist/budgetHold.js +136 -0
- package/dist/cancelOnPlan.js +75 -0
- package/dist/cancelOnSweep.js +188 -0
- package/dist/cancelPendingStart.js +50 -0
- package/dist/cluster.d.ts +304 -280
- package/dist/cluster.js +8 -2
- package/dist/clusterCron.js +11 -0
- package/dist/clusterDiagnostics.js +20 -0
- package/dist/clusterDiagnosticsSql.js +145 -0
- package/dist/clusterLayer.js +110 -0
- package/dist/clusterMigrationGate.js +61 -0
- package/dist/clusterRunnerLayer.js +21 -0
- package/dist/clusterTestSuite.d.ts +11 -11
- package/dist/clusterTestSuite.js +257 -257
- package/dist/controlMonitor.js +13 -0
- package/dist/controlRequest.js +367 -0
- package/dist/executionRefusal.js +162 -0
- package/dist/failureHandoff.js +183 -0
- package/dist/failureMessage.js +12 -0
- package/dist/failureRecovery.js +50 -0
- package/dist/failureReportStore.js +60 -0
- package/dist/flowControl.js +142 -0
- package/dist/forwardStartReceipts.js +41 -0
- package/dist/inMemoryLayer.js +7 -0
- package/dist/inMemoryRecorder.js +56 -0
- package/dist/index.d.ts +456 -360
- package/dist/index.js +49 -5
- package/dist/inspectWorkflow.js +101 -0
- package/dist/journalEncryption.js +147 -0
- package/dist/messages.js +92 -0
- package/dist/nondeterminism.js +51 -0
- package/dist/parentClosure.js +141 -0
- package/dist/parentClosureRecovery.js +54 -0
- package/dist/pendingSubmission.js +85 -0
- package/dist/prepareSubmissionDispatch.js +17 -0
- package/dist/preparedExecution.js +14 -0
- package/dist/primitives.d.ts +56 -31
- package/dist/primitives.js +225 -2
- package/dist/queuedCancellationRequest.js +364 -0
- package/dist/queuedControl.js +21 -0
- package/dist/recorder.js +11 -0
- package/dist/residentExecutionEvidence.js +64 -0
- package/dist/retryPolicy.js +57 -0
- package/dist/runEventsTable.js +17 -0
- package/dist/runGeneration.js +345 -0
- package/dist/runRecording.js +615 -0
- package/dist/runStepsTable.js +28 -0
- package/dist/runnerStorageGuard.js +29 -0
- package/dist/runsTable.js +42 -0
- package/dist/scheduleDeclaration.js +44 -0
- package/dist/signalHint.js +23 -0
- package/dist/signals.js +70 -0
- package/dist/signalsSuspending.js +57 -0
- package/dist/sqliteWorkflowOwnership.js +26 -0
- package/dist/stalenessSweep.js +191 -0
- package/dist/startContextsTable.js +20 -0
- package/dist/startReceipt.js +143 -0
- package/dist/submissionRecovery.js +122 -0
- package/dist/submissionRejection.js +10 -0
- package/dist/submissionRetry.js +71 -0
- package/dist/suspensionUnavailable.js +9 -0
- package/dist/updates.js +87 -0
- package/dist/watermarkTable.js +9 -0
- package/dist/webCrypto.js +17 -0
- package/dist/workflowDelivery.js +24 -0
- package/dist/workflowJournal.js +124 -0
- package/dist/workflowJournalReset.js +111 -0
- package/dist/workflowReadSelections.js +12 -0
- package/dist/workflowRestartDispatch.js +29 -0
- package/dist/workflowRestartObservation.js +60 -0
- package/dist/workflowRestartPreparation.js +109 -0
- package/dist/workflowRestartReservation.js +426 -0
- package/dist/workflowResumeDispatch.js +29 -0
- package/package.json +6 -9
- package/dist/cluster-i3frC3Da.js +0 -646
- package/dist/clusterDiagnostics-CGUtnJkX.js +0 -2084
- package/dist/journalEncryption--n5VDdYO.js +0 -148
- package/dist/primitives-Btjhjm8P.js +0 -483
- package/dist/src-DP-lKqaG.js +0 -3867
package/dist/cluster.js
CHANGED
|
@@ -1,2 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import { WorkflowResumeDispatcher as e } from "./workflowResumeDispatch.js";
|
|
2
|
+
import { JournalExit as t, activityPrimaryKey as n, readWorkflowRunReply as r, redriveFailedRun as i, resumeRunFromStep as a } from "./workflowJournal.js";
|
|
3
|
+
import { WorkflowRestartDispatcher as o } from "./workflowRestartDispatch.js";
|
|
4
|
+
import { clusterMigrationGateLayer as s } from "./clusterMigrationGate.js";
|
|
5
|
+
import { clusterLockNamespaceOid as c, makeClusterDiagnostics as l } from "./clusterDiagnosticsSql.js";
|
|
6
|
+
import { DEFAULT_RUNNER_LISTEN_PORT as u, detectWsrepCluster as d, parseShardLockMode as f, pollWorkflowStorage as p, resolveFailoverTuning as m, resolveRunnerIdentity as h, resolveRunnerStorage as g, resolveShardLockDisableAdvisory as _, workflowEngineLayer as v } from "./clusterLayer.js";
|
|
7
|
+
import { makeClusterCronLayer as y } from "./clusterCron.js";
|
|
8
|
+
export { u as DEFAULT_RUNNER_LISTEN_PORT, t as JournalExit, o as WorkflowRestartDispatcher, e as WorkflowResumeDispatcher, n as activityPrimaryKey, c as clusterLockNamespaceOid, s as clusterMigrationGateLayer, d as detectWsrepCluster, y as makeClusterCronLayer, l as makeClusterDiagnostics, f as parseShardLockMode, p as pollWorkflowStorage, r as readWorkflowRunReply, i as redriveFailedRun, m as resolveFailoverTuning, h as resolveRunnerIdentity, g as resolveRunnerStorage, _ as resolveShardLockDisableAdvisory, a as resumeRunFromStep, v as workflowEngineLayer };
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Duration as e, Effect as t } from "effect";
|
|
2
|
+
import { ClusterCron as n } from "effect/cluster";
|
|
3
|
+
//#region src/clusterCron.ts
|
|
4
|
+
var r = (r) => n.make({
|
|
5
|
+
name: r.name,
|
|
6
|
+
cron: r.cron,
|
|
7
|
+
execute: t.promise(() => r.execute()),
|
|
8
|
+
skipIfOlderThan: r.skipIfOlderThan ?? e.days(1)
|
|
9
|
+
});
|
|
10
|
+
//#endregion
|
|
11
|
+
export { r as makeClusterCronLayer };
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Context as e, Effect as t } from "effect";
|
|
2
|
+
//#region src/clusterDiagnostics.ts
|
|
3
|
+
var n = 6e5, r = class extends e.Service()("@voltro/workflow/ClusterDiagnostics") {}, i = (e, n, r = Date.now()) => t.flatMap(e.messagesFor(n), (n) => {
|
|
4
|
+
if (n.length === 0) return t.succeed(void 0);
|
|
5
|
+
let i = [...new Set(n.filter((e) => !e.processed).map((e) => e.shardId))], a = i.length === 0 ? t.succeed([]) : e.shardLockHolders(i);
|
|
6
|
+
return t.map(a, (t) => ({
|
|
7
|
+
runner: e.ownAddress,
|
|
8
|
+
shardLockMode: e.shardLockMode,
|
|
9
|
+
messages: n.map((e) => ({
|
|
10
|
+
tag: e.tag,
|
|
11
|
+
shardId: e.shardId,
|
|
12
|
+
processed: e.processed,
|
|
13
|
+
lastRead: e.lastRead === null ? null : e.lastRead.toISOString(),
|
|
14
|
+
unreadForMs: !e.processed && e.lastRead !== null ? Math.max(0, r - e.lastRead.getTime()) : null
|
|
15
|
+
})),
|
|
16
|
+
shardLocks: t
|
|
17
|
+
}));
|
|
18
|
+
});
|
|
19
|
+
//#endregion
|
|
20
|
+
export { n as CLUSTER_REREAD_WINDOW_MS, r as ClusterDiagnostics, i as describeStalledRun };
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { CLUSTER_REREAD_WINDOW_MS as e } from "./clusterDiagnostics.js";
|
|
2
|
+
import { Effect as t } from "effect";
|
|
3
|
+
import { SqlClient as n } from "effect/sql";
|
|
4
|
+
//#region src/clusterDiagnosticsSql.ts
|
|
5
|
+
var r = (e) => {
|
|
6
|
+
if (e instanceof Date) return Number.isNaN(e.getTime()) ? null : e;
|
|
7
|
+
if (typeof e == "string" || typeof e == "number") {
|
|
8
|
+
let t = typeof e == "string" && /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(e) ? `${e.replace(" ", "T")}Z` : e, n = new Date(t);
|
|
9
|
+
return Number.isNaN(n.getTime()) ? null : n;
|
|
10
|
+
}
|
|
11
|
+
return null;
|
|
12
|
+
}, i = (e) => e === !0 || e === 1 || e === "1" || e instanceof Uint8Array && e[0] === 1, a = "cluster", o = (e = a) => {
|
|
13
|
+
let t = new TextEncoder().encode(`effect-cluster:${e}`), n = 2166136261;
|
|
14
|
+
for (let e of t) n = Math.imul(n ^ e, 16777619);
|
|
15
|
+
return (n | 0) >>> 0;
|
|
16
|
+
}, s = (e, t) => {
|
|
17
|
+
let n = e.lastIndexOf(":");
|
|
18
|
+
if (n < 0) return;
|
|
19
|
+
let r = e.slice(0, n), i = Number(e.slice(n + 1)), a = t.indexOf(r);
|
|
20
|
+
if (!(a < 0 || !Number.isInteger(i))) return (a + 1) * 1e6 + i;
|
|
21
|
+
}, c = (a) => t.gen(function* () {
|
|
22
|
+
let c = yield* n.SqlClient, l = a.shardGroups ?? ["default"], u = a.dialectId ?? "", d = u.startsWith("postgres") || u === "pg", f = (e) => e.pipe(t.catch(() => t.succeed([]))), p = c.onDialectOrElse({
|
|
23
|
+
pg: () => c.literal("NOW()"),
|
|
24
|
+
mysql: () => c.literal("NOW()"),
|
|
25
|
+
mssql: () => c.literal("GETDATE()"),
|
|
26
|
+
orElse: () => c.literal("CURRENT_TIMESTAMP")
|
|
27
|
+
}), m = (e, t) => {
|
|
28
|
+
let n = r(e.last_read), a = r(e.server_now), o = n !== null && a !== null ? Math.max(0, a.getTime() - n.getTime()) : null;
|
|
29
|
+
return {
|
|
30
|
+
tag: typeof e.tag == "string" ? e.tag : null,
|
|
31
|
+
entityType: String(e.entity_type ?? ""),
|
|
32
|
+
entityId: String(e.entity_id ?? ""),
|
|
33
|
+
shardId: String(e.shard_id ?? ""),
|
|
34
|
+
processed: i(e.processed),
|
|
35
|
+
lastRead: o === null ? null : new Date(t - o)
|
|
36
|
+
};
|
|
37
|
+
}, h = (e) => c.onDialectOrElse({
|
|
38
|
+
pg: () => c.literal(`NOW() - INTERVAL '${e} seconds'`),
|
|
39
|
+
mysql: () => c.literal(`NOW() - INTERVAL ${e} SECOND`),
|
|
40
|
+
mssql: () => c.literal(`DATEADD(SECOND, -${e}, GETDATE())`),
|
|
41
|
+
orElse: () => c.literal(`DATETIME('now', '-${e} seconds')`)
|
|
42
|
+
}), g = c.onDialectOrElse({
|
|
43
|
+
pg: () => c.literal("FALSE"),
|
|
44
|
+
orElse: () => c.literal("0")
|
|
45
|
+
}), _ = (e) => {
|
|
46
|
+
let n = Date.now();
|
|
47
|
+
return f(c`
|
|
48
|
+
SELECT tag, entity_type, entity_id, shard_id, processed, last_read, ${p} AS server_now
|
|
49
|
+
FROM cluster_messages
|
|
50
|
+
WHERE entity_id = ${e}
|
|
51
|
+
ORDER BY rowid ASC
|
|
52
|
+
`).pipe(t.map((e) => e.map((e) => m(e, n))));
|
|
53
|
+
}, v = (n = e) => {
|
|
54
|
+
let r = Math.max(1, Math.ceil(n / 1e3)), i = Date.now();
|
|
55
|
+
return f(c`
|
|
56
|
+
SELECT tag, entity_type, entity_id, shard_id, processed, last_read, ${p} AS server_now
|
|
57
|
+
FROM cluster_messages
|
|
58
|
+
WHERE processed = ${g}
|
|
59
|
+
AND last_read IS NOT NULL
|
|
60
|
+
AND last_read < ${h(r)}
|
|
61
|
+
AND (deliver_at IS NULL OR deliver_at <= ${i})
|
|
62
|
+
ORDER BY last_read ASC
|
|
63
|
+
`).pipe(t.map((e) => e.flatMap((e) => {
|
|
64
|
+
let t = m(e, i);
|
|
65
|
+
return t.lastRead === null ? [] : [{
|
|
66
|
+
...t,
|
|
67
|
+
unreadForMs: Math.max(0, i - t.lastRead.getTime())
|
|
68
|
+
}];
|
|
69
|
+
})));
|
|
70
|
+
}, y = (e) => f(c`
|
|
71
|
+
SELECT shard_id, address, acquired_at FROM cluster_locks WHERE ${c.in("shard_id", e)}
|
|
72
|
+
`).pipe(t.map((e) => e.map((e) => {
|
|
73
|
+
let t = e, n = String(t.address ?? ""), i = String(t.shard_id ?? "");
|
|
74
|
+
return {
|
|
75
|
+
shardId: i,
|
|
76
|
+
mode: "row",
|
|
77
|
+
holder: n,
|
|
78
|
+
own: n === a.ownAddress || (a.hasShard?.(i) ?? !1),
|
|
79
|
+
since: r(t.acquired_at)
|
|
80
|
+
};
|
|
81
|
+
}))), b = (e) => {
|
|
82
|
+
let n = /* @__PURE__ */ new Map();
|
|
83
|
+
for (let t of e) {
|
|
84
|
+
let e = s(t, l);
|
|
85
|
+
e !== void 0 && n.set(e, t);
|
|
86
|
+
}
|
|
87
|
+
return n.size === 0 ? t.succeed([]) : t.gen(function* () {
|
|
88
|
+
return (yield* f(c`
|
|
89
|
+
SELECT l.objid, a.pid, a.client_addr, a.application_name, a.state, a.state_change, a.backend_start, a.wait_event_type
|
|
90
|
+
FROM pg_locks l
|
|
91
|
+
JOIN pg_stat_activity a ON a.pid = l.pid
|
|
92
|
+
WHERE l.locktype = 'advisory' AND l.granted AND l.objsubid = 2
|
|
93
|
+
AND l.classid = ${o()} AND ${c.in("l.objid", [...n.keys()])}
|
|
94
|
+
`)).flatMap((e) => {
|
|
95
|
+
let t = e, i = n.get(Number(t.objid));
|
|
96
|
+
if (i === void 0) return [];
|
|
97
|
+
let o = Number(t.pid), s = t.client_addr === null || t.client_addr === void 0 ? "local socket" : String(t.client_addr), c = a.hasShard?.(i) ?? !1;
|
|
98
|
+
return [{
|
|
99
|
+
shardId: i,
|
|
100
|
+
mode: "advisory",
|
|
101
|
+
holder: c ? `this runner (pid ${o})` : `pid ${o} from ${s}`,
|
|
102
|
+
own: c,
|
|
103
|
+
since: r(t.state_change),
|
|
104
|
+
detail: {
|
|
105
|
+
pid: o,
|
|
106
|
+
clientAddr: t.client_addr ?? null,
|
|
107
|
+
applicationName: t.application_name ?? null,
|
|
108
|
+
state: t.state ?? null,
|
|
109
|
+
waitEventType: t.wait_event_type ?? null,
|
|
110
|
+
backendStart: r(t.backend_start)?.toISOString() ?? null
|
|
111
|
+
}
|
|
112
|
+
}];
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
};
|
|
116
|
+
return {
|
|
117
|
+
ownAddress: a.ownAddress,
|
|
118
|
+
shardLockMode: a.shardLockMode,
|
|
119
|
+
messagesFor: _,
|
|
120
|
+
unreadMessages: v,
|
|
121
|
+
shardLockHolders: (e) => e.length === 0 || a.runnerStorage !== "sql" ? t.succeed([]) : a.shardLockMode === "row" ? y(e) : a.shardLockMode === "advisory" && d ? b(e) : t.succeed([]),
|
|
122
|
+
runners: () => a.runnerStorage === "sql" ? f(c`SELECT address, last_heartbeat, healthy FROM cluster_runners ORDER BY last_heartbeat DESC`).pipe(t.map((e) => e.map((e) => {
|
|
123
|
+
let t = e;
|
|
124
|
+
return {
|
|
125
|
+
address: String(t.address ?? ""),
|
|
126
|
+
lastHeartbeat: r(t.last_heartbeat),
|
|
127
|
+
healthy: i(t.healthy)
|
|
128
|
+
};
|
|
129
|
+
}))) : t.succeed([]),
|
|
130
|
+
discardMessages: (e) => {
|
|
131
|
+
let r = e.unreadForMs === void 0 ? c.literal("") : c`AND last_read IS NOT NULL AND last_read < ${h(Math.max(1, Math.ceil(e.unreadForMs / 1e3)))}`;
|
|
132
|
+
return t.gen(function* () {
|
|
133
|
+
let t = (yield* c`
|
|
134
|
+
SELECT id FROM cluster_messages
|
|
135
|
+
WHERE entity_type = ${e.entityType} AND entity_id = ${e.entityId}
|
|
136
|
+
AND processed = ${g}
|
|
137
|
+
${r}
|
|
138
|
+
`).map((e) => e.id).filter((e) => e != null).map(String);
|
|
139
|
+
return t.length === 0 ? 0 : (yield* c`DELETE FROM cluster_replies WHERE ${c.in("request_id", t)}`, yield* c`DELETE FROM cluster_messages WHERE ${c.in("id", t)}`, t.length);
|
|
140
|
+
}).pipe(c.withTransaction, t.provideService(n.SafeIntegers, !0), t.catch(() => t.succeed(0)));
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
});
|
|
144
|
+
//#endregion
|
|
145
|
+
export { o as clusterLockNamespaceOid, c as makeClusterDiagnostics };
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { withWorkflowDelivery as e } from "./workflowDelivery.js";
|
|
2
|
+
import { ClusterDiagnostics as t } from "./clusterDiagnostics.js";
|
|
3
|
+
import { WorkflowResumeDispatcher as n, makeWorkflowResumeDispatcher as r } from "./workflowResumeDispatch.js";
|
|
4
|
+
import { WorkflowRestartDispatcher as i, makeWorkflowRestartDispatcher as a } from "./workflowRestartDispatch.js";
|
|
5
|
+
import { clusterMigrationGateLayer as o } from "./clusterMigrationGate.js";
|
|
6
|
+
import { clusterRunnerLayer as s } from "./clusterRunnerLayer.js";
|
|
7
|
+
import { acquireSqliteWorkflowOwnership as c } from "./sqliteWorkflowOwnership.js";
|
|
8
|
+
import { makeClusterDiagnostics as l } from "./clusterDiagnosticsSql.js";
|
|
9
|
+
import { RateLimiter as u } from "effect/persistence";
|
|
10
|
+
import { Context as d, Duration as f, Effect as p, Layer as m, Option as h } from "effect";
|
|
11
|
+
import { SqlClient as g } from "effect/sql";
|
|
12
|
+
import { ClusterWorkflowEngine as _, MessageStorage as v, RunnerAddress as y, ShardId as b, Sharding as x } from "effect/cluster";
|
|
13
|
+
import { createLogger as S } from "@voltro/logger";
|
|
14
|
+
//#region src/clusterLayer.ts
|
|
15
|
+
var C = S({ scope: "voltro:workflow" }), w = 34e3, T = (e) => {
|
|
16
|
+
let t = e.runnerListenHost ?? process.env.POD_IP ?? process.env.VOLTRO_WORKFLOW_RUNNER_HOST ?? "localhost";
|
|
17
|
+
return {
|
|
18
|
+
host: t,
|
|
19
|
+
port: e.runnerListenPort ?? 34e3,
|
|
20
|
+
localhostRisk: e.runnerStorage === "sql" && (t === "localhost" || t === "127.0.0.1")
|
|
21
|
+
};
|
|
22
|
+
}, E = (e) => {
|
|
23
|
+
let t = e.sqlRunnerSupported ?? !e.sqliteFamily, n = t ? "sql" : "memory", r = e.override ?? process.env.VOLTRO_WORKFLOW_RUNNER_STORAGE;
|
|
24
|
+
if (r === void 0 || r.trim() === "") return n;
|
|
25
|
+
let i = r.trim().toLowerCase();
|
|
26
|
+
if (i !== "memory" && i !== "sql") throw Error(`VOLTRO_WORKFLOW_RUNNER_STORAGE must be 'memory' or 'sql', got '${r}'.`);
|
|
27
|
+
if (i === "sql" && !t) throw Error("VOLTRO_WORKFLOW_RUNNER_STORAGE='sql' is unsupported for this database connection. Use 'memory' for a local or embedded SQLite-family connection.");
|
|
28
|
+
return i;
|
|
29
|
+
}, D = (e = process.env.VOLTRO_WORKFLOW_SHARD_LOCK) => {
|
|
30
|
+
if (e === void 0 || e.trim() === "") return "auto";
|
|
31
|
+
let t = e.trim().toLowerCase();
|
|
32
|
+
if (t === "auto" || t === "row" || t === "advisory") return t;
|
|
33
|
+
throw Error(`VOLTRO_WORKFLOW_SHARD_LOCK must be 'auto', 'row', or 'advisory', got '${e}'.`);
|
|
34
|
+
}, O = p.gen(function* () {
|
|
35
|
+
let e = yield* (yield* g.SqlClient)`SELECT @@wsrep_on AS wsrep`.pipe(p.catch(() => p.succeed([])));
|
|
36
|
+
if (e.length === 0) return !1;
|
|
37
|
+
let t = e[0].wsrep;
|
|
38
|
+
return t === 1 || t === "1" || t === !0 || t === "ON" || t === "on";
|
|
39
|
+
}), k = (e) => p.gen(function* () {
|
|
40
|
+
return e.runnerStorage === "memory" ? !1 : e.dialectId === "sqlite" || e.dialectId === "turso" || e.mode === "row" ? !0 : e.mode === "advisory" || e.dialectId !== "mysql" && e.dialectId !== "mariadb" ? !1 : yield* O;
|
|
41
|
+
}), A = (e) => ({
|
|
42
|
+
coordination: e.disableAdvisory ? "row-based" : "advisory",
|
|
43
|
+
dialect: e.dialectId ?? "unknown",
|
|
44
|
+
mode: e.mode,
|
|
45
|
+
...e.disableAdvisory && e.mode === "auto" ? { reason: e.dialectId === "sqlite" || e.dialectId === "turso" ? "SQLite-family connection uses row locks" : "wsrep/Galera cluster detected" } : {}
|
|
46
|
+
}), j = () => {
|
|
47
|
+
let e = (e) => {
|
|
48
|
+
if (e === void 0) return;
|
|
49
|
+
let t = Number(e);
|
|
50
|
+
return Number.isFinite(t) && t > 0 ? t : void 0;
|
|
51
|
+
}, t = e(process.env.VOLTRO_WORKFLOW_FAILOVER_LEASE), n = e(process.env.VOLTRO_WORKFLOW_FAILOVER_HEARTBEAT), r = e(process.env.VOLTRO_WORKFLOW_POLL_INTERVAL);
|
|
52
|
+
return {
|
|
53
|
+
...t === void 0 ? {} : { failoverLeaseSeconds: t },
|
|
54
|
+
...n === void 0 ? {} : { failoverHeartbeatSeconds: n },
|
|
55
|
+
...r === void 0 ? {} : { messagePollSeconds: r }
|
|
56
|
+
};
|
|
57
|
+
}, M = p.flatMap(x.Sharding, (e) => e.pollStorage), N = /* @__PURE__ */ new Set(), P = (e) => Object.assign(Object.create(Object.getPrototypeOf(e)), e, { requestIdForPrimaryKey: (t) => e.requestIdForPrimaryKey(t).pipe(p.provideService(g.SafeIntegers, !0)) }), F = (S) => {
|
|
58
|
+
let w = T({
|
|
59
|
+
runnerStorage: S.runnerStorage,
|
|
60
|
+
runnerListenHost: S.runnerListenHost,
|
|
61
|
+
runnerListenPort: S.runnerListenPort
|
|
62
|
+
}), E = w.host, O = w.port;
|
|
63
|
+
w.localhostRisk && !N.has(E) && (N.add(E), C.warn("runner host is not routable from other pods with SQL cluster storage — a workflow cannot resume on another pod after a reschedule. Inject POD_IP via the K8s downward API (fieldRef: status.podIP) or set runnerListenHost / VOLTRO_WORKFLOW_RUNNER_HOST.", { runnerHost: E }));
|
|
64
|
+
let j = S.dialectId === "sqlite" ? m.effect(g.SqlClient, p.gen(function* () {
|
|
65
|
+
let e = yield* g.SqlClient, t = (yield* e.unsafe("PRAGMA database_list")).find((e) => e.name === "main");
|
|
66
|
+
return t === void 0 ? yield* p.die(/* @__PURE__ */ Error("SQLite workflow connection has no main database")) : (t.file !== "" && (yield* c(t.file)), e);
|
|
67
|
+
}).pipe(p.orDie)).pipe(m.provide(S.sqlClientLayer)) : S.sqlClientLayer, M = o({
|
|
68
|
+
runnerStorage: S.runnerStorage,
|
|
69
|
+
charset: S.charset
|
|
70
|
+
}).pipe(m.provide(j)), F = S.shardLockMode ?? D(), I = m.unwrap(k({
|
|
71
|
+
dialectId: S.dialectId,
|
|
72
|
+
runnerStorage: S.runnerStorage,
|
|
73
|
+
mode: F
|
|
74
|
+
}).pipe(p.map((e) => {
|
|
75
|
+
S.runnerStorage === "sql" && C.info("shard-lock coordination", A({
|
|
76
|
+
disableAdvisory: e,
|
|
77
|
+
dialectId: S.dialectId,
|
|
78
|
+
mode: F
|
|
79
|
+
}));
|
|
80
|
+
let o = m.effect(t, p.flatMap(x.Sharding, (t) => l({
|
|
81
|
+
ownAddress: y.make(E, O).toString(),
|
|
82
|
+
shardLockMode: S.runnerStorage === "sql" ? e ? "row" : "advisory" : "none",
|
|
83
|
+
runnerStorage: S.runnerStorage,
|
|
84
|
+
dialectId: S.dialectId,
|
|
85
|
+
hasShard: (e) => t.hasShardId(b.fromString(e))
|
|
86
|
+
}))), c = {
|
|
87
|
+
runnerAddress: h.some(y.make(E, O)),
|
|
88
|
+
runnerListenAddress: h.some(y.make(E, O)),
|
|
89
|
+
...e ? { shardLockDisableAdvisory: !0 } : {},
|
|
90
|
+
...S.failoverLeaseSeconds === void 0 ? {} : { shardLockExpiration: f.seconds(S.failoverLeaseSeconds) },
|
|
91
|
+
...S.failoverHeartbeatSeconds === void 0 ? {} : { shardLockRefreshInterval: f.seconds(S.failoverHeartbeatSeconds) },
|
|
92
|
+
...S.messagePollSeconds === void 0 ? {} : { entityMessagePollInterval: f.seconds(S.messagePollSeconds) }
|
|
93
|
+
}, u = f.seconds(S.failoverHeartbeatSeconds ?? 10), d = s({
|
|
94
|
+
runnerStorage: S.runnerStorage,
|
|
95
|
+
shardingConfig: c,
|
|
96
|
+
guard: {
|
|
97
|
+
interval: u,
|
|
98
|
+
giveUpAfter: f.times(u, 3),
|
|
99
|
+
log: C
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
return m.mergeAll(o, m.effect(n, r), m.effect(i, a)).pipe(m.provideMerge(d));
|
|
103
|
+
}))).pipe(m.provide(M)), L = S.extraShardingLayers ?? [], R = L.length === 0 ? _.layer : m.mergeAll(_.layer, ...L), z = m.flatMap(I, (t) => {
|
|
104
|
+
let n = d.add(t, x.Sharding, e(d.get(t, x.Sharding)));
|
|
105
|
+
return m.succeedContext(S.dialectId === "sqlite" ? d.add(n, v.MessageStorage, P(d.get(n, v.MessageStorage))) : n);
|
|
106
|
+
}), B = u.layer.pipe(m.provide(u.layerStoreMemory));
|
|
107
|
+
return m.merge(R, B).pipe(m.provideMerge(z));
|
|
108
|
+
};
|
|
109
|
+
//#endregion
|
|
110
|
+
export { w as DEFAULT_RUNNER_LISTEN_PORT, O as detectWsrepCluster, D as parseShardLockMode, M as pollWorkflowStorage, j as resolveFailoverTuning, T as resolveRunnerIdentity, E as resolveRunnerStorage, k as resolveShardLockDisableAdvisory, A as shardLockLogFields, F as workflowEngineLayer };
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { webCryptoLayer as e } from "./webCrypto.js";
|
|
2
|
+
import { Effect as t, Layer as n } from "effect";
|
|
3
|
+
import { SqlClient as r } from "effect/sql";
|
|
4
|
+
import { ShardingConfig as i, SqlMessageStorage as a, SqlRunnerStorage as o } from "effect/cluster";
|
|
5
|
+
import { applyUndeclaredCharsetWork as s, fnv1a64 as c, readUndeclaredCharsetColumns as l, resolveMigrationLockSchema as u, sqlPoolCapacity as d, undeclaredCharsetWork as f } from "@voltro/database/sql";
|
|
6
|
+
//#region src/clusterMigrationGate.ts
|
|
7
|
+
var p = 6322741009312601n, m = () => {
|
|
8
|
+
let e = u();
|
|
9
|
+
return e === void 0 ? p : c(`voltro_cluster_migration_lock:${e}`) & 9223372036854775807n;
|
|
10
|
+
}, h = () => {
|
|
11
|
+
let e = `voltro_cluster_migration_${p.toString(36)}`, t = u();
|
|
12
|
+
if (t === void 0) return e;
|
|
13
|
+
let n = `${e}_${t}`;
|
|
14
|
+
return /^[A-Za-z0-9_]+$/.test(t) && n.length <= 64 ? n : `${e}_h${c(`voltro_cluster_migration_lock:${t}`).toString(16)}`;
|
|
15
|
+
}, g = () => {
|
|
16
|
+
let e = Number(process.env.VOLTRO_CLUSTER_MIGRATION_LOCK_TIMEOUT_MS);
|
|
17
|
+
return Number.isFinite(e) && e > 0 ? e : 6e4;
|
|
18
|
+
}, _ = 250, v = (e) => e[0]?.[0], y = (e, n) => e.onDialectOrElse({
|
|
19
|
+
sqlite: () => t.void,
|
|
20
|
+
mysql: () => n.executeValues(`SELECT GET_LOCK('${h()}', ${Math.ceil(g() / 1e3)})`, []).pipe(t.flatMap((e) => v(e) === 1 || v(e) === "1" ? t.void : t.die(/* @__PURE__ */ Error(`could not acquire cluster migration lock '${h()}' (GET_LOCK timed out)`)))),
|
|
21
|
+
mssql: () => n.executeValues(`DECLARE @res int; EXEC @res = sp_getapplock @Resource = '${h()}', @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = ${g()}; SELECT @res`, []).pipe(t.flatMap((e) => Number(v(e) ?? -999) >= 0 ? t.void : t.die(/* @__PURE__ */ Error(`could not acquire cluster migration lock '${h()}' (sp_getapplock ${String(v(e))})`)))),
|
|
22
|
+
orElse: () => {
|
|
23
|
+
let e = m().toString(), r = Date.now(), i = n.executeValues(`SELECT pg_try_advisory_lock(${e}::bigint)`, []).pipe(t.flatMap((e) => {
|
|
24
|
+
let n = v(e);
|
|
25
|
+
return n === !0 || n === "t" || n === 1 ? t.void : Date.now() - r >= g() ? t.die(/* @__PURE__ */ Error(`could not acquire the cluster migration advisory lock within ${Math.round(g() / 1e3)}s. Another runner is migrating the cluster schema, or a prior boot wedged while holding it. (Raise VOLTRO_CLUSTER_MIGRATION_LOCK_TIMEOUT_MS, or inspect pg_locks for the holder.)`)) : t.sleep(`${_} millis`).pipe(t.andThen(i));
|
|
26
|
+
}));
|
|
27
|
+
return i;
|
|
28
|
+
}
|
|
29
|
+
}), b = (e, n) => e.onDialectOrElse({
|
|
30
|
+
sqlite: () => t.void,
|
|
31
|
+
mysql: () => t.asVoid(n.executeValues(`SELECT RELEASE_LOCK('${h()}')`, [])),
|
|
32
|
+
mssql: () => t.asVoid(n.executeValues(`EXEC sp_releaseapplock @Resource = '${h()}', @LockOwner = 'Session'`, [])),
|
|
33
|
+
orElse: () => t.asVoid(n.executeValues(`SELECT pg_advisory_unlock(${m().toString()}::bigint)`, []))
|
|
34
|
+
}), x = (e, n) => e.onDialectOrElse({
|
|
35
|
+
sqlite: () => !0,
|
|
36
|
+
orElse: () => !1
|
|
37
|
+
}) ? n : t.scoped(t.gen(function* () {
|
|
38
|
+
let r = yield* e.reserve;
|
|
39
|
+
return yield* t.acquireRelease(y(e, r), () => b(e, r).pipe(t.orDie)), yield* n;
|
|
40
|
+
})), S = (e) => e === "sql" ? ["cluster_messages", "cluster_runners"] : ["cluster_messages"], C = (e, n) => e.onDialectOrElse({
|
|
41
|
+
pg: () => e`SELECT (to_regclass(${n}) IS NOT NULL) AS present`,
|
|
42
|
+
mysql: () => e`SELECT (COUNT(*) > 0) AS present FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ${n}`,
|
|
43
|
+
mssql: () => e`SELECT CASE WHEN OBJECT_ID(${n}, 'U') IS NULL THEN 0 ELSE 1 END AS present`,
|
|
44
|
+
orElse: () => e`SELECT (COUNT(*) > 0) AS present FROM sqlite_master WHERE type = 'table' AND name = ${n}`
|
|
45
|
+
}).pipe(t.map((e) => {
|
|
46
|
+
let t = e[0]?.present;
|
|
47
|
+
return t === !0 || t === 1 || t === "1" || t === "t" || typeof t == "bigint" && t === 1n;
|
|
48
|
+
}), t.catch(() => t.succeed(!1))), w = (e, n) => t.forEach(S(n), (t) => C(e, t), { concurrency: 1 }).pipe(t.map((e) => e.every(Boolean))), T = (a, o) => t.scoped(t.asVoid(n.build(o.pipe(n.provide(i.layerDefaults), n.provide(n.succeed(r.SqlClient, a)), n.provide(e))))), E = (e, n) => T(e, a.layer).pipe(t.andThen(n === "sql" ? T(e, o.layer) : t.void)), D = (e) => t.map(e.unsafe("SELECT table_name AS t FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name LIKE 'cluster\\_%'"), (e) => new Set(e.map((e) => String(e.t ?? e.T)))), O = (e, n, r) => r === void 0 || !e.onDialectOrElse({
|
|
49
|
+
mysql: () => !0,
|
|
50
|
+
orElse: () => !1
|
|
51
|
+
}) ? E(e, n) : t.gen(function* () {
|
|
52
|
+
let t = yield* D(e);
|
|
53
|
+
yield* E(e, n);
|
|
54
|
+
let i = new Set([...yield* D(e)].filter((e) => !t.has(e)));
|
|
55
|
+
i.size !== 0 && (yield* s(e, f(yield* l(e), /* @__PURE__ */ new Set(), r, i)));
|
|
56
|
+
}), k = (e) => e === "sql" ? 3 : 2, A = (e) => n.effect(r.SqlClient, t.gen(function* () {
|
|
57
|
+
let n = yield* r.SqlClient, i = k(e.runnerStorage), a = d(n);
|
|
58
|
+
return a !== void 0 && a < i ? yield* t.die(/* @__PURE__ */ Error(`the workflow runtime needs ${i} database connections at once and the pool has ${a}: ` + (e.runnerStorage === "sql" ? "on a first boot it holds its migration lock and its runner lock while the migration runs on a third, and once running it keeps the runner lock and needs one more to work. " : "on a first boot it holds its migration lock while the migration runs on a second. ") + `Set DB_MAX_CONNECTIONS (or PG_MAX_CONNECTIONS) to at least ${i}` + (e.runnerStorage === "sql" ? ", or VOLTRO_WORKFLOW_RUNNER_STORAGE=memory for a single-process runtime that needs 2 (no cross-pod shard handoff)." : "."))) : ((yield* w(n, e.runnerStorage)) || (yield* x(n, O(n, e.runnerStorage, e.charset))), n);
|
|
59
|
+
}));
|
|
60
|
+
//#endregion
|
|
61
|
+
export { k as clusterConnectionFloor, A as clusterMigrationGateLayer, C as tableExists, x as withClusterMigrationLock };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { webCryptoLayer as e } from "./webCrypto.js";
|
|
2
|
+
import { guardRunnerStorage as t } from "./runnerStorageGuard.js";
|
|
3
|
+
import { Effect as n, Layer as r, Semaphore as i } from "effect";
|
|
4
|
+
import { SqlClient as a } from "effect/sql";
|
|
5
|
+
import { RunnerHealth as o, RunnerStorage as s, Runners as c, Sharding as l, ShardingConfig as u, SqlMessageStorage as d, SqlRunnerStorage as f } from "effect/cluster";
|
|
6
|
+
//#region src/clusterRunnerLayer.ts
|
|
7
|
+
var p = (e) => n.map(i.make(1), (t) => ({
|
|
8
|
+
...e,
|
|
9
|
+
release: (r, i) => t.withPermits(1)(n.suspend(() => e.release(r, i)))
|
|
10
|
+
})), m = r.unwrap(n.map(a.SqlClient, (e) => e.onDialectOrElse({
|
|
11
|
+
sqlite: () => r.effect(s.RunnerStorage, n.flatMap(s.RunnerStorage, p)).pipe(r.provide(f.layer)),
|
|
12
|
+
orElse: () => f.layer
|
|
13
|
+
}))), h = (e) => r.effect(s.RunnerStorage, n.gen(function* () {
|
|
14
|
+
let n = yield* a.SqlClient, r = yield* s.RunnerStorage;
|
|
15
|
+
return yield* t(r, {
|
|
16
|
+
...e,
|
|
17
|
+
probe: n`SELECT 1`
|
|
18
|
+
});
|
|
19
|
+
})).pipe(r.provide(r.orDie(m))), g = (t) => l.layer.pipe(r.provideMerge(c.layerNoop), r.provideMerge(d.layer), r.provide([t.runnerStorage === "memory" ? s.layerMemory : h(t.guard), o.layerNoop]), r.provide(u.layerFromEnv(t.shardingConfig)), r.provide(e));
|
|
20
|
+
//#endregion
|
|
21
|
+
export { g as clusterRunnerLayer, p as limitSqliteShardReleases };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Config } from 'effect';
|
|
2
2
|
import { Layer } from 'effect';
|
|
3
3
|
import { MysqlCharset } from '@voltro/database/sql';
|
|
4
|
-
import { Sharding } from '
|
|
5
|
-
import { SqlClient } from '
|
|
6
|
-
import { SqlError } from '
|
|
4
|
+
import { Sharding } from 'effect/cluster';
|
|
5
|
+
import { SqlClient } from 'effect/sql';
|
|
6
|
+
import { SqlError } from 'effect/sql';
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Everything a dialect supplies to run the cluster-engine suite against
|
|
@@ -14,7 +14,7 @@ export declare interface ClusterEngineFixture {
|
|
|
14
14
|
readonly name: string;
|
|
15
15
|
/** The dialect's `SqlClient` layer (postgres/mysql/mariadb/mssql/sqlite).
|
|
16
16
|
* Typed via the engine's own option so `@voltro/testing` needs no
|
|
17
|
-
* direct
|
|
17
|
+
* direct `effect/sql` dependency. */
|
|
18
18
|
readonly sqlClientLayer: WorkflowEngineLayerOptions['sqlClientLayer'];
|
|
19
19
|
/** `'sql'` for server dialects, `'memory'` for sqlite. */
|
|
20
20
|
readonly runnerStorage: 'sql' | 'memory';
|
|
@@ -75,20 +75,20 @@ export declare const runClusterEngineSuite: (fixture: ClusterEngineFixture) => v
|
|
|
75
75
|
export declare const runJournalEncryptionSuite: (fixture: ClusterEngineFixture) => void;
|
|
76
76
|
|
|
77
77
|
/**
|
|
78
|
-
* Runner-storage backend for
|
|
78
|
+
* Runner-storage backend for `effect/cluster.SingleRunner`.
|
|
79
79
|
*
|
|
80
80
|
* - `'sql'` — durable across crashes + multi-instance scale-out.
|
|
81
81
|
* Used for postgres / mysql / mariadb / mssql.
|
|
82
82
|
* - `'memory'` — durable replay within the SAME process; no
|
|
83
83
|
* horizontal scale-out. Used for `sqlite` because
|
|
84
|
-
*
|
|
84
|
+
* `effect/cluster.SqlRunnerStorage` has no SQLite
|
|
85
85
|
* lock-acquisition branch (`pg_advisory_lock` and
|
|
86
86
|
* `GET_LOCK` aren't available there).
|
|
87
87
|
*/
|
|
88
88
|
declare type RunnerStorage = 'sql' | 'memory';
|
|
89
89
|
|
|
90
90
|
/**
|
|
91
|
-
* How
|
|
91
|
+
* How `effect/cluster`'s `SqlRunnerStorage` coordinates shard ownership across
|
|
92
92
|
* a fleet of runners:
|
|
93
93
|
*
|
|
94
94
|
* - `'advisory'` — session-scoped DB advisory locks (`GET_LOCK` on
|
|
@@ -129,7 +129,7 @@ declare interface WorkflowEngineLayerOptions {
|
|
|
129
129
|
* runs through it for sqlite, the layer's resource lifecycle keeps
|
|
130
130
|
* the cluster's accounting clean.
|
|
131
131
|
*/
|
|
132
|
-
readonly sqlClientLayer: Layer.Layer<SqlClient.SqlClient,
|
|
132
|
+
readonly sqlClientLayer: Layer.Layer<SqlClient.SqlClient, Config.ConfigError | SqlError.SqlError, never>;
|
|
133
133
|
/** SQL ownership for shared primaries; memory for local/embedded stores. */
|
|
134
134
|
readonly runnerStorage: RunnerStorage;
|
|
135
135
|
/**
|
|
@@ -161,7 +161,7 @@ declare interface WorkflowEngineLayerOptions {
|
|
|
161
161
|
* lease-expiry problem, NOT a polling one: a dead runner keeps its shards
|
|
162
162
|
* until its heartbeat goes stale, so the floor is the lease TTL.
|
|
163
163
|
*
|
|
164
|
-
* - `failoverLeaseSeconds` (default 35,
|
|
164
|
+
* - `failoverLeaseSeconds` (default 35, `effect/cluster`
|
|
165
165
|
* `shardLockExpiration`) — how long after a runner's last heartbeat its
|
|
166
166
|
* shards become claimable. Lower = faster failover on a hard crash, at the
|
|
167
167
|
* cost of false-positive reclaims if a healthy runner is paused longer than
|
|
@@ -178,7 +178,7 @@ declare interface WorkflowEngineLayerOptions {
|
|
|
178
178
|
readonly failoverLeaseSeconds?: number;
|
|
179
179
|
readonly failoverHeartbeatSeconds?: number;
|
|
180
180
|
/**
|
|
181
|
-
* NEW-MESSAGE latency across replicas (default 10,
|
|
181
|
+
* NEW-MESSAGE latency across replicas (default 10, `effect/cluster`
|
|
182
182
|
* `entityMessagePollInterval`). DISTINCT from failover: this is how long a
|
|
183
183
|
* freshly-triggered workflow can wait before the replica that OWNS its shard
|
|
184
184
|
* picks it up. When the triggering replica owns the shard it's immediate
|