@voltro/workflow 0.85.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +666 -0
- package/THIRD-PARTY-NOTICES.md +5 -33
- package/dist/admission.js +140 -0
- package/dist/admissionDrainer.js +705 -0
- package/dist/admissionGate.js +115 -0
- package/dist/admissionStore.js +347 -0
- package/dist/admissionSubmission.js +297 -0
- package/dist/admissionTables.js +137 -0
- package/dist/admissionTransaction.js +21 -0
- package/dist/awaitEvent.js +82 -0
- package/dist/budgetHold.js +136 -0
- package/dist/cancelOnPlan.js +75 -0
- package/dist/cancelOnSweep.js +188 -0
- package/dist/cancelPendingStart.js +50 -0
- package/dist/cluster.js +8 -2
- package/dist/clusterCron.js +11 -0
- package/dist/clusterDiagnostics.js +20 -0
- package/dist/clusterDiagnosticsSql.js +145 -0
- package/dist/clusterLayer.js +110 -0
- package/dist/clusterMigrationGate.js +61 -0
- package/dist/clusterRunnerLayer.js +21 -0
- package/dist/clusterTestSuite.js +101 -101
- package/dist/controlMonitor.js +13 -0
- package/dist/controlRequest.js +367 -0
- package/dist/executionRefusal.js +162 -0
- package/dist/failureHandoff.js +183 -0
- package/dist/failureMessage.js +12 -0
- package/dist/failureRecovery.js +50 -0
- package/dist/failureReportStore.js +60 -0
- package/dist/flowControl.js +142 -0
- package/dist/forwardStartReceipts.js +41 -0
- package/dist/inMemoryLayer.js +7 -0
- package/dist/inMemoryRecorder.js +56 -0
- package/dist/index.d.ts +82 -8
- package/dist/index.js +49 -5
- package/dist/inspectWorkflow.js +101 -0
- package/dist/{journalEncryption-DTyhUzaS.js → journalEncryption.js} +30 -37
- package/dist/messages.js +92 -0
- package/dist/nondeterminism.js +51 -0
- package/dist/parentClosure.js +141 -0
- package/dist/parentClosureRecovery.js +54 -0
- package/dist/pendingSubmission.js +85 -0
- package/dist/prepareSubmissionDispatch.js +17 -0
- package/dist/preparedExecution.js +14 -0
- package/dist/primitives.js +225 -2
- package/dist/queuedCancellationRequest.js +364 -0
- package/dist/queuedControl.js +21 -0
- package/dist/recorder.js +11 -0
- package/dist/residentExecutionEvidence.js +64 -0
- package/dist/retryPolicy.js +57 -0
- package/dist/runEventsTable.js +17 -0
- package/dist/runGeneration.js +345 -0
- package/dist/runRecording.js +615 -0
- package/dist/runStepsTable.js +28 -0
- package/dist/runnerStorageGuard.js +29 -0
- package/dist/runsTable.js +42 -0
- package/dist/scheduleDeclaration.js +44 -0
- package/dist/signalHint.js +23 -0
- package/dist/signals.js +70 -0
- package/dist/signalsSuspending.js +57 -0
- package/dist/sqliteWorkflowOwnership.js +26 -0
- package/dist/stalenessSweep.js +191 -0
- package/dist/startContextsTable.js +20 -0
- package/dist/startReceipt.js +143 -0
- package/dist/submissionRecovery.js +122 -0
- package/dist/submissionRejection.js +10 -0
- package/dist/submissionRetry.js +71 -0
- package/dist/suspensionUnavailable.js +9 -0
- package/dist/updates.js +87 -0
- package/dist/watermarkTable.js +9 -0
- package/dist/webCrypto.js +17 -0
- package/dist/workflowDelivery.js +24 -0
- package/dist/workflowJournal.js +124 -0
- package/dist/workflowJournalReset.js +111 -0
- package/dist/workflowReadSelections.js +12 -0
- package/dist/workflowRestartDispatch.js +29 -0
- package/dist/workflowRestartObservation.js +60 -0
- package/dist/workflowRestartPreparation.js +109 -0
- package/dist/workflowRestartReservation.js +426 -0
- package/dist/workflowResumeDispatch.js +29 -0
- package/package.json +4 -4
- package/dist/cluster-F_bhqVP7.js +0 -679
- package/dist/primitives-DX04WTWh.js +0 -507
- package/dist/src-C2NEyUM8.js +0 -3925
- package/dist/workflowReadSelections-CZAUjP7f.js +0 -2096
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { Result as e, Schema as t } from "effect";
|
|
2
|
+
//#region src/cancelOnPlan.ts
|
|
3
|
+
var n = (e) => {
|
|
4
|
+
let t = e instanceof Error ? e.message : String(e);
|
|
5
|
+
return t.length > 400 ? `${t.slice(0, 400)}…` : t;
|
|
6
|
+
}, r = (r) => {
|
|
7
|
+
let i = [], a = [], o = [], s = /* @__PURE__ */ new Set(), c = /* @__PURE__ */ new Set(), l = r.payloadSchema === void 0 ? (t) => e.succeed(t) : (i) => {
|
|
8
|
+
let a = t.decodeUnknownResult(r.payloadSchema)(i);
|
|
9
|
+
return e.isSuccess(a) ? e.succeed(a.success) : e.fail(n(a.failure));
|
|
10
|
+
}, u = r.runs.map((e) => ({
|
|
11
|
+
run: e,
|
|
12
|
+
decoded: l(e.payload)
|
|
13
|
+
})), d = r.intents.map((e) => ({
|
|
14
|
+
intent: e,
|
|
15
|
+
decoded: l(e.payload)
|
|
16
|
+
}));
|
|
17
|
+
for (let l of r.entries) for (let f of r.events) {
|
|
18
|
+
if (f.name !== l.event) continue;
|
|
19
|
+
let p = t.decodeUnknownResult(l.schema)(f.payload);
|
|
20
|
+
if (e.isFailure(p)) {
|
|
21
|
+
o.push({
|
|
22
|
+
kind: "event-decode",
|
|
23
|
+
workflowName: r.workflowName,
|
|
24
|
+
event: l.event,
|
|
25
|
+
subject: f.id,
|
|
26
|
+
detail: n(p.failure)
|
|
27
|
+
});
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
let m = p.success, h = (e) => e.getTime() > f.occurredAt.getTime() ? !1 : l.withinMs === void 0 || f.occurredAt.getTime() - e.getTime() <= l.withinMs, g = (t, i) => {
|
|
31
|
+
if (e.isFailure(t)) return o.push({
|
|
32
|
+
kind: "payload-decode",
|
|
33
|
+
workflowName: r.workflowName,
|
|
34
|
+
event: l.event,
|
|
35
|
+
subject: i,
|
|
36
|
+
detail: t.failure
|
|
37
|
+
}), !1;
|
|
38
|
+
try {
|
|
39
|
+
return l.match(m, t.success) === !0;
|
|
40
|
+
} catch (e) {
|
|
41
|
+
return o.push({
|
|
42
|
+
kind: "match-threw",
|
|
43
|
+
workflowName: r.workflowName,
|
|
44
|
+
event: l.event,
|
|
45
|
+
subject: i,
|
|
46
|
+
detail: n(e)
|
|
47
|
+
}), !1;
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
for (let { run: e, decoded: t } of u) s.has(e.runId) || h(e.startedAt) && g(t, e.runId) && (s.add(e.runId), i.push({
|
|
51
|
+
runId: e.runId,
|
|
52
|
+
executionId: e.executionId,
|
|
53
|
+
reason: l.reason,
|
|
54
|
+
event: l.event,
|
|
55
|
+
eventId: f.id
|
|
56
|
+
}));
|
|
57
|
+
for (let { intent: e, decoded: t } of d) c.has(e.intentId) || h(e.firstSeenAt) && g(t, e.intentId) && (c.add(e.intentId), a.push({
|
|
58
|
+
intentId: e.intentId,
|
|
59
|
+
reason: l.reason,
|
|
60
|
+
event: l.event,
|
|
61
|
+
eventId: f.id
|
|
62
|
+
}));
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
cancels: i,
|
|
66
|
+
discards: a,
|
|
67
|
+
problems: o
|
|
68
|
+
};
|
|
69
|
+
}, i = (e) => {
|
|
70
|
+
let t;
|
|
71
|
+
for (let n of e) (t === void 0 || n.occurredAt.getTime() > t.getTime()) && (t = n.occurredAt);
|
|
72
|
+
return t;
|
|
73
|
+
};
|
|
74
|
+
//#endregion
|
|
75
|
+
export { i as newestEventAt, r as planCancellations };
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { newestEventAt as e, planCancellations as t } from "./cancelOnPlan.js";
|
|
2
|
+
import { and as n, eq as r, gte as i } from "@voltro/database";
|
|
3
|
+
//#region src/cancelOnSweep.ts
|
|
4
|
+
var a = "_voltro_workflow_watermarks", o = "_voltro_workflow_events", s = "cancelOn", c = 36e5, l = 500, u = 2e3, d = (e) => {
|
|
5
|
+
if (e instanceof Date) return e;
|
|
6
|
+
if (typeof e == "number") return new Date(e);
|
|
7
|
+
if (typeof e == "string") {
|
|
8
|
+
let t = new Date(e);
|
|
9
|
+
if (!Number.isNaN(t.getTime())) return t;
|
|
10
|
+
}
|
|
11
|
+
}, f = (e) => e instanceof Error ? e.message : String(e), p = async (e, t) => d((await e.query({
|
|
12
|
+
table: a,
|
|
13
|
+
predicate: r("id", t),
|
|
14
|
+
order: [],
|
|
15
|
+
take: 1,
|
|
16
|
+
skip: void 0,
|
|
17
|
+
projection: void 0
|
|
18
|
+
}))[0]?.positionAt), m = async (e, t, n) => {
|
|
19
|
+
await e.upsert(a, {
|
|
20
|
+
id: t,
|
|
21
|
+
positionAt: n,
|
|
22
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
23
|
+
}, {
|
|
24
|
+
conflictColumns: ["id"],
|
|
25
|
+
update: (e) => {
|
|
26
|
+
let t = d(e.positionAt);
|
|
27
|
+
return t !== void 0 && t.getTime() >= n.getTime() ? {
|
|
28
|
+
positionAt: t,
|
|
29
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
30
|
+
} : {
|
|
31
|
+
positionAt: n,
|
|
32
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
}, h = async (e, t, i) => {
|
|
37
|
+
let a = async (a) => e.query({
|
|
38
|
+
table: "_voltro_workflow_runs",
|
|
39
|
+
predicate: n(r("tag", t), r("status", a)),
|
|
40
|
+
order: [{
|
|
41
|
+
column: "startedAt",
|
|
42
|
+
direction: "asc"
|
|
43
|
+
}],
|
|
44
|
+
take: i,
|
|
45
|
+
skip: void 0,
|
|
46
|
+
projection: void 0
|
|
47
|
+
});
|
|
48
|
+
return [...await a("running"), ...await a("suspended")].flatMap((e) => {
|
|
49
|
+
let t = e, n = d(t.startedAt);
|
|
50
|
+
return typeof t.id != "string" || typeof t.executionId != "string" || n === void 0 ? [] : [{
|
|
51
|
+
runId: t.id,
|
|
52
|
+
executionId: t.executionId,
|
|
53
|
+
payload: t.payload,
|
|
54
|
+
startedAt: n
|
|
55
|
+
}];
|
|
56
|
+
});
|
|
57
|
+
}, g = async (e, t, n) => (await e.query({
|
|
58
|
+
table: "_voltro_workflow_pending",
|
|
59
|
+
predicate: r("workflowName", t),
|
|
60
|
+
order: [{
|
|
61
|
+
column: "firstSeenAt",
|
|
62
|
+
direction: "asc"
|
|
63
|
+
}],
|
|
64
|
+
take: n,
|
|
65
|
+
skip: void 0,
|
|
66
|
+
projection: void 0
|
|
67
|
+
})).flatMap((e) => {
|
|
68
|
+
let t = e, n = d(t.firstSeenAt);
|
|
69
|
+
return typeof t.id != "string" || n === void 0 ? [] : [{
|
|
70
|
+
intentId: t.id,
|
|
71
|
+
payload: t.payload,
|
|
72
|
+
firstSeenAt: n
|
|
73
|
+
}];
|
|
74
|
+
}), _ = async (n, r) => {
|
|
75
|
+
let a = r.workflows.filter((e) => e.entries.length > 0);
|
|
76
|
+
if (a.length === 0) return {
|
|
77
|
+
examined: !1,
|
|
78
|
+
events: 0,
|
|
79
|
+
cancelled: 0,
|
|
80
|
+
discarded: 0,
|
|
81
|
+
catchingUp: !1,
|
|
82
|
+
sawFullRunPage: !1,
|
|
83
|
+
problems: [],
|
|
84
|
+
failures: [],
|
|
85
|
+
watermark: void 0
|
|
86
|
+
};
|
|
87
|
+
let c = n.now?.() ?? Date.now(), l = r.batch ?? 500, u = r.coldLookbackMs ?? 36e5, _ = /* @__PURE__ */ new Set();
|
|
88
|
+
for (let e of a) for (let t of e.entries) _.add(t.event);
|
|
89
|
+
let v = await p(n.store, s), y = v ?? new Date(c - u);
|
|
90
|
+
v === void 0 && (await m(n.store, s, y), v = y);
|
|
91
|
+
let b = await n.store.query({
|
|
92
|
+
table: o,
|
|
93
|
+
predicate: i("occurredAt", y),
|
|
94
|
+
order: [{
|
|
95
|
+
column: "occurredAt",
|
|
96
|
+
direction: "asc"
|
|
97
|
+
}],
|
|
98
|
+
take: l,
|
|
99
|
+
skip: void 0,
|
|
100
|
+
projection: void 0
|
|
101
|
+
}), x = [];
|
|
102
|
+
for (let e of b) {
|
|
103
|
+
let t = e, n = d(t.occurredAt);
|
|
104
|
+
typeof t.id == "string" && typeof t.name == "string" && n !== void 0 && _.has(t.name) && x.push({
|
|
105
|
+
id: t.id,
|
|
106
|
+
name: t.name,
|
|
107
|
+
payload: t.payload,
|
|
108
|
+
occurredAt: n
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
let S = [], C = [], w = 0, T = 0, E = !1, D = !1, O = r.runPage ?? 2e3;
|
|
112
|
+
if (x.length > 0) for (let e of a) {
|
|
113
|
+
let r = x.filter((t) => e.entries.some((e) => e.event === t.name));
|
|
114
|
+
if (r.length === 0) continue;
|
|
115
|
+
let i = [], a = [];
|
|
116
|
+
try {
|
|
117
|
+
i = await h(n.store, e.workflowName, O), a = await g(n.store, e.workflowName, O), (i.length >= O || a.length >= O) && (E = !0);
|
|
118
|
+
} catch (t) {
|
|
119
|
+
C.push({
|
|
120
|
+
subject: e.workflowName,
|
|
121
|
+
detail: f(t)
|
|
122
|
+
}), D = !0;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
let o = t({
|
|
126
|
+
workflowName: e.workflowName,
|
|
127
|
+
entries: e.entries,
|
|
128
|
+
payloadSchema: e.payloadSchema,
|
|
129
|
+
events: r,
|
|
130
|
+
runs: i,
|
|
131
|
+
intents: a
|
|
132
|
+
});
|
|
133
|
+
S.push(...o.problems);
|
|
134
|
+
for (let t of o.cancels) try {
|
|
135
|
+
await n.cancelRun({
|
|
136
|
+
...t,
|
|
137
|
+
workflowName: e.workflowName
|
|
138
|
+
}), w += 1, n.log?.info("workflow.cancelOn: run cancelled", {
|
|
139
|
+
workflow: e.workflowName,
|
|
140
|
+
runId: t.runId,
|
|
141
|
+
event: t.event,
|
|
142
|
+
eventId: t.eventId
|
|
143
|
+
});
|
|
144
|
+
} catch (e) {
|
|
145
|
+
C.push({
|
|
146
|
+
subject: t.runId,
|
|
147
|
+
detail: f(e)
|
|
148
|
+
}), D = !0;
|
|
149
|
+
}
|
|
150
|
+
for (let t of o.discards) try {
|
|
151
|
+
await n.discardIntent({
|
|
152
|
+
...t,
|
|
153
|
+
workflowName: e.workflowName
|
|
154
|
+
}), T += 1;
|
|
155
|
+
} catch (e) {
|
|
156
|
+
C.push({
|
|
157
|
+
subject: t.intentId,
|
|
158
|
+
detail: f(e)
|
|
159
|
+
}), D = !0;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
let k = e(x), A = v;
|
|
163
|
+
if (D) n.log?.warn("workflow.cancelOn: holding the watermark — evaluation or delivery remains incomplete", { failures: C.length });
|
|
164
|
+
else if (k !== void 0) try {
|
|
165
|
+
await m(n.store, s, k), A = k;
|
|
166
|
+
} catch (e) {
|
|
167
|
+
C.push({
|
|
168
|
+
subject: s,
|
|
169
|
+
detail: f(e)
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
return S.length > 0 && n.log?.warn("workflow.cancelOn: some events could not be evaluated", {
|
|
173
|
+
count: S.length,
|
|
174
|
+
first: S[0]
|
|
175
|
+
}), {
|
|
176
|
+
examined: !0,
|
|
177
|
+
events: x.length,
|
|
178
|
+
cancelled: w,
|
|
179
|
+
discarded: T,
|
|
180
|
+
catchingUp: b.length >= l,
|
|
181
|
+
sawFullRunPage: E,
|
|
182
|
+
problems: S,
|
|
183
|
+
failures: C,
|
|
184
|
+
watermark: A
|
|
185
|
+
};
|
|
186
|
+
};
|
|
187
|
+
//#endregion
|
|
188
|
+
export { s as CANCEL_ON_WATERMARK, c as DEFAULT_COLD_LOOKBACK_MS, l as DEFAULT_EVENT_BATCH, u as DEFAULT_RUN_PAGE, o as EVENTS_TABLE, a as WATERMARKS_TABLE, p as readWatermark, _ as sweepCancelOn, m as writeWatermark };
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { readAdmissionSubmission as e } from "./admissionSubmission.js";
|
|
2
|
+
import { readWorkflowStartReceipt as t, settlePendingStartReceipts as n } from "./startReceipt.js";
|
|
3
|
+
import { PENDING_TABLE as r, toPendingIntent as i } from "./admissionStore.js";
|
|
4
|
+
import { eq as a } from "@voltro/database";
|
|
5
|
+
//#region src/cancelPendingStart.ts
|
|
6
|
+
var o = async (o, s) => {
|
|
7
|
+
if (!Number.isFinite(s.now) || !Number.isFinite(new Date(s.now).getTime())) throw Error("Queued cancellation requires a valid timestamp");
|
|
8
|
+
let c = await t(o, s.receiptId);
|
|
9
|
+
if (c === void 0 || c.workflowName !== s.workflowName || c.tenantId !== s.tenantId) throw Error("Queued cancellation receipt is missing or belongs to another workflow or tenant");
|
|
10
|
+
if (c.terminal !== null) return {
|
|
11
|
+
kind: "terminal",
|
|
12
|
+
outcome: c.terminal
|
|
13
|
+
};
|
|
14
|
+
if (c.submissionId !== null) {
|
|
15
|
+
let t = await e(o, c.submissionId);
|
|
16
|
+
if (t === void 0 || !t.receiptsBound || t.workflowName !== c.workflowName || t.tenantId !== c.tenantId || !t.intentIds.includes(c.pendingId)) throw Error("Queued cancellation receipt has no matching submission");
|
|
17
|
+
return {
|
|
18
|
+
kind: "submitted",
|
|
19
|
+
submissionId: t.id
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
let [l] = await o.query({
|
|
23
|
+
table: r,
|
|
24
|
+
predicate: a("id", c.pendingId),
|
|
25
|
+
lock: "update",
|
|
26
|
+
order: [],
|
|
27
|
+
take: 1,
|
|
28
|
+
skip: void 0,
|
|
29
|
+
projection: void 0
|
|
30
|
+
});
|
|
31
|
+
if (l === void 0) throw Error("Queued cancellation has no pending input or submission");
|
|
32
|
+
let u = i(l);
|
|
33
|
+
if (u.workflowName !== c.workflowName || u.tenantId !== c.tenantId || u.deadLetteredAt !== void 0) throw Error("Queued cancellation input has contradictory ownership or is dead-lettered");
|
|
34
|
+
let d = {
|
|
35
|
+
kind: "cancelled",
|
|
36
|
+
at: s.now
|
|
37
|
+
};
|
|
38
|
+
if (await n(o, {
|
|
39
|
+
pendingId: c.pendingId,
|
|
40
|
+
workflowName: c.workflowName,
|
|
41
|
+
tenantId: c.tenantId,
|
|
42
|
+
terminal: d
|
|
43
|
+
}), !await o.delete("_voltro_workflow_pending", c.pendingId)) throw Error("Queued cancellation lost its pending input");
|
|
44
|
+
return {
|
|
45
|
+
kind: "terminal",
|
|
46
|
+
outcome: d
|
|
47
|
+
};
|
|
48
|
+
};
|
|
49
|
+
//#endregion
|
|
50
|
+
export { o as cancelPendingWorkflowStartInTransaction };
|
package/dist/cluster.js
CHANGED
|
@@ -1,2 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import { WorkflowResumeDispatcher as e } from "./workflowResumeDispatch.js";
|
|
2
|
+
import { JournalExit as t, activityPrimaryKey as n, readWorkflowRunReply as r, redriveFailedRun as i, resumeRunFromStep as a } from "./workflowJournal.js";
|
|
3
|
+
import { WorkflowRestartDispatcher as o } from "./workflowRestartDispatch.js";
|
|
4
|
+
import { clusterMigrationGateLayer as s } from "./clusterMigrationGate.js";
|
|
5
|
+
import { clusterLockNamespaceOid as c, makeClusterDiagnostics as l } from "./clusterDiagnosticsSql.js";
|
|
6
|
+
import { DEFAULT_RUNNER_LISTEN_PORT as u, detectWsrepCluster as d, parseShardLockMode as f, pollWorkflowStorage as p, resolveFailoverTuning as m, resolveRunnerIdentity as h, resolveRunnerStorage as g, resolveShardLockDisableAdvisory as _, workflowEngineLayer as v } from "./clusterLayer.js";
|
|
7
|
+
import { makeClusterCronLayer as y } from "./clusterCron.js";
|
|
8
|
+
export { u as DEFAULT_RUNNER_LISTEN_PORT, t as JournalExit, o as WorkflowRestartDispatcher, e as WorkflowResumeDispatcher, n as activityPrimaryKey, c as clusterLockNamespaceOid, s as clusterMigrationGateLayer, d as detectWsrepCluster, y as makeClusterCronLayer, l as makeClusterDiagnostics, f as parseShardLockMode, p as pollWorkflowStorage, r as readWorkflowRunReply, i as redriveFailedRun, m as resolveFailoverTuning, h as resolveRunnerIdentity, g as resolveRunnerStorage, _ as resolveShardLockDisableAdvisory, a as resumeRunFromStep, v as workflowEngineLayer };
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Duration as e, Effect as t } from "effect";
|
|
2
|
+
import { ClusterCron as n } from "effect/cluster";
|
|
3
|
+
//#region src/clusterCron.ts
|
|
4
|
+
var r = (r) => n.make({
|
|
5
|
+
name: r.name,
|
|
6
|
+
cron: r.cron,
|
|
7
|
+
execute: t.promise(() => r.execute()),
|
|
8
|
+
skipIfOlderThan: r.skipIfOlderThan ?? e.days(1)
|
|
9
|
+
});
|
|
10
|
+
//#endregion
|
|
11
|
+
export { r as makeClusterCronLayer };
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Context as e, Effect as t } from "effect";
|
|
2
|
+
//#region src/clusterDiagnostics.ts
|
|
3
|
+
var n = 6e5, r = class extends e.Service()("@voltro/workflow/ClusterDiagnostics") {}, i = (e, n, r = Date.now()) => t.flatMap(e.messagesFor(n), (n) => {
|
|
4
|
+
if (n.length === 0) return t.succeed(void 0);
|
|
5
|
+
let i = [...new Set(n.filter((e) => !e.processed).map((e) => e.shardId))], a = i.length === 0 ? t.succeed([]) : e.shardLockHolders(i);
|
|
6
|
+
return t.map(a, (t) => ({
|
|
7
|
+
runner: e.ownAddress,
|
|
8
|
+
shardLockMode: e.shardLockMode,
|
|
9
|
+
messages: n.map((e) => ({
|
|
10
|
+
tag: e.tag,
|
|
11
|
+
shardId: e.shardId,
|
|
12
|
+
processed: e.processed,
|
|
13
|
+
lastRead: e.lastRead === null ? null : e.lastRead.toISOString(),
|
|
14
|
+
unreadForMs: !e.processed && e.lastRead !== null ? Math.max(0, r - e.lastRead.getTime()) : null
|
|
15
|
+
})),
|
|
16
|
+
shardLocks: t
|
|
17
|
+
}));
|
|
18
|
+
});
|
|
19
|
+
//#endregion
|
|
20
|
+
export { n as CLUSTER_REREAD_WINDOW_MS, r as ClusterDiagnostics, i as describeStalledRun };
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { CLUSTER_REREAD_WINDOW_MS as e } from "./clusterDiagnostics.js";
|
|
2
|
+
import { Effect as t } from "effect";
|
|
3
|
+
import { SqlClient as n } from "effect/sql";
|
|
4
|
+
//#region src/clusterDiagnosticsSql.ts
|
|
5
|
+
var r = (e) => {
|
|
6
|
+
if (e instanceof Date) return Number.isNaN(e.getTime()) ? null : e;
|
|
7
|
+
if (typeof e == "string" || typeof e == "number") {
|
|
8
|
+
let t = typeof e == "string" && /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(e) ? `${e.replace(" ", "T")}Z` : e, n = new Date(t);
|
|
9
|
+
return Number.isNaN(n.getTime()) ? null : n;
|
|
10
|
+
}
|
|
11
|
+
return null;
|
|
12
|
+
}, i = (e) => e === !0 || e === 1 || e === "1" || e instanceof Uint8Array && e[0] === 1, a = "cluster", o = (e = a) => {
|
|
13
|
+
let t = new TextEncoder().encode(`effect-cluster:${e}`), n = 2166136261;
|
|
14
|
+
for (let e of t) n = Math.imul(n ^ e, 16777619);
|
|
15
|
+
return (n | 0) >>> 0;
|
|
16
|
+
}, s = (e, t) => {
|
|
17
|
+
let n = e.lastIndexOf(":");
|
|
18
|
+
if (n < 0) return;
|
|
19
|
+
let r = e.slice(0, n), i = Number(e.slice(n + 1)), a = t.indexOf(r);
|
|
20
|
+
if (!(a < 0 || !Number.isInteger(i))) return (a + 1) * 1e6 + i;
|
|
21
|
+
}, c = (a) => t.gen(function* () {
|
|
22
|
+
let c = yield* n.SqlClient, l = a.shardGroups ?? ["default"], u = a.dialectId ?? "", d = u.startsWith("postgres") || u === "pg", f = (e) => e.pipe(t.catch(() => t.succeed([]))), p = c.onDialectOrElse({
|
|
23
|
+
pg: () => c.literal("NOW()"),
|
|
24
|
+
mysql: () => c.literal("NOW()"),
|
|
25
|
+
mssql: () => c.literal("GETDATE()"),
|
|
26
|
+
orElse: () => c.literal("CURRENT_TIMESTAMP")
|
|
27
|
+
}), m = (e, t) => {
|
|
28
|
+
let n = r(e.last_read), a = r(e.server_now), o = n !== null && a !== null ? Math.max(0, a.getTime() - n.getTime()) : null;
|
|
29
|
+
return {
|
|
30
|
+
tag: typeof e.tag == "string" ? e.tag : null,
|
|
31
|
+
entityType: String(e.entity_type ?? ""),
|
|
32
|
+
entityId: String(e.entity_id ?? ""),
|
|
33
|
+
shardId: String(e.shard_id ?? ""),
|
|
34
|
+
processed: i(e.processed),
|
|
35
|
+
lastRead: o === null ? null : new Date(t - o)
|
|
36
|
+
};
|
|
37
|
+
}, h = (e) => c.onDialectOrElse({
|
|
38
|
+
pg: () => c.literal(`NOW() - INTERVAL '${e} seconds'`),
|
|
39
|
+
mysql: () => c.literal(`NOW() - INTERVAL ${e} SECOND`),
|
|
40
|
+
mssql: () => c.literal(`DATEADD(SECOND, -${e}, GETDATE())`),
|
|
41
|
+
orElse: () => c.literal(`DATETIME('now', '-${e} seconds')`)
|
|
42
|
+
}), g = c.onDialectOrElse({
|
|
43
|
+
pg: () => c.literal("FALSE"),
|
|
44
|
+
orElse: () => c.literal("0")
|
|
45
|
+
}), _ = (e) => {
|
|
46
|
+
let n = Date.now();
|
|
47
|
+
return f(c`
|
|
48
|
+
SELECT tag, entity_type, entity_id, shard_id, processed, last_read, ${p} AS server_now
|
|
49
|
+
FROM cluster_messages
|
|
50
|
+
WHERE entity_id = ${e}
|
|
51
|
+
ORDER BY rowid ASC
|
|
52
|
+
`).pipe(t.map((e) => e.map((e) => m(e, n))));
|
|
53
|
+
}, v = (n = e) => {
|
|
54
|
+
let r = Math.max(1, Math.ceil(n / 1e3)), i = Date.now();
|
|
55
|
+
return f(c`
|
|
56
|
+
SELECT tag, entity_type, entity_id, shard_id, processed, last_read, ${p} AS server_now
|
|
57
|
+
FROM cluster_messages
|
|
58
|
+
WHERE processed = ${g}
|
|
59
|
+
AND last_read IS NOT NULL
|
|
60
|
+
AND last_read < ${h(r)}
|
|
61
|
+
AND (deliver_at IS NULL OR deliver_at <= ${i})
|
|
62
|
+
ORDER BY last_read ASC
|
|
63
|
+
`).pipe(t.map((e) => e.flatMap((e) => {
|
|
64
|
+
let t = m(e, i);
|
|
65
|
+
return t.lastRead === null ? [] : [{
|
|
66
|
+
...t,
|
|
67
|
+
unreadForMs: Math.max(0, i - t.lastRead.getTime())
|
|
68
|
+
}];
|
|
69
|
+
})));
|
|
70
|
+
}, y = (e) => f(c`
|
|
71
|
+
SELECT shard_id, address, acquired_at FROM cluster_locks WHERE ${c.in("shard_id", e)}
|
|
72
|
+
`).pipe(t.map((e) => e.map((e) => {
|
|
73
|
+
let t = e, n = String(t.address ?? ""), i = String(t.shard_id ?? "");
|
|
74
|
+
return {
|
|
75
|
+
shardId: i,
|
|
76
|
+
mode: "row",
|
|
77
|
+
holder: n,
|
|
78
|
+
own: n === a.ownAddress || (a.hasShard?.(i) ?? !1),
|
|
79
|
+
since: r(t.acquired_at)
|
|
80
|
+
};
|
|
81
|
+
}))), b = (e) => {
|
|
82
|
+
let n = /* @__PURE__ */ new Map();
|
|
83
|
+
for (let t of e) {
|
|
84
|
+
let e = s(t, l);
|
|
85
|
+
e !== void 0 && n.set(e, t);
|
|
86
|
+
}
|
|
87
|
+
return n.size === 0 ? t.succeed([]) : t.gen(function* () {
|
|
88
|
+
return (yield* f(c`
|
|
89
|
+
SELECT l.objid, a.pid, a.client_addr, a.application_name, a.state, a.state_change, a.backend_start, a.wait_event_type
|
|
90
|
+
FROM pg_locks l
|
|
91
|
+
JOIN pg_stat_activity a ON a.pid = l.pid
|
|
92
|
+
WHERE l.locktype = 'advisory' AND l.granted AND l.objsubid = 2
|
|
93
|
+
AND l.classid = ${o()} AND ${c.in("l.objid", [...n.keys()])}
|
|
94
|
+
`)).flatMap((e) => {
|
|
95
|
+
let t = e, i = n.get(Number(t.objid));
|
|
96
|
+
if (i === void 0) return [];
|
|
97
|
+
let o = Number(t.pid), s = t.client_addr === null || t.client_addr === void 0 ? "local socket" : String(t.client_addr), c = a.hasShard?.(i) ?? !1;
|
|
98
|
+
return [{
|
|
99
|
+
shardId: i,
|
|
100
|
+
mode: "advisory",
|
|
101
|
+
holder: c ? `this runner (pid ${o})` : `pid ${o} from ${s}`,
|
|
102
|
+
own: c,
|
|
103
|
+
since: r(t.state_change),
|
|
104
|
+
detail: {
|
|
105
|
+
pid: o,
|
|
106
|
+
clientAddr: t.client_addr ?? null,
|
|
107
|
+
applicationName: t.application_name ?? null,
|
|
108
|
+
state: t.state ?? null,
|
|
109
|
+
waitEventType: t.wait_event_type ?? null,
|
|
110
|
+
backendStart: r(t.backend_start)?.toISOString() ?? null
|
|
111
|
+
}
|
|
112
|
+
}];
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
};
|
|
116
|
+
return {
|
|
117
|
+
ownAddress: a.ownAddress,
|
|
118
|
+
shardLockMode: a.shardLockMode,
|
|
119
|
+
messagesFor: _,
|
|
120
|
+
unreadMessages: v,
|
|
121
|
+
shardLockHolders: (e) => e.length === 0 || a.runnerStorage !== "sql" ? t.succeed([]) : a.shardLockMode === "row" ? y(e) : a.shardLockMode === "advisory" && d ? b(e) : t.succeed([]),
|
|
122
|
+
runners: () => a.runnerStorage === "sql" ? f(c`SELECT address, last_heartbeat, healthy FROM cluster_runners ORDER BY last_heartbeat DESC`).pipe(t.map((e) => e.map((e) => {
|
|
123
|
+
let t = e;
|
|
124
|
+
return {
|
|
125
|
+
address: String(t.address ?? ""),
|
|
126
|
+
lastHeartbeat: r(t.last_heartbeat),
|
|
127
|
+
healthy: i(t.healthy)
|
|
128
|
+
};
|
|
129
|
+
}))) : t.succeed([]),
|
|
130
|
+
discardMessages: (e) => {
|
|
131
|
+
let r = e.unreadForMs === void 0 ? c.literal("") : c`AND last_read IS NOT NULL AND last_read < ${h(Math.max(1, Math.ceil(e.unreadForMs / 1e3)))}`;
|
|
132
|
+
return t.gen(function* () {
|
|
133
|
+
let t = (yield* c`
|
|
134
|
+
SELECT id FROM cluster_messages
|
|
135
|
+
WHERE entity_type = ${e.entityType} AND entity_id = ${e.entityId}
|
|
136
|
+
AND processed = ${g}
|
|
137
|
+
${r}
|
|
138
|
+
`).map((e) => e.id).filter((e) => e != null).map(String);
|
|
139
|
+
return t.length === 0 ? 0 : (yield* c`DELETE FROM cluster_replies WHERE ${c.in("request_id", t)}`, yield* c`DELETE FROM cluster_messages WHERE ${c.in("id", t)}`, t.length);
|
|
140
|
+
}).pipe(c.withTransaction, t.provideService(n.SafeIntegers, !0), t.catch(() => t.succeed(0)));
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
});
|
|
144
|
+
//#endregion
|
|
145
|
+
export { o as clusterLockNamespaceOid, c as makeClusterDiagnostics };
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { withWorkflowDelivery as e } from "./workflowDelivery.js";
|
|
2
|
+
import { ClusterDiagnostics as t } from "./clusterDiagnostics.js";
|
|
3
|
+
import { WorkflowResumeDispatcher as n, makeWorkflowResumeDispatcher as r } from "./workflowResumeDispatch.js";
|
|
4
|
+
import { WorkflowRestartDispatcher as i, makeWorkflowRestartDispatcher as a } from "./workflowRestartDispatch.js";
|
|
5
|
+
import { clusterMigrationGateLayer as o } from "./clusterMigrationGate.js";
|
|
6
|
+
import { clusterRunnerLayer as s } from "./clusterRunnerLayer.js";
|
|
7
|
+
import { acquireSqliteWorkflowOwnership as c } from "./sqliteWorkflowOwnership.js";
|
|
8
|
+
import { makeClusterDiagnostics as l } from "./clusterDiagnosticsSql.js";
|
|
9
|
+
import { RateLimiter as u } from "effect/persistence";
|
|
10
|
+
import { Context as d, Duration as f, Effect as p, Layer as m, Option as h } from "effect";
|
|
11
|
+
import { SqlClient as g } from "effect/sql";
|
|
12
|
+
import { ClusterWorkflowEngine as _, MessageStorage as v, RunnerAddress as y, ShardId as b, Sharding as x } from "effect/cluster";
|
|
13
|
+
import { createLogger as S } from "@voltro/logger";
|
|
14
|
+
//#region src/clusterLayer.ts
|
|
15
|
+
var C = S({ scope: "voltro:workflow" }), w = 34e3, T = (e) => {
|
|
16
|
+
let t = e.runnerListenHost ?? process.env.POD_IP ?? process.env.VOLTRO_WORKFLOW_RUNNER_HOST ?? "localhost";
|
|
17
|
+
return {
|
|
18
|
+
host: t,
|
|
19
|
+
port: e.runnerListenPort ?? 34e3,
|
|
20
|
+
localhostRisk: e.runnerStorage === "sql" && (t === "localhost" || t === "127.0.0.1")
|
|
21
|
+
};
|
|
22
|
+
}, E = (e) => {
|
|
23
|
+
let t = e.sqlRunnerSupported ?? !e.sqliteFamily, n = t ? "sql" : "memory", r = e.override ?? process.env.VOLTRO_WORKFLOW_RUNNER_STORAGE;
|
|
24
|
+
if (r === void 0 || r.trim() === "") return n;
|
|
25
|
+
let i = r.trim().toLowerCase();
|
|
26
|
+
if (i !== "memory" && i !== "sql") throw Error(`VOLTRO_WORKFLOW_RUNNER_STORAGE must be 'memory' or 'sql', got '${r}'.`);
|
|
27
|
+
if (i === "sql" && !t) throw Error("VOLTRO_WORKFLOW_RUNNER_STORAGE='sql' is unsupported for this database connection. Use 'memory' for a local or embedded SQLite-family connection.");
|
|
28
|
+
return i;
|
|
29
|
+
}, D = (e = process.env.VOLTRO_WORKFLOW_SHARD_LOCK) => {
|
|
30
|
+
if (e === void 0 || e.trim() === "") return "auto";
|
|
31
|
+
let t = e.trim().toLowerCase();
|
|
32
|
+
if (t === "auto" || t === "row" || t === "advisory") return t;
|
|
33
|
+
throw Error(`VOLTRO_WORKFLOW_SHARD_LOCK must be 'auto', 'row', or 'advisory', got '${e}'.`);
|
|
34
|
+
}, O = p.gen(function* () {
|
|
35
|
+
let e = yield* (yield* g.SqlClient)`SELECT @@wsrep_on AS wsrep`.pipe(p.catch(() => p.succeed([])));
|
|
36
|
+
if (e.length === 0) return !1;
|
|
37
|
+
let t = e[0].wsrep;
|
|
38
|
+
return t === 1 || t === "1" || t === !0 || t === "ON" || t === "on";
|
|
39
|
+
}), k = (e) => p.gen(function* () {
|
|
40
|
+
return e.runnerStorage === "memory" ? !1 : e.dialectId === "sqlite" || e.dialectId === "turso" || e.mode === "row" ? !0 : e.mode === "advisory" || e.dialectId !== "mysql" && e.dialectId !== "mariadb" ? !1 : yield* O;
|
|
41
|
+
}), A = (e) => ({
|
|
42
|
+
coordination: e.disableAdvisory ? "row-based" : "advisory",
|
|
43
|
+
dialect: e.dialectId ?? "unknown",
|
|
44
|
+
mode: e.mode,
|
|
45
|
+
...e.disableAdvisory && e.mode === "auto" ? { reason: e.dialectId === "sqlite" || e.dialectId === "turso" ? "SQLite-family connection uses row locks" : "wsrep/Galera cluster detected" } : {}
|
|
46
|
+
}), j = () => {
|
|
47
|
+
let e = (e) => {
|
|
48
|
+
if (e === void 0) return;
|
|
49
|
+
let t = Number(e);
|
|
50
|
+
return Number.isFinite(t) && t > 0 ? t : void 0;
|
|
51
|
+
}, t = e(process.env.VOLTRO_WORKFLOW_FAILOVER_LEASE), n = e(process.env.VOLTRO_WORKFLOW_FAILOVER_HEARTBEAT), r = e(process.env.VOLTRO_WORKFLOW_POLL_INTERVAL);
|
|
52
|
+
return {
|
|
53
|
+
...t === void 0 ? {} : { failoverLeaseSeconds: t },
|
|
54
|
+
...n === void 0 ? {} : { failoverHeartbeatSeconds: n },
|
|
55
|
+
...r === void 0 ? {} : { messagePollSeconds: r }
|
|
56
|
+
};
|
|
57
|
+
}, M = p.flatMap(x.Sharding, (e) => e.pollStorage), N = /* @__PURE__ */ new Set(), P = (e) => Object.assign(Object.create(Object.getPrototypeOf(e)), e, { requestIdForPrimaryKey: (t) => e.requestIdForPrimaryKey(t).pipe(p.provideService(g.SafeIntegers, !0)) }), F = (S) => {
|
|
58
|
+
let w = T({
|
|
59
|
+
runnerStorage: S.runnerStorage,
|
|
60
|
+
runnerListenHost: S.runnerListenHost,
|
|
61
|
+
runnerListenPort: S.runnerListenPort
|
|
62
|
+
}), E = w.host, O = w.port;
|
|
63
|
+
w.localhostRisk && !N.has(E) && (N.add(E), C.warn("runner host is not routable from other pods with SQL cluster storage — a workflow cannot resume on another pod after a reschedule. Inject POD_IP via the K8s downward API (fieldRef: status.podIP) or set runnerListenHost / VOLTRO_WORKFLOW_RUNNER_HOST.", { runnerHost: E }));
|
|
64
|
+
let j = S.dialectId === "sqlite" ? m.effect(g.SqlClient, p.gen(function* () {
|
|
65
|
+
let e = yield* g.SqlClient, t = (yield* e.unsafe("PRAGMA database_list")).find((e) => e.name === "main");
|
|
66
|
+
return t === void 0 ? yield* p.die(/* @__PURE__ */ Error("SQLite workflow connection has no main database")) : (t.file !== "" && (yield* c(t.file)), e);
|
|
67
|
+
}).pipe(p.orDie)).pipe(m.provide(S.sqlClientLayer)) : S.sqlClientLayer, M = o({
|
|
68
|
+
runnerStorage: S.runnerStorage,
|
|
69
|
+
charset: S.charset
|
|
70
|
+
}).pipe(m.provide(j)), F = S.shardLockMode ?? D(), I = m.unwrap(k({
|
|
71
|
+
dialectId: S.dialectId,
|
|
72
|
+
runnerStorage: S.runnerStorage,
|
|
73
|
+
mode: F
|
|
74
|
+
}).pipe(p.map((e) => {
|
|
75
|
+
S.runnerStorage === "sql" && C.info("shard-lock coordination", A({
|
|
76
|
+
disableAdvisory: e,
|
|
77
|
+
dialectId: S.dialectId,
|
|
78
|
+
mode: F
|
|
79
|
+
}));
|
|
80
|
+
let o = m.effect(t, p.flatMap(x.Sharding, (t) => l({
|
|
81
|
+
ownAddress: y.make(E, O).toString(),
|
|
82
|
+
shardLockMode: S.runnerStorage === "sql" ? e ? "row" : "advisory" : "none",
|
|
83
|
+
runnerStorage: S.runnerStorage,
|
|
84
|
+
dialectId: S.dialectId,
|
|
85
|
+
hasShard: (e) => t.hasShardId(b.fromString(e))
|
|
86
|
+
}))), c = {
|
|
87
|
+
runnerAddress: h.some(y.make(E, O)),
|
|
88
|
+
runnerListenAddress: h.some(y.make(E, O)),
|
|
89
|
+
...e ? { shardLockDisableAdvisory: !0 } : {},
|
|
90
|
+
...S.failoverLeaseSeconds === void 0 ? {} : { shardLockExpiration: f.seconds(S.failoverLeaseSeconds) },
|
|
91
|
+
...S.failoverHeartbeatSeconds === void 0 ? {} : { shardLockRefreshInterval: f.seconds(S.failoverHeartbeatSeconds) },
|
|
92
|
+
...S.messagePollSeconds === void 0 ? {} : { entityMessagePollInterval: f.seconds(S.messagePollSeconds) }
|
|
93
|
+
}, u = f.seconds(S.failoverHeartbeatSeconds ?? 10), d = s({
|
|
94
|
+
runnerStorage: S.runnerStorage,
|
|
95
|
+
shardingConfig: c,
|
|
96
|
+
guard: {
|
|
97
|
+
interval: u,
|
|
98
|
+
giveUpAfter: f.times(u, 3),
|
|
99
|
+
log: C
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
return m.mergeAll(o, m.effect(n, r), m.effect(i, a)).pipe(m.provideMerge(d));
|
|
103
|
+
}))).pipe(m.provide(M)), L = S.extraShardingLayers ?? [], R = L.length === 0 ? _.layer : m.mergeAll(_.layer, ...L), z = m.flatMap(I, (t) => {
|
|
104
|
+
let n = d.add(t, x.Sharding, e(d.get(t, x.Sharding)));
|
|
105
|
+
return m.succeedContext(S.dialectId === "sqlite" ? d.add(n, v.MessageStorage, P(d.get(n, v.MessageStorage))) : n);
|
|
106
|
+
}), B = u.layer.pipe(m.provide(u.layerStoreMemory));
|
|
107
|
+
return m.merge(R, B).pipe(m.provideMerge(z));
|
|
108
|
+
};
|
|
109
|
+
//#endregion
|
|
110
|
+
export { w as DEFAULT_RUNNER_LISTEN_PORT, O as detectWsrepCluster, D as parseShardLockMode, M as pollWorkflowStorage, j as resolveFailoverTuning, T as resolveRunnerIdentity, E as resolveRunnerStorage, k as resolveShardLockDisableAdvisory, A as shardLockLogFields, F as workflowEngineLayer };
|