@voltro/workflow 0.84.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1215 -0
- package/THIRD-PARTY-NOTICES.md +5 -151
- package/dist/admission.js +140 -0
- package/dist/admissionDrainer.js +705 -0
- package/dist/admissionGate.js +115 -0
- package/dist/admissionStore.js +347 -0
- package/dist/admissionSubmission.js +297 -0
- package/dist/admissionTables.js +137 -0
- package/dist/admissionTransaction.js +21 -0
- package/dist/awaitEvent.js +82 -0
- package/dist/budgetHold.js +136 -0
- package/dist/cancelOnPlan.js +75 -0
- package/dist/cancelOnSweep.js +188 -0
- package/dist/cancelPendingStart.js +50 -0
- package/dist/cluster.d.ts +304 -280
- package/dist/cluster.js +8 -2
- package/dist/clusterCron.js +11 -0
- package/dist/clusterDiagnostics.js +20 -0
- package/dist/clusterDiagnosticsSql.js +145 -0
- package/dist/clusterLayer.js +110 -0
- package/dist/clusterMigrationGate.js +61 -0
- package/dist/clusterRunnerLayer.js +21 -0
- package/dist/clusterTestSuite.d.ts +11 -11
- package/dist/clusterTestSuite.js +257 -257
- package/dist/controlMonitor.js +13 -0
- package/dist/controlRequest.js +367 -0
- package/dist/executionRefusal.js +162 -0
- package/dist/failureHandoff.js +183 -0
- package/dist/failureMessage.js +12 -0
- package/dist/failureRecovery.js +50 -0
- package/dist/failureReportStore.js +60 -0
- package/dist/flowControl.js +142 -0
- package/dist/forwardStartReceipts.js +41 -0
- package/dist/inMemoryLayer.js +7 -0
- package/dist/inMemoryRecorder.js +56 -0
- package/dist/index.d.ts +456 -360
- package/dist/index.js +49 -5
- package/dist/inspectWorkflow.js +101 -0
- package/dist/journalEncryption.js +147 -0
- package/dist/messages.js +92 -0
- package/dist/nondeterminism.js +51 -0
- package/dist/parentClosure.js +141 -0
- package/dist/parentClosureRecovery.js +54 -0
- package/dist/pendingSubmission.js +85 -0
- package/dist/prepareSubmissionDispatch.js +17 -0
- package/dist/preparedExecution.js +14 -0
- package/dist/primitives.d.ts +56 -31
- package/dist/primitives.js +225 -2
- package/dist/queuedCancellationRequest.js +364 -0
- package/dist/queuedControl.js +21 -0
- package/dist/recorder.js +11 -0
- package/dist/residentExecutionEvidence.js +64 -0
- package/dist/retryPolicy.js +57 -0
- package/dist/runEventsTable.js +17 -0
- package/dist/runGeneration.js +345 -0
- package/dist/runRecording.js +615 -0
- package/dist/runStepsTable.js +28 -0
- package/dist/runnerStorageGuard.js +29 -0
- package/dist/runsTable.js +42 -0
- package/dist/scheduleDeclaration.js +44 -0
- package/dist/signalHint.js +23 -0
- package/dist/signals.js +70 -0
- package/dist/signalsSuspending.js +57 -0
- package/dist/sqliteWorkflowOwnership.js +26 -0
- package/dist/stalenessSweep.js +191 -0
- package/dist/startContextsTable.js +20 -0
- package/dist/startReceipt.js +143 -0
- package/dist/submissionRecovery.js +122 -0
- package/dist/submissionRejection.js +10 -0
- package/dist/submissionRetry.js +71 -0
- package/dist/suspensionUnavailable.js +9 -0
- package/dist/updates.js +87 -0
- package/dist/watermarkTable.js +9 -0
- package/dist/webCrypto.js +17 -0
- package/dist/workflowDelivery.js +24 -0
- package/dist/workflowJournal.js +124 -0
- package/dist/workflowJournalReset.js +111 -0
- package/dist/workflowReadSelections.js +12 -0
- package/dist/workflowRestartDispatch.js +29 -0
- package/dist/workflowRestartObservation.js +60 -0
- package/dist/workflowRestartPreparation.js +109 -0
- package/dist/workflowRestartReservation.js +426 -0
- package/dist/workflowResumeDispatch.js +29 -0
- package/package.json +6 -9
- package/dist/cluster-i3frC3Da.js +0 -646
- package/dist/clusterDiagnostics-CGUtnJkX.js +0 -2084
- package/dist/journalEncryption--n5VDdYO.js +0 -148
- package/dist/primitives-Btjhjm8P.js +0 -483
- package/dist/src-DP-lKqaG.js +0 -3867
|
@@ -0,0 +1,705 @@
|
|
|
1
|
+
import { loggableCause as e, sealErrorText as t } from "./journalEncryption.js";
|
|
2
|
+
import { isFinishExpired as n, isStartExpired as r, resolveAdmissionKeys as i } from "./admission.js";
|
|
3
|
+
import { recordWorkflowFailureInTransaction as a } from "./failureReportStore.js";
|
|
4
|
+
import { handoffWorkflowFailureInTransaction as o, isFailureDeliveryPending as s } from "./failureHandoff.js";
|
|
5
|
+
import { acknowledgeAdmissionSubmissionInTransaction as c, beginAdmissionSubmissionDispatchInTransaction as l, newAdmissionSubmissionId as u, noteAdmissionSubmissionFailure as d, readAdmissionSubmission as f, rejectAdmissionSubmissionInTransaction as p } from "./admissionSubmission.js";
|
|
6
|
+
import { settlePendingStartReceipts as m } from "./startReceipt.js";
|
|
7
|
+
import { PENDING_TABLE as h, deadLetterIntent as g, decodePendingPayload as _, deleteIntent as v, encodePendingPayload as y, evaluateAdmission as b, noteIntentAttempt as x, pendingWindow as S, persistDefer as C, readPausedWorkflows as w, recordAdmission as T, releaseLease as E, retryBackoffMs as D, toPendingIntent as O } from "./admissionStore.js";
|
|
8
|
+
import { FLOW_CONTROL_FINISH_TIMEOUT as k } from "./controlRequest.js";
|
|
9
|
+
import { takeAdmittedSubmissionInTransaction as A, takePendingAdmissionSubmissionInTransaction as j } from "./pendingSubmission.js";
|
|
10
|
+
import { admitStart as M } from "./admissionGate.js";
|
|
11
|
+
import { prepareSubmissionDispatch as N } from "./prepareSubmissionDispatch.js";
|
|
12
|
+
import { recoverAdmissionSubmissions as P } from "./submissionRecovery.js";
|
|
13
|
+
import { recoverWorkflowFailures as F } from "./failureRecovery.js";
|
|
14
|
+
import { recoverWorkflowParentClosures as I } from "./parentClosureRecovery.js";
|
|
15
|
+
import { forwardStartReceipts as L } from "./forwardStartReceipts.js";
|
|
16
|
+
import { Schema as R } from "effect";
|
|
17
|
+
import { eq as z } from "@voltro/database";
|
|
18
|
+
import { randomUUID as B } from "node:crypto";
|
|
19
|
+
//#region src/admissionDrainer.ts
|
|
20
|
+
var V = {
|
|
21
|
+
scanned: 0,
|
|
22
|
+
admitted: 0,
|
|
23
|
+
deferred: 0,
|
|
24
|
+
dropped: 0,
|
|
25
|
+
skipped: 0,
|
|
26
|
+
expired: 0,
|
|
27
|
+
evicted: 0,
|
|
28
|
+
failed: 0,
|
|
29
|
+
deadLettered: 0,
|
|
30
|
+
timedOut: 0,
|
|
31
|
+
examined: !1
|
|
32
|
+
}, H = 200, U = class extends Error {
|
|
33
|
+
submissionId;
|
|
34
|
+
rejected;
|
|
35
|
+
constructor(e, t, n = !1) {
|
|
36
|
+
super(n ? `Submission ${e} rejected; failure report retained` : `Submission ${e} remains unacknowledged`, { cause: t }), this.submissionId = e, this.rejected = n;
|
|
37
|
+
}
|
|
38
|
+
}, W = (e) => e instanceof U && !e.rejected, G = async (n, r = {}) => {
|
|
39
|
+
let i = n.now ?? (() => Date.now()), a = n.log, o = r.batchSize ?? 200, s, c, l, u, d, f;
|
|
40
|
+
try {
|
|
41
|
+
u = await P(n, i(), o, n.deadLetterAfterAttempts ?? 50), d = await F(n, i(), o), f = await I(n, i(), o), s = await (n.readPauses?.() ?? w(n.store));
|
|
42
|
+
let e = r.skipPending === !0 && !d.hasPending ? {
|
|
43
|
+
due: [],
|
|
44
|
+
nextDueAt: void 0
|
|
45
|
+
} : await S(n.store, i(), o);
|
|
46
|
+
c = e.due, l = u.nextDueAt === void 0 ? e.nextDueAt : Math.min(e.nextDueAt ?? u.nextDueAt, u.nextDueAt), d.nextDueAt !== void 0 && (l = Math.min(l ?? d.nextDueAt, d.nextDueAt)), f.nextDueAt !== void 0 && (l = Math.min(l ?? f.nextDueAt, f.nextDueAt));
|
|
47
|
+
} catch (e) {
|
|
48
|
+
return a?.warn?.("workflow flow control: could not read the admission queue", { cause: String(e) }), {
|
|
49
|
+
...V,
|
|
50
|
+
scanned: (u?.scanned ?? 0) + (d?.scanned ?? 0) + (f?.scanned ?? 0),
|
|
51
|
+
admitted: u?.admitted ?? 0,
|
|
52
|
+
deadLettered: u?.rejected ?? 0,
|
|
53
|
+
failed: (u?.failed ?? 0) + (d?.failed ?? 0) + (f?.failed ?? 0) + 1,
|
|
54
|
+
nextDueAt: i()
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
let p = {
|
|
58
|
+
scanned: c.length + u.scanned + d.scanned + f.scanned,
|
|
59
|
+
admitted: u.admitted,
|
|
60
|
+
deferred: 0,
|
|
61
|
+
dropped: 0,
|
|
62
|
+
skipped: 0,
|
|
63
|
+
expired: 0,
|
|
64
|
+
evicted: 0,
|
|
65
|
+
failed: u.failed + d.failed + f.failed,
|
|
66
|
+
deadLettered: u.rejected,
|
|
67
|
+
timedOut: 0
|
|
68
|
+
}, m = /* @__PURE__ */ new Map(), g = [];
|
|
69
|
+
for (let e of c) if (e.mode === "batch") {
|
|
70
|
+
let t = `${e.workflowName}\u0000${e.controlKey}`, n = m.get(t);
|
|
71
|
+
n === void 0 ? m.set(t, [e]) : n.push(e);
|
|
72
|
+
} else g.push(e);
|
|
73
|
+
let _ = n.deadLetterAfterAttempts === void 0 ? 50 : n.deadLetterAfterAttempts, v = (e, t, r) => {
|
|
74
|
+
if (n.isPermanentStartError?.(t) === !0) return `pending ${r} was dead-lettered after a permanent start failure: ${String(t)}`;
|
|
75
|
+
if (_ !== !1 && e.attempts + 1 >= _) return `pending ${r} was dead-lettered after ${e.attempts + 1} failed attempts: ${String(t)}`;
|
|
76
|
+
};
|
|
77
|
+
for (let t of g) try {
|
|
78
|
+
await Y(n, t, s, p, i());
|
|
79
|
+
} catch (r) {
|
|
80
|
+
if (p.failed++, r instanceof U) {
|
|
81
|
+
r.rejected && p.deadLettered++, a?.warn?.(r.message, {
|
|
82
|
+
submissionId: r.submissionId,
|
|
83
|
+
cause: e(t.workflowName, r.cause)
|
|
84
|
+
});
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
let o = v(t, r, "intent");
|
|
88
|
+
if (o !== void 0) {
|
|
89
|
+
let s = i(), c = String(r);
|
|
90
|
+
if (await q(n, [t], o, c, s, !1).catch((e) => (a?.warn?.("workflow flow control: intent could not be dead-lettered with its report", {
|
|
91
|
+
workflow: t.workflowName,
|
|
92
|
+
intent: t.id,
|
|
93
|
+
cause: String(e)
|
|
94
|
+
}), 0)) > 0) {
|
|
95
|
+
p.deadLettered++, a?.warn?.("workflow flow control: intent dead-lettered", {
|
|
96
|
+
workflow: t.workflowName,
|
|
97
|
+
intent: t.id,
|
|
98
|
+
cause: e(t.workflowName, c)
|
|
99
|
+
});
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
a?.warn?.("workflow flow control: an intent failed to drain", {
|
|
104
|
+
workflow: t.workflowName,
|
|
105
|
+
intent: t.id,
|
|
106
|
+
cause: e(t.workflowName, r)
|
|
107
|
+
}), await x(n.store, t, String(r), i()).catch(() => {});
|
|
108
|
+
}
|
|
109
|
+
for (let r of m.values()) try {
|
|
110
|
+
await Z(n, r, s, p, i());
|
|
111
|
+
} catch (o) {
|
|
112
|
+
if (p.failed++, o instanceof U) {
|
|
113
|
+
o.rejected && p.deadLettered++, a?.warn?.(o.message, {
|
|
114
|
+
submissionId: o.submissionId,
|
|
115
|
+
cause: e(r[0]?.workflowName, o.cause)
|
|
116
|
+
});
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
let s = r[0], c = s === void 0 ? void 0 : v(s, o, "batch");
|
|
120
|
+
if (s !== void 0 && c !== void 0) {
|
|
121
|
+
let t = i(), l = String(o);
|
|
122
|
+
if (await q(n, r, c, l, t, !0).catch((e) => (a?.warn?.("workflow flow control: batch could not be dead-lettered with its report", {
|
|
123
|
+
workflow: s.workflowName,
|
|
124
|
+
intents: r.map((e) => e.id),
|
|
125
|
+
size: r.length,
|
|
126
|
+
cause: String(e)
|
|
127
|
+
}), 0)) === r.length) {
|
|
128
|
+
p.deadLettered += r.length, a?.warn?.("workflow flow control: batch dead-lettered", {
|
|
129
|
+
workflow: s.workflowName,
|
|
130
|
+
intents: r.map((e) => e.id),
|
|
131
|
+
size: r.length,
|
|
132
|
+
cause: e(s.workflowName, l)
|
|
133
|
+
});
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
a?.warn?.("workflow flow control: a batch failed to drain", {
|
|
138
|
+
workflow: s?.workflowName,
|
|
139
|
+
size: r.length,
|
|
140
|
+
cause: e(s?.workflowName, o)
|
|
141
|
+
});
|
|
142
|
+
let l = i(), u = Math.random();
|
|
143
|
+
await n.transaction(async (e) => {
|
|
144
|
+
let n = [];
|
|
145
|
+
for (let t of r) {
|
|
146
|
+
let r = await e.query({
|
|
147
|
+
table: h,
|
|
148
|
+
predicate: z("id", t.id),
|
|
149
|
+
order: [],
|
|
150
|
+
take: 1,
|
|
151
|
+
skip: void 0,
|
|
152
|
+
projection: void 0,
|
|
153
|
+
lock: "update"
|
|
154
|
+
});
|
|
155
|
+
if (r[0] === void 0) return;
|
|
156
|
+
let i = O(r[0]);
|
|
157
|
+
if (i.workflowName !== t.workflowName || i.tenantId !== t.tenantId || i.mode !== t.mode || i.firstSeenAt !== t.firstSeenAt || i.dueAt !== t.dueAt || i.deadLetteredAt !== void 0) return;
|
|
158
|
+
n.push(i);
|
|
159
|
+
}
|
|
160
|
+
let i = new Date(l + D(Math.max(...n.map((e) => e.attempts + 1)), () => u));
|
|
161
|
+
for (let r of n) if (await e.update("_voltro_workflow_pending", r.id, {
|
|
162
|
+
attempts: r.attempts + 1,
|
|
163
|
+
lastError: t(r.workflowName, String(o).slice(0, 500)),
|
|
164
|
+
dueAt: i,
|
|
165
|
+
updatedAt: new Date(l)
|
|
166
|
+
}) === null) throw Error("Batch input disappeared while recording the shared retry deadline");
|
|
167
|
+
}).catch((e) => a?.warn?.("workflow flow control: batch retry could not be persisted", {
|
|
168
|
+
workflow: s?.workflowName,
|
|
169
|
+
intents: r.map((e) => e.id),
|
|
170
|
+
cause: String(e)
|
|
171
|
+
}));
|
|
172
|
+
}
|
|
173
|
+
let y = await te(n, i()).catch(() => ({ cancelled: 0 }));
|
|
174
|
+
return p.timedOut = y.cancelled, y.nextDeadlineAt !== void 0 && (l === void 0 || y.nextDeadlineAt < l) && (l = y.nextDeadlineAt), {
|
|
175
|
+
...p,
|
|
176
|
+
examined: r.skipPending !== !0 || u.examined || d.examined || f.examined,
|
|
177
|
+
...l === void 0 ? {} : { nextDueAt: l }
|
|
178
|
+
};
|
|
179
|
+
}, K = async (e, t, n) => {
|
|
180
|
+
let r = t.map((e) => ({
|
|
181
|
+
intent: e,
|
|
182
|
+
failureId: `wff_${B()}`
|
|
183
|
+
}));
|
|
184
|
+
return e.transaction(async (e) => {
|
|
185
|
+
let t = 0;
|
|
186
|
+
for (let { intent: i, failureId: o } of r) {
|
|
187
|
+
let r = (await e.query({
|
|
188
|
+
table: h,
|
|
189
|
+
predicate: z("id", i.id),
|
|
190
|
+
order: [],
|
|
191
|
+
take: 1,
|
|
192
|
+
skip: void 0,
|
|
193
|
+
projection: void 0,
|
|
194
|
+
lock: "update"
|
|
195
|
+
}))[0];
|
|
196
|
+
if (r === void 0) continue;
|
|
197
|
+
let s = O(r);
|
|
198
|
+
if (s.workflowName !== i.workflowName || s.tenantId !== i.tenantId) throw Error("Abandoned queue selection belongs to another workflow or tenant");
|
|
199
|
+
if (s.dueAt > n || s.deadLetteredAt !== void 0) continue;
|
|
200
|
+
let c = `workflow '${s.workflowName}' no longer has its queued declaration`;
|
|
201
|
+
if (await a(e, {
|
|
202
|
+
id: o,
|
|
203
|
+
sourceKind: "intent",
|
|
204
|
+
sourceId: s.id,
|
|
205
|
+
workflowName: s.workflowName,
|
|
206
|
+
tenantId: s.tenantId,
|
|
207
|
+
handler: null,
|
|
208
|
+
callerContext: s.callerContext,
|
|
209
|
+
report: {
|
|
210
|
+
workflow: s.workflowName,
|
|
211
|
+
payload: s.payload,
|
|
212
|
+
reason: c,
|
|
213
|
+
failedAt: n,
|
|
214
|
+
errorTag: null,
|
|
215
|
+
errorMessage: null,
|
|
216
|
+
runId: null,
|
|
217
|
+
executionId: null
|
|
218
|
+
}
|
|
219
|
+
}), await m(e, {
|
|
220
|
+
pendingId: s.id,
|
|
221
|
+
workflowName: s.workflowName,
|
|
222
|
+
tenantId: s.tenantId,
|
|
223
|
+
terminal: {
|
|
224
|
+
kind: "abandoned",
|
|
225
|
+
at: n,
|
|
226
|
+
failureId: o
|
|
227
|
+
}
|
|
228
|
+
}), !await e.delete("_voltro_workflow_pending", s.id)) throw Error("Abandoned queue row disappeared before deletion");
|
|
229
|
+
t++;
|
|
230
|
+
}
|
|
231
|
+
return t;
|
|
232
|
+
});
|
|
233
|
+
}, q = async (e, t, n, r, i, c) => {
|
|
234
|
+
let l = `wff_${B()}`;
|
|
235
|
+
return e.transaction(async (u) => {
|
|
236
|
+
let d = [];
|
|
237
|
+
for (let e of t) {
|
|
238
|
+
let t = await u.query({
|
|
239
|
+
table: h,
|
|
240
|
+
predicate: z("id", e.id),
|
|
241
|
+
order: [],
|
|
242
|
+
take: 1,
|
|
243
|
+
skip: void 0,
|
|
244
|
+
projection: void 0,
|
|
245
|
+
lock: "update"
|
|
246
|
+
});
|
|
247
|
+
if (t[0] === void 0) return 0;
|
|
248
|
+
let n = O(t[0]);
|
|
249
|
+
if (n.workflowName !== e.workflowName || n.tenantId !== e.tenantId) throw Error("Dead-letter selection belongs to another workflow or tenant");
|
|
250
|
+
if (n.deadLetteredAt !== void 0) return 0;
|
|
251
|
+
d.push(n);
|
|
252
|
+
}
|
|
253
|
+
let f = d[0];
|
|
254
|
+
if (f === void 0) return 0;
|
|
255
|
+
let p = await Promise.all(d.map((e) => s(u, e.id)));
|
|
256
|
+
if (p.some(Boolean) && !p.every(Boolean)) throw Error("Dead-lettered batch mixes failure delivery and ordinary callers; reconciliation required");
|
|
257
|
+
let _ = e.controls.get(f.workflowName);
|
|
258
|
+
await a(u, {
|
|
259
|
+
id: l,
|
|
260
|
+
sourceKind: "intent",
|
|
261
|
+
sourceId: f.id,
|
|
262
|
+
workflowName: f.workflowName,
|
|
263
|
+
tenantId: f.tenantId,
|
|
264
|
+
handler: p.every(Boolean) ? null : _?.onFailure ?? null,
|
|
265
|
+
callerContext: f.callerContext,
|
|
266
|
+
report: {
|
|
267
|
+
workflow: f.workflowName,
|
|
268
|
+
payload: c ? { items: d.map((e) => e.payload) } : f.payload,
|
|
269
|
+
reason: n,
|
|
270
|
+
failedAt: i,
|
|
271
|
+
errorTag: null,
|
|
272
|
+
errorMessage: r.slice(0, 500),
|
|
273
|
+
runId: null,
|
|
274
|
+
executionId: null
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
for (let e of d) if (await m(u, {
|
|
278
|
+
pendingId: e.id,
|
|
279
|
+
workflowName: e.workflowName,
|
|
280
|
+
tenantId: e.tenantId,
|
|
281
|
+
terminal: {
|
|
282
|
+
kind: "abandoned",
|
|
283
|
+
at: i,
|
|
284
|
+
failureId: l
|
|
285
|
+
}
|
|
286
|
+
}), !await g(u, e, r, i)) throw Error("Dead-lettered queue row disappeared before marking");
|
|
287
|
+
return await o(u, l, i), d.length;
|
|
288
|
+
});
|
|
289
|
+
}, J = async (e, t, n, c, l) => {
|
|
290
|
+
let u = `wff_${B()}`;
|
|
291
|
+
return e.transaction(async (e) => {
|
|
292
|
+
let d = [];
|
|
293
|
+
for (let r of t) {
|
|
294
|
+
let t = await e.query({
|
|
295
|
+
table: h,
|
|
296
|
+
predicate: z("id", r.id),
|
|
297
|
+
order: [],
|
|
298
|
+
take: 1,
|
|
299
|
+
skip: void 0,
|
|
300
|
+
projection: void 0,
|
|
301
|
+
lock: "update"
|
|
302
|
+
});
|
|
303
|
+
if (t[0] === void 0) return 0;
|
|
304
|
+
let i = O(t[0]);
|
|
305
|
+
if (i.workflowName !== n.workflowName || i.tenantId !== r.tenantId) throw Error("Expired queue selection belongs to another workflow or tenant");
|
|
306
|
+
if (i.mode !== r.mode || i.dueAt > c || i.deadLetteredAt !== void 0) return 0;
|
|
307
|
+
d.push(i);
|
|
308
|
+
}
|
|
309
|
+
d.sort((e, t) => e.firstSeenAt - t.firstSeenAt || e.id.localeCompare(t.id));
|
|
310
|
+
let f = d[0];
|
|
311
|
+
if (f === void 0 || !r(n, f, c)) return 0;
|
|
312
|
+
let p = await Promise.all(d.map((t) => s(e, t.id)));
|
|
313
|
+
if (p.some(Boolean) && !p.every(Boolean)) throw Error("Expired batch mixes failure delivery and ordinary callers; reconciliation required");
|
|
314
|
+
let g = `timeouts.start (${n.timeouts?.startMs}ms) elapsed while ${d.length} queued input(s) waited`;
|
|
315
|
+
await a(e, {
|
|
316
|
+
id: u,
|
|
317
|
+
sourceKind: "intent",
|
|
318
|
+
sourceId: f.id,
|
|
319
|
+
workflowName: f.workflowName,
|
|
320
|
+
tenantId: f.tenantId,
|
|
321
|
+
handler: p.every(Boolean) ? null : n.onFailure ?? null,
|
|
322
|
+
callerContext: f.callerContext,
|
|
323
|
+
report: {
|
|
324
|
+
workflow: f.workflowName,
|
|
325
|
+
payload: l ? { items: d.map((e) => e.payload) } : f.payload,
|
|
326
|
+
reason: g,
|
|
327
|
+
failedAt: c,
|
|
328
|
+
errorTag: null,
|
|
329
|
+
errorMessage: null,
|
|
330
|
+
runId: null,
|
|
331
|
+
executionId: null
|
|
332
|
+
}
|
|
333
|
+
}), await T({
|
|
334
|
+
store: e,
|
|
335
|
+
control: n,
|
|
336
|
+
keys: i(n, _(e, f.workflowName, f.payload)),
|
|
337
|
+
outcome: "expired",
|
|
338
|
+
mode: f.mode,
|
|
339
|
+
reason: g,
|
|
340
|
+
collapsed: d.reduce((e, t) => e + t.collapsed, 0),
|
|
341
|
+
waitedMs: c - f.firstSeenAt,
|
|
342
|
+
executionId: null,
|
|
343
|
+
runId: null,
|
|
344
|
+
tenantId: f.tenantId,
|
|
345
|
+
now: c
|
|
346
|
+
});
|
|
347
|
+
for (let t of d) if (await m(e, {
|
|
348
|
+
pendingId: t.id,
|
|
349
|
+
workflowName: t.workflowName,
|
|
350
|
+
tenantId: t.tenantId,
|
|
351
|
+
terminal: {
|
|
352
|
+
kind: "abandoned",
|
|
353
|
+
at: c,
|
|
354
|
+
failureId: u
|
|
355
|
+
}
|
|
356
|
+
}), !await e.delete("_voltro_workflow_pending", t.id)) throw Error("Expired queue row disappeared before deletion");
|
|
357
|
+
return await o(e, u, c), d.length;
|
|
358
|
+
});
|
|
359
|
+
}, Y = async (e, t, n, a, o) => {
|
|
360
|
+
if (t = {
|
|
361
|
+
...t,
|
|
362
|
+
payload: _(e.store, t.workflowName, t.payload)
|
|
363
|
+
}, t.mode === "delayed") return X(e, t, n, a, o);
|
|
364
|
+
let s = e.controls.get(t.workflowName);
|
|
365
|
+
if (s === void 0) {
|
|
366
|
+
a.expired += await K(e, [t], o);
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
if (r(s, t, o)) {
|
|
370
|
+
a.expired += await J(e, [t], s, o, !1);
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
let c = i(s, t.payload), l = await b({
|
|
374
|
+
store: e.store,
|
|
375
|
+
control: s,
|
|
376
|
+
keys: c,
|
|
377
|
+
paused: n.has(t.workflowName),
|
|
378
|
+
now: o,
|
|
379
|
+
reconsidering: !0
|
|
380
|
+
});
|
|
381
|
+
await Q(e, {
|
|
382
|
+
control: s,
|
|
383
|
+
intent: t,
|
|
384
|
+
decision: l,
|
|
385
|
+
keys: c,
|
|
386
|
+
counters: a,
|
|
387
|
+
now: o,
|
|
388
|
+
paused: n,
|
|
389
|
+
items: void 0
|
|
390
|
+
});
|
|
391
|
+
}, X = async (e, t, n, r, i) => {
|
|
392
|
+
if (e.registeredWorkflows !== void 0 && !e.registeredWorkflows.has(t.workflowName)) {
|
|
393
|
+
r.expired += await K(e, [t], i);
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
let a = e.controls.get(t.workflowName) ?? { workflowName: t.workflowName }, o = u(), s = await e.transaction(async (e) => {
|
|
397
|
+
let r = await e.query({
|
|
398
|
+
table: h,
|
|
399
|
+
predicate: z("id", t.id),
|
|
400
|
+
lock: "update",
|
|
401
|
+
order: [],
|
|
402
|
+
take: 1,
|
|
403
|
+
skip: void 0,
|
|
404
|
+
projection: void 0
|
|
405
|
+
});
|
|
406
|
+
if (r[0] === void 0) return;
|
|
407
|
+
let s = O(r[0]);
|
|
408
|
+
if (s.workflowName !== t.workflowName || s.tenantId !== t.tenantId) throw Error("Delayed arrival belongs to a different workflow or tenant");
|
|
409
|
+
if (s.mode !== "delayed" || s.deadLetteredAt !== void 0 || s.dueAt > i) return;
|
|
410
|
+
let c = await M({
|
|
411
|
+
store: e,
|
|
412
|
+
control: a,
|
|
413
|
+
payload: _(e, s.workflowName, s.payload),
|
|
414
|
+
callerContext: s.callerContext,
|
|
415
|
+
tenantId: s.tenantId,
|
|
416
|
+
paused: n.has(s.workflowName),
|
|
417
|
+
now: i,
|
|
418
|
+
waiting: !1
|
|
419
|
+
});
|
|
420
|
+
if (c.kind === "queued" && (await L(e, {
|
|
421
|
+
from: s.id,
|
|
422
|
+
to: c.intentId,
|
|
423
|
+
workflowName: s.workflowName,
|
|
424
|
+
tenantId: s.tenantId
|
|
425
|
+
}), !await v(e, s.id))) throw Error("Delayed arrival disappeared during queue transfer");
|
|
426
|
+
if (c.kind === "start") {
|
|
427
|
+
if (c.ledgerId === void 0) throw Error("Delayed arrival has no admission reservation");
|
|
428
|
+
let t = a.batch === void 0 ? y(e, s.workflowName, c.payload) : { items: R.decodeUnknownSync(R.Struct({ items: R.Array(R.Unknown) }))(c.payload).items.map((t) => y(e, s.workflowName, t)) };
|
|
429
|
+
return {
|
|
430
|
+
kind: "submission",
|
|
431
|
+
submission: await A(e, {
|
|
432
|
+
id: o,
|
|
433
|
+
workflowName: s.workflowName,
|
|
434
|
+
tenantId: s.tenantId,
|
|
435
|
+
ledgerId: c.ledgerId,
|
|
436
|
+
payload: t,
|
|
437
|
+
callerContext: s.callerContext,
|
|
438
|
+
intentIds: [.../* @__PURE__ */ new Set([s.id, ...c.consume])],
|
|
439
|
+
eviction: c.evict ?? null
|
|
440
|
+
}),
|
|
441
|
+
keys: c.keys
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
if ((c.kind === "dropped" || c.kind === "skipped") && (await m(e, {
|
|
445
|
+
pendingId: s.id,
|
|
446
|
+
workflowName: s.workflowName,
|
|
447
|
+
tenantId: s.tenantId,
|
|
448
|
+
terminal: {
|
|
449
|
+
...c,
|
|
450
|
+
at: i
|
|
451
|
+
}
|
|
452
|
+
}), !await v(e, s.id))) throw Error("Delayed arrival disappeared while recording its outcome");
|
|
453
|
+
return c;
|
|
454
|
+
});
|
|
455
|
+
s !== void 0 && (s.kind === "queued" ? r.deferred++ : s.kind === "dropped" ? r.dropped++ : s.kind === "skipped" ? r.skipped++ : await $(e, s.submission, a, r, i, a.batch !== void 0, s.keys));
|
|
456
|
+
}, Z = async (e, t, n, a, o) => {
|
|
457
|
+
t = t.map((t) => ({
|
|
458
|
+
...t,
|
|
459
|
+
payload: _(e.store, t.workflowName, t.payload)
|
|
460
|
+
}));
|
|
461
|
+
let s = t[0];
|
|
462
|
+
if (s === void 0) return;
|
|
463
|
+
let c = e.controls.get(s.workflowName);
|
|
464
|
+
if (c === void 0 || c.batch === void 0) {
|
|
465
|
+
a.expired += await K(e, t, o);
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
let l = [...t].sort((e, t) => e.firstSeenAt - t.firstSeenAt), u = l[0];
|
|
469
|
+
if (r(c, u, o)) {
|
|
470
|
+
a.expired += await J(e, l, c, o, !0);
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
let d = i(c, u.payload);
|
|
474
|
+
await Q(e, {
|
|
475
|
+
control: c,
|
|
476
|
+
intent: u,
|
|
477
|
+
decision: await b({
|
|
478
|
+
store: e.store,
|
|
479
|
+
control: c,
|
|
480
|
+
keys: d,
|
|
481
|
+
paused: n.has(s.workflowName),
|
|
482
|
+
now: o,
|
|
483
|
+
reconsidering: !0
|
|
484
|
+
}),
|
|
485
|
+
keys: d,
|
|
486
|
+
counters: a,
|
|
487
|
+
now: o,
|
|
488
|
+
paused: n,
|
|
489
|
+
items: l
|
|
490
|
+
});
|
|
491
|
+
}, Q = async (e, t) => {
|
|
492
|
+
let { control: n, intent: r, decision: a, keys: o, counters: s, now: c, items: l, paused: d } = t;
|
|
493
|
+
if (a.kind === "defer") {
|
|
494
|
+
await C({
|
|
495
|
+
store: e.store,
|
|
496
|
+
control: n,
|
|
497
|
+
decision: a,
|
|
498
|
+
payload: r.payload,
|
|
499
|
+
callerContext: r.callerContext,
|
|
500
|
+
tenantId: r.tenantId,
|
|
501
|
+
now: c,
|
|
502
|
+
existingId: r.id
|
|
503
|
+
}), s.deferred++;
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
if (a.kind === "drop" || a.kind === "skip") {
|
|
507
|
+
let t = await e.transaction(async (e) => {
|
|
508
|
+
let t = [];
|
|
509
|
+
for (let i of l ?? [r]) {
|
|
510
|
+
let a = await e.query({
|
|
511
|
+
table: h,
|
|
512
|
+
predicate: z("id", i.id),
|
|
513
|
+
lock: "update",
|
|
514
|
+
order: [],
|
|
515
|
+
take: 1,
|
|
516
|
+
skip: void 0,
|
|
517
|
+
projection: void 0
|
|
518
|
+
});
|
|
519
|
+
if (a[0] === void 0) return;
|
|
520
|
+
let o = O(a[0]);
|
|
521
|
+
if (o.workflowName !== n.workflowName || o.tenantId !== r.tenantId) throw Error("Terminal queue selection belongs to a different workflow or tenant");
|
|
522
|
+
if (o.dueAt > c || o.deadLetteredAt !== void 0 || o.mode !== i.mode) return;
|
|
523
|
+
t.push(o);
|
|
524
|
+
}
|
|
525
|
+
t.sort((e, t) => e.firstSeenAt - t.firstSeenAt);
|
|
526
|
+
let a = t[0], o = i(n, _(e, n.workflowName, a.payload)), s = await b({
|
|
527
|
+
store: e,
|
|
528
|
+
control: n,
|
|
529
|
+
keys: o,
|
|
530
|
+
paused: d.has(n.workflowName) || (await w(e)).has(n.workflowName),
|
|
531
|
+
now: c,
|
|
532
|
+
reconsidering: !0
|
|
533
|
+
});
|
|
534
|
+
if (s.kind !== "drop" && s.kind !== "skip") return;
|
|
535
|
+
let u = s.kind === "drop" ? {
|
|
536
|
+
kind: "dropped",
|
|
537
|
+
at: c,
|
|
538
|
+
retryAfterMs: s.retryAfterMs
|
|
539
|
+
} : {
|
|
540
|
+
kind: "skipped",
|
|
541
|
+
at: c,
|
|
542
|
+
executionId: s.incumbent.executionId,
|
|
543
|
+
runId: s.incumbent.runId
|
|
544
|
+
};
|
|
545
|
+
await T({
|
|
546
|
+
store: e,
|
|
547
|
+
control: n,
|
|
548
|
+
keys: o,
|
|
549
|
+
outcome: u.kind,
|
|
550
|
+
mode: u.kind === "dropped" ? "rateLimit" : "singleton",
|
|
551
|
+
reason: u.kind === "dropped" ? `rate limit reached while queued; retry after ${u.retryAfterMs}ms` : `singleton key held by ${u.executionId}`,
|
|
552
|
+
collapsed: t.reduce((e, t) => e + t.collapsed, 0),
|
|
553
|
+
waitedMs: c - a.firstSeenAt,
|
|
554
|
+
executionId: u.kind === "skipped" ? u.executionId : null,
|
|
555
|
+
runId: u.kind === "skipped" && u.runId || null,
|
|
556
|
+
tenantId: r.tenantId,
|
|
557
|
+
now: c
|
|
558
|
+
});
|
|
559
|
+
for (let n of t) if (await m(e, {
|
|
560
|
+
pendingId: n.id,
|
|
561
|
+
workflowName: n.workflowName,
|
|
562
|
+
tenantId: n.tenantId,
|
|
563
|
+
terminal: u
|
|
564
|
+
}), !await v(e, n.id)) throw Error("Queued input disappeared while recording its terminal outcome");
|
|
565
|
+
return u.kind;
|
|
566
|
+
});
|
|
567
|
+
t !== void 0 && s[t]++;
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
let f = u(), p = await e.transaction((t) => j(t, {
|
|
571
|
+
id: f,
|
|
572
|
+
workflowName: n.workflowName,
|
|
573
|
+
tenantId: r.tenantId,
|
|
574
|
+
intentIds: (l ?? [r]).map((e) => e.id),
|
|
575
|
+
now: c,
|
|
576
|
+
prepare: (t) => {
|
|
577
|
+
let r = [...t].sort((e, t) => e.firstSeenAt - t.firstSeenAt), i = r[0], a = _(e.store, n.workflowName, i.payload);
|
|
578
|
+
return {
|
|
579
|
+
control: n,
|
|
580
|
+
keyPayload: a,
|
|
581
|
+
encodedPayload: l === void 0 ? i.payload : { items: r.map((e) => e.payload) },
|
|
582
|
+
callerContext: i.callerContext,
|
|
583
|
+
paused: d.has(n.workflowName),
|
|
584
|
+
firstSeenAt: i.firstSeenAt,
|
|
585
|
+
collapsed: r.reduce((e, t) => e + t.collapsed, 0),
|
|
586
|
+
mode: i.mode
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
}));
|
|
590
|
+
p.kind !== "selection-unavailable" && (p.kind === "not-admitted" ? s.deferred++ : await $(e, p.submission, n, s, c, l !== void 0, o));
|
|
591
|
+
}, ee = async (e, t, n, r, i) => {
|
|
592
|
+
let a = await e.transaction((e) => f(e, t.id));
|
|
593
|
+
if (a === void 0 || !a.receiptsBound || a.workflowName !== n.workflowName || a.workflowName !== t.workflowName || a.tenantId !== t.tenantId || a.ledgerId !== t.ledgerId) throw Error("Admission acceptance callback no longer has its resident submission owner");
|
|
594
|
+
if (a.executionId !== null) {
|
|
595
|
+
if (a.failureId !== null || a.executionId.length === 0 || a.executionId !== a.preparedExecutionId) throw Error("Admission acceptance has contradictory resident execution evidence");
|
|
596
|
+
return a.executionId;
|
|
597
|
+
}
|
|
598
|
+
await $(e, a, n, {
|
|
599
|
+
admitted: 0,
|
|
600
|
+
deferred: 0,
|
|
601
|
+
dropped: 0,
|
|
602
|
+
skipped: 0,
|
|
603
|
+
expired: 0,
|
|
604
|
+
evicted: 0
|
|
605
|
+
}, i, n.batch !== void 0, r);
|
|
606
|
+
let o = await e.transaction((e) => f(e, t.id));
|
|
607
|
+
if (o?.executionId == null || o.executionId !== o.preparedExecutionId) throw Error("Admission submission has no confirmed engine acceptance");
|
|
608
|
+
return o.executionId;
|
|
609
|
+
}, $ = async (e, t, n, r, i, a, o) => {
|
|
610
|
+
let s = t.dispatchAttempts === 0 ? 0 : void 0, u = t.retry.recovery?.attempts ?? 0;
|
|
611
|
+
try {
|
|
612
|
+
if (e.acquireSubmissionCapacity !== void 0 && !await e.acquireSubmissionCapacity(t)) {
|
|
613
|
+
r.deferred++;
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
let d = a ? { items: R.decodeUnknownSync(R.Struct({ items: R.Array(R.Unknown) }))(t.payload).items.map((t) => _(e.store, n.workflowName, t)) } : _(e.store, n.workflowName, t.payload), f = await N(e, t, d), p = await e.transaction((n) => l(n, t.id, e.now?.() ?? Date.now()));
|
|
617
|
+
if (p === null) return;
|
|
618
|
+
u = p.retry.recovery?.attempts ?? 0, s = p.dispatchAttempts === 1 ? 1 : void 0;
|
|
619
|
+
let m = await e.startAdmitted(n.workflowName, d, t.callerContext, f);
|
|
620
|
+
if (s = void 0, m.executionId !== f) throw Error("Engine returned a different prepared execution identity");
|
|
621
|
+
t.eviction !== null && (await e.cancelRun({
|
|
622
|
+
workflowName: n.workflowName,
|
|
623
|
+
runId: t.eviction.runId,
|
|
624
|
+
executionId: t.eviction.executionId,
|
|
625
|
+
reason: "cancelled to make room for a newer start (singleton mode 'cancel')",
|
|
626
|
+
detail: {
|
|
627
|
+
source: "flow-control",
|
|
628
|
+
mode: "singleton"
|
|
629
|
+
}
|
|
630
|
+
}), await E(e.store, t.eviction.executionId, i), await T({
|
|
631
|
+
store: e.store,
|
|
632
|
+
control: n,
|
|
633
|
+
keys: o,
|
|
634
|
+
outcome: "evicted",
|
|
635
|
+
mode: "singleton",
|
|
636
|
+
reason: "cancelled to make room for a newer start (singleton mode 'cancel')",
|
|
637
|
+
collapsed: 1,
|
|
638
|
+
waitedMs: null,
|
|
639
|
+
executionId: t.eviction.executionId,
|
|
640
|
+
runId: t.eviction.runId || null,
|
|
641
|
+
tenantId: t.tenantId,
|
|
642
|
+
now: i
|
|
643
|
+
}), r.evicted++), await e.transaction((e) => c(e, t.id, {
|
|
644
|
+
executionId: m.executionId,
|
|
645
|
+
runId: m.runId ?? null
|
|
646
|
+
})), r.admitted++;
|
|
647
|
+
} catch (r) {
|
|
648
|
+
let a = !1;
|
|
649
|
+
try {
|
|
650
|
+
if (s !== void 0 && e.isPermanentStartError?.(r) === !0) {
|
|
651
|
+
let o = s;
|
|
652
|
+
a = await e.transaction((e) => p(e, t.id, o, {
|
|
653
|
+
handler: n.onFailure ?? null,
|
|
654
|
+
errorMessage: String(r),
|
|
655
|
+
now: i
|
|
656
|
+
}, u)) !== void 0;
|
|
657
|
+
}
|
|
658
|
+
if (!a) {
|
|
659
|
+
let n = await e.transaction((n) => d(n, t.id, String(r), i, e.deadLetterAfterAttempts ?? 50, u));
|
|
660
|
+
n.retry.stoppedAt !== null && e.log?.warn?.("workflow submission automatic retry stopped; engine acceptance unknown", {
|
|
661
|
+
submissionId: t.id,
|
|
662
|
+
workflow: t.workflowName,
|
|
663
|
+
attempts: n.retry.attempts
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
} catch (e) {
|
|
667
|
+
throw new U(t.id, AggregateError([r, e], "Submission and retry recording failed"));
|
|
668
|
+
}
|
|
669
|
+
throw new U(t.id, r, a);
|
|
670
|
+
}
|
|
671
|
+
}, te = async (e, t) => {
|
|
672
|
+
if (e.listRunningRuns === void 0) return { cancelled: 0 };
|
|
673
|
+
let r = 0, i;
|
|
674
|
+
for (let a of e.controls.values()) {
|
|
675
|
+
if (a.timeouts?.finishMs === void 0) continue;
|
|
676
|
+
let o = await e.listRunningRuns(a.workflowName);
|
|
677
|
+
for (let s of o) {
|
|
678
|
+
if (!n(a, {
|
|
679
|
+
startedAt: s.startedAt,
|
|
680
|
+
terminal: !1
|
|
681
|
+
}, t)) {
|
|
682
|
+
let e = s.startedAt + a.timeouts.finishMs;
|
|
683
|
+
(i === void 0 || e < i) && (i = e);
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
let o = `timeouts.finish (${a.timeouts.finishMs}ms) elapsed; the run was cancelled after ${t - s.startedAt}ms`;
|
|
687
|
+
await e.cancelRun({
|
|
688
|
+
workflowName: s.workflowName,
|
|
689
|
+
runId: s.runId,
|
|
690
|
+
executionId: s.executionId,
|
|
691
|
+
reason: o,
|
|
692
|
+
detail: {
|
|
693
|
+
...k,
|
|
694
|
+
onFailure: a.onFailure ?? null
|
|
695
|
+
}
|
|
696
|
+
}) && (await E(e.store, s.executionId, t), r++);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return {
|
|
700
|
+
cancelled: r,
|
|
701
|
+
...i === void 0 ? {} : { nextDeadlineAt: i }
|
|
702
|
+
};
|
|
703
|
+
};
|
|
704
|
+
//#endregion
|
|
705
|
+
export { H as DEFAULT_DRAIN_BATCH, ee as acceptAdmissionSubmission, G as drainTick, W as isUnacknowledgedSubmission };
|