@voltro/workflow 0.86.0 → 0.88.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5484 -0
- package/THIRD-PARTY-NOTICES.md +3 -227
- package/dist/admission.js +1 -2
- package/dist/admissionDrainer.js +5 -3
- package/dist/admissionGate.js +1 -2
- package/dist/admissionStore.js +1 -2
- package/dist/admissionSubmission.js +1 -2
- package/dist/admissionTables.js +4 -4
- package/dist/admissionTransaction.js +1 -2
- package/dist/awaitEvent.js +1 -2
- package/dist/budgetHold.js +1 -2
- package/dist/cancelOnPlan.js +1 -2
- package/dist/cancelOnSweep.js +1 -2
- package/dist/cancelPendingStart.js +1 -2
- package/dist/cluster.d.ts +111 -27
- package/dist/cluster.js +12 -8
- package/dist/clusterCron.js +18 -11
- package/dist/clusterDiagnostics.js +1 -2
- package/dist/clusterDiagnosticsSql.js +1 -2
- package/dist/clusterLayer.js +105 -71
- package/dist/clusterMigrationGate.js +1 -2
- package/dist/clusterRunnerLayer.js +27 -18
- package/dist/clusterShardGroups.js +32 -0
- package/dist/clusterTestSuite.d.ts +45 -2
- package/dist/clusterTestSuite.js +225 -197
- package/dist/controlMonitor.js +1 -2
- package/dist/controlRequest.js +1 -2
- package/dist/executionRefusal.js +1 -2
- package/dist/failureHandoff.js +1 -2
- package/dist/failureMessage.js +1 -2
- package/dist/failureRecovery.js +1 -2
- package/dist/failureReportStore.js +1 -2
- package/dist/flowControl.js +1 -2
- package/dist/forwardStartReceipts.js +1 -2
- package/dist/inMemoryLayer.js +1 -2
- package/dist/inMemoryRecorder.js +1 -2
- package/dist/index.d.ts +50 -23
- package/dist/index.js +24 -23
- package/dist/inspectWorkflow.js +2 -3
- package/dist/journalEncryption.js +1 -2
- package/dist/journalRoutingGuard.js +49 -0
- package/dist/journalRoutingLayout.js +46 -0
- package/dist/journalRoutingLock.js +38 -0
- package/dist/journalRoutingMaintenance.js +23 -0
- package/dist/journalRoutingMessageStorage.js +31 -0
- package/dist/journalRoutingNamespace.js +22 -0
- package/dist/journalRoutingRuntime.js +32 -0
- package/dist/journalRoutingSharding.js +27 -0
- package/dist/journalRoutingState.js +92 -0
- package/dist/journalRoutingTable.js +10 -0
- package/dist/journalRoutingTransaction.js +15 -0
- package/dist/messages.js +1 -2
- package/dist/nondeterminism.js +1 -2
- package/dist/parentClosure.js +1 -2
- package/dist/parentClosureRecovery.js +1 -2
- package/dist/pendingSubmission.js +1 -2
- package/dist/prepareSubmissionDispatch.js +1 -2
- package/dist/preparedExecution.js +1 -2
- package/dist/primitives.js +1 -2
- package/dist/queuedCancellationRequest.js +1 -2
- package/dist/queuedControl.js +1 -2
- package/dist/recorder.js +1 -2
- package/dist/residentExecutionEvidence.js +1 -2
- package/dist/retryPolicy.js +1 -2
- package/dist/runEventsTable.js +1 -2
- package/dist/runGeneration.js +1 -2
- package/dist/runRecording.js +1 -2
- package/dist/runStepsTable.js +1 -2
- package/dist/runnerStorageGuard.js +1 -2
- package/dist/runsTable.js +1 -2
- package/dist/scheduleDeclaration.js +1 -2
- package/dist/signalHint.js +1 -2
- package/dist/signals.js +1 -2
- package/dist/signalsSuspending.js +1 -2
- package/dist/sqliteWorkflowOwnership.js +1 -2
- package/dist/stalenessSweep.js +1 -2
- package/dist/startContextsTable.js +1 -2
- package/dist/startReceipt.js +1 -2
- package/dist/submissionRecovery.js +1 -2
- package/dist/submissionRejection.js +1 -2
- package/dist/submissionRetry.js +1 -2
- package/dist/suspensionUnavailable.js +1 -2
- package/dist/updates.js +1 -2
- package/dist/watermarkTable.js +1 -2
- package/dist/webCrypto.js +4 -17
- package/dist/workflowDelivery.js +1 -2
- package/dist/workflowExecutorRouting.js +32 -0
- package/dist/workflowJournal.js +1 -2
- package/dist/workflowJournalReset.js +1 -2
- package/dist/workflowReadSelections.js +1 -2
- package/dist/workflowRestartDispatch.js +30 -25
- package/dist/workflowRestartObservation.js +1 -2
- package/dist/workflowRestartPreparation.js +1 -2
- package/dist/workflowRestartReservation.js +158 -157
- package/dist/workflowResumeDispatch.js +18 -18
- package/dist/workflowRuntimeReadiness.js +11 -0
- package/dist/workflowShardRelocation.js +57 -0
- package/dist/workflowShardRelocationSql.js +71 -0
- package/package.json +5 -11
package/dist/clusterLayer.js
CHANGED
|
@@ -1,49 +1,65 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
|
|
1
|
+
import { JournalRoutingGuard as e } from "./journalRoutingGuard.js";
|
|
2
|
+
import { withWorkflowDelivery as t } from "./workflowDelivery.js";
|
|
3
|
+
import { ClusterDiagnostics as n } from "./clusterDiagnostics.js";
|
|
4
|
+
import { WorkflowRuntimeReadiness as r, makeWorkflowRuntimeReadiness as i } from "./workflowRuntimeReadiness.js";
|
|
5
|
+
import { journalRoutingLayout as a } from "./journalRoutingLayout.js";
|
|
6
|
+
import { recordWorkflowExecutorRouting as o } from "./workflowExecutorRouting.js";
|
|
7
|
+
import { JournalRoutingLayout as s, withJournalWorkflowRouting as c } from "./journalRoutingRuntime.js";
|
|
8
|
+
import { captureWorkflowShardGroups as ee } from "./clusterShardGroups.js";
|
|
9
|
+
import { acquireGuardedPostgresJournalRoutingAdmission as te, captureJournalRoutingState as l } from "./journalRoutingState.js";
|
|
10
|
+
import { WorkflowResumeDispatcher as u, makeWorkflowResumeDispatcher as d } from "./workflowResumeDispatch.js";
|
|
11
|
+
import { WorkflowRestartDispatcher as f, makeWorkflowRestartDispatcher as p } from "./workflowRestartDispatch.js";
|
|
12
|
+
import { clusterMigrationGateLayer as ne } from "./clusterMigrationGate.js";
|
|
13
|
+
import { clusterRunnerLayer as m } from "./clusterRunnerLayer.js";
|
|
14
|
+
import { acquireSqliteWorkflowOwnership as h } from "./sqliteWorkflowOwnership.js";
|
|
15
|
+
import { makeClusterDiagnostics as g } from "./clusterDiagnosticsSql.js";
|
|
16
|
+
import { WorkflowEngine as _ } from "effect/workflow";
|
|
17
|
+
import { RateLimiter as v } from "effect/persistence";
|
|
18
|
+
import { Context as y, Duration as b, Effect as x, Layer as S, Option as C } from "effect";
|
|
19
|
+
import { SqlClient as w } from "effect/sql";
|
|
20
|
+
import { ClusterWorkflowEngine as T, MessageStorage as E, RunnerAddress as D, ShardId as O, Sharding as k } from "effect/cluster";
|
|
21
|
+
import { sqlPoolCapacity as A } from "@voltro/database/sql";
|
|
22
|
+
import { createLogger as j } from "@voltro/logger";
|
|
23
|
+
// src/clusterLayer.ts
|
|
24
|
+
var M = j({ scope: "voltro:workflow" }), N = 34e3, P = (e) => {
|
|
16
25
|
let t = e.runnerListenHost ?? process.env.POD_IP ?? process.env.VOLTRO_WORKFLOW_RUNNER_HOST ?? "localhost";
|
|
17
26
|
return {
|
|
18
27
|
host: t,
|
|
19
28
|
port: e.runnerListenPort ?? 34e3,
|
|
20
29
|
localhostRisk: e.runnerStorage === "sql" && (t === "localhost" || t === "127.0.0.1")
|
|
21
30
|
};
|
|
22
|
-
},
|
|
31
|
+
}, F = (e) => {
|
|
23
32
|
let t = e.sqlRunnerSupported ?? !e.sqliteFamily, n = t ? "sql" : "memory", r = e.override ?? process.env.VOLTRO_WORKFLOW_RUNNER_STORAGE;
|
|
24
33
|
if (r === void 0 || r.trim() === "") return n;
|
|
25
34
|
let i = r.trim().toLowerCase();
|
|
26
35
|
if (i !== "memory" && i !== "sql") throw Error(`VOLTRO_WORKFLOW_RUNNER_STORAGE must be 'memory' or 'sql', got '${r}'.`);
|
|
27
36
|
if (i === "sql" && !t) throw Error("VOLTRO_WORKFLOW_RUNNER_STORAGE='sql' is unsupported for this database connection. Use 'memory' for a local or embedded SQLite-family connection.");
|
|
28
37
|
return i;
|
|
29
|
-
},
|
|
38
|
+
}, I = (e = process.env.VOLTRO_WORKFLOW_SHARD_LOCK) => {
|
|
30
39
|
if (e === void 0 || e.trim() === "") return "auto";
|
|
31
40
|
let t = e.trim().toLowerCase();
|
|
32
41
|
if (t === "auto" || t === "row" || t === "advisory") return t;
|
|
33
42
|
throw Error(`VOLTRO_WORKFLOW_SHARD_LOCK must be 'auto', 'row', or 'advisory', got '${e}'.`);
|
|
34
|
-
},
|
|
35
|
-
let e = yield* (yield*
|
|
43
|
+
}, L = x.gen(function* () {
|
|
44
|
+
let e = yield* (yield* w.SqlClient)`SELECT @@wsrep_on AS wsrep`.pipe(x.catch(() => x.succeed([])));
|
|
36
45
|
if (e.length === 0) return !1;
|
|
37
46
|
let t = e[0].wsrep;
|
|
38
47
|
return t === 1 || t === "1" || t === !0 || t === "ON" || t === "on";
|
|
39
|
-
}),
|
|
40
|
-
|
|
41
|
-
|
|
48
|
+
}), R = (e) => x.gen(function* () {
|
|
49
|
+
if (e.runnerStorage === "memory") return !1;
|
|
50
|
+
if (e.dialectId === "sqlite" || e.dialectId === "turso" || e.mode === "row") return !0;
|
|
51
|
+
if (e.mode === "advisory") return !1;
|
|
52
|
+
if (e.dialectId === "postgres") {
|
|
53
|
+
let e = (yield* (yield* w.SqlClient)`SELECT current_schema() AS schema`.pipe(x.orDie))[0]?.schema;
|
|
54
|
+
return e ? e !== "public" : yield* x.die(/* @__PURE__ */ Error("PostgreSQL workflow coordination requires an accessible current schema"));
|
|
55
|
+
}
|
|
56
|
+
return e.dialectId !== "mysql" && e.dialectId !== "mariadb" ? !1 : yield* L;
|
|
57
|
+
}), z = (e) => ({
|
|
42
58
|
coordination: e.disableAdvisory ? "row-based" : "advisory",
|
|
43
59
|
dialect: e.dialectId ?? "unknown",
|
|
44
60
|
mode: e.mode,
|
|
45
|
-
...e.disableAdvisory && e.mode === "auto" ? { reason: e.dialectId === "sqlite" || e.dialectId === "turso" ? "SQLite-family connection uses row locks" : "wsrep/Galera cluster detected" } : {}
|
|
46
|
-
}),
|
|
61
|
+
...e.disableAdvisory && e.mode === "auto" ? { reason: e.dialectId === "sqlite" || e.dialectId === "turso" ? "SQLite-family connection uses row locks" : e.dialectId === "postgres" ? "PostgreSQL non-public schema uses row leases" : "wsrep/Galera cluster detected" } : {}
|
|
62
|
+
}), B = () => {
|
|
47
63
|
let e = (e) => {
|
|
48
64
|
if (e === void 0) return;
|
|
49
65
|
let t = Number(e);
|
|
@@ -54,57 +70,75 @@ var C = S({ scope: "voltro:workflow" }), w = 34e3, T = (e) => {
|
|
|
54
70
|
...n === void 0 ? {} : { failoverHeartbeatSeconds: n },
|
|
55
71
|
...r === void 0 ? {} : { messagePollSeconds: r }
|
|
56
72
|
};
|
|
57
|
-
},
|
|
58
|
-
let
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
}, V = x.flatMap(k.Sharding, (e) => e.pollStorage), H = /* @__PURE__ */ new Set(), U = (e) => Object.assign(Object.create(Object.getPrototypeOf(e)), e, { requestIdForPrimaryKey: (t) => e.requestIdForPrimaryKey(t).pipe(x.provideService(w.SafeIntegers, !0)) }), W = (j) => {
|
|
74
|
+
let N = j.runnerMode === void 0 ? "worker" : j.runnerMode;
|
|
75
|
+
if (N !== "worker" && N !== "client") throw Error("Invalid workflow runner mode");
|
|
76
|
+
let F = N === "client", L = j.journalRouting === void 0 ? void 0 : l(j.journalRouting);
|
|
77
|
+
if (L !== void 0 && j.dialectId !== "postgres") throw Error("Guarded journal routing currently requires PostgreSQL");
|
|
78
|
+
let B = L === void 0 ? void 0 : a(L.layout), V = ee(j.shardGroups ?? (B === void 0 ? void 0 : {
|
|
79
|
+
available: B.available,
|
|
80
|
+
assigned: F ? [] : B.available
|
|
81
|
+
}), F);
|
|
82
|
+
if (B !== void 0 && (V?.available.length !== B.available.length || B.available.some((e) => !V?.available.includes(e)))) throw Error("Shard inventory must match the admitted journal routing layout");
|
|
83
|
+
if (F && (j.runnerStorage !== "sql" || j.dialectId === "sqlite")) throw Error("Client-only workflows require shared remote SQL runner storage");
|
|
84
|
+
if (F && (j.extraShardingLayers?.length ?? 0) > 0) throw Error("Client-only workflows cannot own extra sharding layers");
|
|
85
|
+
let W = P({
|
|
86
|
+
runnerStorage: j.runnerStorage,
|
|
87
|
+
runnerListenHost: j.runnerListenHost,
|
|
88
|
+
runnerListenPort: j.runnerListenPort
|
|
89
|
+
}), G = W.host, K = W.port;
|
|
90
|
+
!F && W.localhostRisk && !H.has(G) && (H.add(G), M.warn("runner host is not routable from other pods with SQL cluster storage — a workflow cannot resume on another pod after a reschedule. Inject POD_IP via the K8s downward API (fieldRef: status.podIP) or set runnerListenHost / VOLTRO_WORKFLOW_RUNNER_HOST.", { runnerHost: G }));
|
|
91
|
+
let q = j.dialectId === "sqlite" ? S.effect(w.SqlClient, x.gen(function* () {
|
|
92
|
+
let e = yield* w.SqlClient, t = (yield* e.unsafe("PRAGMA database_list")).find((e) => e.name === "main");
|
|
93
|
+
return t === void 0 ? yield* x.die(/* @__PURE__ */ Error("SQLite workflow connection has no main database")) : (t.file !== "" && (yield* h(t.file)), e);
|
|
94
|
+
}).pipe(x.orDie)).pipe(S.provide(j.sqlClientLayer)) : j.sqlClientLayer, J = ne({
|
|
95
|
+
runnerStorage: j.runnerStorage,
|
|
96
|
+
charset: j.charset
|
|
97
|
+
}).pipe(S.provide(q)), Y = L === void 0 ? J : S.effect(e, x.gen(function* () {
|
|
98
|
+
let e = yield* w.SqlClient, t = A(e), n = j.runnerStorage === "sql" ? 3 : 2;
|
|
99
|
+
return t !== void 0 && t < n ? yield* x.die(/* @__PURE__ */ Error(`Guarded journal routing requires at least ${n} SQL connections for the admission lock, runner ownership and journal work`)) : yield* te(L);
|
|
100
|
+
}).pipe(x.orDie)).pipe(S.provideMerge(J)), X = j.shardLockMode ?? I(), re = S.unwrap(R({
|
|
101
|
+
dialectId: j.dialectId,
|
|
102
|
+
runnerStorage: j.runnerStorage,
|
|
103
|
+
mode: X
|
|
104
|
+
}).pipe(x.map((e) => {
|
|
105
|
+
!F && j.runnerStorage === "sql" && M.info("shard-lock coordination", z({
|
|
76
106
|
disableAdvisory: e,
|
|
77
|
-
dialectId:
|
|
78
|
-
mode:
|
|
107
|
+
dialectId: j.dialectId,
|
|
108
|
+
mode: X
|
|
79
109
|
}));
|
|
80
|
-
let
|
|
81
|
-
ownAddress:
|
|
82
|
-
shardLockMode:
|
|
83
|
-
runnerStorage:
|
|
84
|
-
dialectId:
|
|
85
|
-
hasShard: (e) => t.hasShardId(
|
|
86
|
-
}))),
|
|
87
|
-
|
|
88
|
-
|
|
110
|
+
let t = S.effect(n, x.flatMap(k.Sharding, (t) => g({
|
|
111
|
+
ownAddress: D.make(G, K).toString(),
|
|
112
|
+
shardLockMode: j.runnerStorage === "sql" ? e ? "row" : "advisory" : "none",
|
|
113
|
+
runnerStorage: j.runnerStorage,
|
|
114
|
+
dialectId: j.dialectId,
|
|
115
|
+
hasShard: (e) => t.hasShardId(O.fromString(e))
|
|
116
|
+
}))), a = {
|
|
117
|
+
...V === void 0 ? {} : {
|
|
118
|
+
availableShardGroups: V.available,
|
|
119
|
+
assignedShardGroups: V.assigned
|
|
120
|
+
},
|
|
121
|
+
runnerAddress: F ? C.none() : C.some(D.make(G, K)),
|
|
122
|
+
runnerListenAddress: F ? C.none() : C.some(D.make(G, K)),
|
|
89
123
|
...e ? { shardLockDisableAdvisory: !0 } : {},
|
|
90
|
-
...
|
|
91
|
-
...
|
|
92
|
-
...
|
|
93
|
-
},
|
|
94
|
-
|
|
95
|
-
|
|
124
|
+
...j.failoverLeaseSeconds === void 0 ? {} : { shardLockExpiration: b.seconds(j.failoverLeaseSeconds) },
|
|
125
|
+
...j.failoverHeartbeatSeconds === void 0 ? {} : { shardLockRefreshInterval: b.seconds(j.failoverHeartbeatSeconds) },
|
|
126
|
+
...j.messagePollSeconds === void 0 ? {} : { entityMessagePollInterval: b.seconds(j.messagePollSeconds) }
|
|
127
|
+
}, o = b.seconds(j.failoverHeartbeatSeconds ?? 10), s = m({
|
|
128
|
+
clientOnly: F,
|
|
129
|
+
runnerStorage: j.runnerStorage,
|
|
130
|
+
shardingConfig: a,
|
|
96
131
|
guard: {
|
|
97
|
-
interval:
|
|
98
|
-
giveUpAfter:
|
|
99
|
-
log:
|
|
132
|
+
interval: o,
|
|
133
|
+
giveUpAfter: b.times(o, 3),
|
|
134
|
+
log: M
|
|
100
135
|
}
|
|
101
136
|
});
|
|
102
|
-
return
|
|
103
|
-
}))).pipe(
|
|
104
|
-
let n =
|
|
105
|
-
return
|
|
106
|
-
}),
|
|
107
|
-
return
|
|
137
|
+
return S.mergeAll(S.effect(r, i), t, S.effect(u, d), S.effect(f, p)).pipe(S.provideMerge(s));
|
|
138
|
+
}))).pipe(S.provide(L === void 0 ? Y : S.merge(Y, S.succeed(s, L.layout)))), Z = j.extraShardingLayers ?? [], Q = F ? S.effect(_.WorkflowEngine, T.make) : T.layer, $ = L === void 0 ? Q : Q.pipe(S.flatMap((e) => S.succeed(_.WorkflowEngine, c(y.get(e, _.WorkflowEngine), L.layout)))), ie = Z.length === 0 ? $ : S.mergeAll($, ...Z), ae = S.flatMap(re, (e) => {
|
|
139
|
+
let n = y.add(e, k.Sharding, t(y.get(e, k.Sharding)));
|
|
140
|
+
return S.succeedContext(j.dialectId === "sqlite" ? y.add(n, E.MessageStorage, U(y.get(n, E.MessageStorage))) : n);
|
|
141
|
+
}), oe = v.layer.pipe(S.provide(v.layerStoreMemory));
|
|
142
|
+
return S.merge(ie, oe).pipe(S.provideMerge(ae), S.flatMap((e) => S.succeedContext(o(e, y.get(e, _.WorkflowEngine), N, L, V))));
|
|
108
143
|
};
|
|
109
|
-
|
|
110
|
-
export { w as DEFAULT_RUNNER_LISTEN_PORT, O as detectWsrepCluster, D as parseShardLockMode, M as pollWorkflowStorage, j as resolveFailoverTuning, T as resolveRunnerIdentity, E as resolveRunnerStorage, k as resolveShardLockDisableAdvisory, A as shardLockLogFields, F as workflowEngineLayer };
|
|
144
|
+
export { N as DEFAULT_RUNNER_LISTEN_PORT, L as detectWsrepCluster, I as parseShardLockMode, V as pollWorkflowStorage, B as resolveFailoverTuning, P as resolveRunnerIdentity, F as resolveRunnerStorage, R as resolveShardLockDisableAdvisory, z as shardLockLogFields, W as workflowEngineLayer };
|
|
@@ -3,7 +3,7 @@ import { Effect as t, Layer as n } from "effect";
|
|
|
3
3
|
import { SqlClient as r } from "effect/sql";
|
|
4
4
|
import { ShardingConfig as i, SqlMessageStorage as a, SqlRunnerStorage as o } from "effect/cluster";
|
|
5
5
|
import { applyUndeclaredCharsetWork as s, fnv1a64 as c, readUndeclaredCharsetColumns as l, resolveMigrationLockSchema as u, sqlPoolCapacity as d, undeclaredCharsetWork as f } from "@voltro/database/sql";
|
|
6
|
-
|
|
6
|
+
// src/clusterMigrationGate.ts
|
|
7
7
|
var p = 6322741009312601n, m = () => {
|
|
8
8
|
let e = u();
|
|
9
9
|
return e === void 0 ? p : c(`voltro_cluster_migration_lock:${e}`) & 9223372036854775807n;
|
|
@@ -57,5 +57,4 @@ var p = 6322741009312601n, m = () => {
|
|
|
57
57
|
let n = yield* r.SqlClient, i = k(e.runnerStorage), a = d(n);
|
|
58
58
|
return a !== void 0 && a < i ? yield* t.die(/* @__PURE__ */ Error(`the workflow runtime needs ${i} database connections at once and the pool has ${a}: ` + (e.runnerStorage === "sql" ? "on a first boot it holds its migration lock and its runner lock while the migration runs on a third, and once running it keeps the runner lock and needs one more to work. " : "on a first boot it holds its migration lock while the migration runs on a second. ") + `Set DB_MAX_CONNECTIONS (or PG_MAX_CONNECTIONS) to at least ${i}` + (e.runnerStorage === "sql" ? ", or VOLTRO_WORKFLOW_RUNNER_STORAGE=memory for a single-process runtime that needs 2 (no cross-pod shard handoff)." : "."))) : ((yield* w(n, e.runnerStorage)) || (yield* x(n, O(n, e.runnerStorage, e.charset))), n);
|
|
59
59
|
}));
|
|
60
|
-
//#endregion
|
|
61
60
|
export { k as clusterConnectionFloor, A as clusterMigrationGateLayer, C as tableExists, x as withClusterMigrationLock };
|
|
@@ -1,21 +1,30 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
import { JournalRoutingGuard as e } from "./journalRoutingGuard.js";
|
|
2
|
+
import { JournalRoutingLayout as t, withJournalEntityRouting as n } from "./journalRoutingRuntime.js";
|
|
3
|
+
import { webCryptoLayer as r } from "./webCrypto.js";
|
|
4
|
+
import { withJournalRoutingGuard as i } from "./journalRoutingSharding.js";
|
|
5
|
+
import { guardJournalMessageStorage as a } from "./journalRoutingMessageStorage.js";
|
|
6
|
+
import { guardRunnerStorage as o } from "./runnerStorageGuard.js";
|
|
7
|
+
import { Effect as s, Layer as c, Option as l, Semaphore as u } from "effect";
|
|
8
|
+
import { SqlClient as d } from "effect/sql";
|
|
9
|
+
import { MessageStorage as f, RunnerHealth as p, RunnerStorage as m, Runners as h, Sharding as g, ShardingConfig as _, SqlMessageStorage as v, SqlRunnerStorage as y } from "effect/cluster";
|
|
10
|
+
// src/clusterRunnerLayer.ts
|
|
11
|
+
var b = (e) => s.map(u.make(1), (t) => ({
|
|
8
12
|
...e,
|
|
9
|
-
release: (
|
|
10
|
-
})),
|
|
11
|
-
sqlite: () =>
|
|
12
|
-
orElse: () =>
|
|
13
|
-
}))),
|
|
14
|
-
let
|
|
15
|
-
return yield*
|
|
13
|
+
release: (n, r) => t.withPermits(1)(s.suspend(() => e.release(n, r)))
|
|
14
|
+
})), x = c.unwrap(s.map(d.SqlClient, (e) => e.onDialectOrElse({
|
|
15
|
+
sqlite: () => c.effect(m.RunnerStorage, s.flatMap(m.RunnerStorage, b)).pipe(c.provide(y.layer)),
|
|
16
|
+
orElse: () => y.layer
|
|
17
|
+
}))), S = (e) => c.effect(m.RunnerStorage, s.gen(function* () {
|
|
18
|
+
let t = yield* d.SqlClient, n = yield* m.RunnerStorage;
|
|
19
|
+
return yield* o(n, {
|
|
16
20
|
...e,
|
|
17
|
-
probe:
|
|
21
|
+
probe: t`SELECT 1`
|
|
18
22
|
});
|
|
19
|
-
})).pipe(
|
|
20
|
-
|
|
21
|
-
|
|
23
|
+
})).pipe(c.provide(c.orDie(x))), C = c.effect(f.MessageStorage, s.gen(function* () {
|
|
24
|
+
let t = yield* f.MessageStorage, n = yield* s.serviceOption(e);
|
|
25
|
+
return l.isSome(n) ? a(t, n.value) : t;
|
|
26
|
+
})).pipe(c.provide(v.layer)), w = c.effect(g.Sharding, s.gen(function* () {
|
|
27
|
+
let r = yield* g.Sharding, a = yield* s.serviceOption(e), o = yield* s.serviceOption(t), c = l.isSome(o) ? n(r, o.value) : r;
|
|
28
|
+
return l.isSome(a) ? i(c, a.value) : c;
|
|
29
|
+
})).pipe(c.provide(g.layer)), T = (e) => w.pipe(c.provideMerge(h.layerNoop), c.provideMerge(C), c.provide([e.runnerStorage === "memory" ? m.layerMemory : e.clientOnly ? c.orDie(x) : S(e.guard), p.layerNoop]), c.provide(_.layerFromEnv(e.shardingConfig)), c.provide(r));
|
|
30
|
+
export { T as clusterRunnerLayer, b as limitSqliteShardReleases };
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// src/clusterShardGroups.ts
|
|
2
|
+
var e = (e, t) => {
|
|
3
|
+
if (e === void 0) return;
|
|
4
|
+
if (!e || typeof e != "object" || Array.isArray(e) || ![Object.prototype, null].includes(Object.getPrototypeOf(e))) throw Error("Workflow shard groups require a plain data object");
|
|
5
|
+
let n = /* @__PURE__ */ new Map();
|
|
6
|
+
for (let t of Reflect.ownKeys(e)) {
|
|
7
|
+
let r = Object.getOwnPropertyDescriptor(e, t);
|
|
8
|
+
if (typeof t != "string" || !["available", "assigned"].includes(t) || !r.enumerable || !("value" in r)) throw Error("Invalid workflow shard group field");
|
|
9
|
+
n.set(t, r.value);
|
|
10
|
+
}
|
|
11
|
+
let r = (e) => {
|
|
12
|
+
let t = n.get(e);
|
|
13
|
+
if (!Array.isArray(t) || t.length > 64 || Reflect.ownKeys(t).length !== t.length + 1) throw Error("Invalid workflow shard group list");
|
|
14
|
+
let r = [];
|
|
15
|
+
for (let e = 0; e < t.length; e++) {
|
|
16
|
+
let n = Object.getOwnPropertyDescriptor(t, String(e));
|
|
17
|
+
if (!n?.enumerable || !("value" in n) || typeof n.value != "string" || !/^[a-z][a-z0-9._-]{0,31}$/.test(n.value)) throw Error("Invalid workflow shard group name");
|
|
18
|
+
r.push(n.value);
|
|
19
|
+
}
|
|
20
|
+
if (new Set(r).size !== r.length) throw Error("Duplicate workflow shard group");
|
|
21
|
+
return Object.freeze(r);
|
|
22
|
+
}, i = r("available"), a = r("assigned");
|
|
23
|
+
if (i.length === 0) throw Error("Workflow shard group inventory cannot be empty");
|
|
24
|
+
if (a.some((e) => !i.includes(e))) throw Error("Assigned workflow shard group is unavailable");
|
|
25
|
+
if (t && a.length > 0) throw Error("Client-only workflows cannot own shard groups");
|
|
26
|
+
if (!t && a.length === 0) throw Error("Workflow workers require an assigned shard group");
|
|
27
|
+
return Object.freeze({
|
|
28
|
+
available: i,
|
|
29
|
+
assigned: a
|
|
30
|
+
});
|
|
31
|
+
};
|
|
32
|
+
export { e as captureWorkflowShardGroups };
|
|
@@ -36,6 +36,31 @@ export declare interface ClusterEngineFixture {
|
|
|
36
36
|
readonly reachable: () => Promise<boolean>;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
/** Test-runner operations supplied by the suite caller, outside the runtime graph. */
|
|
40
|
+
export declare interface ClusterSuiteTestApi {
|
|
41
|
+
readonly describe: (name: string, body: () => void) => unknown;
|
|
42
|
+
readonly test: (name: string, body: (context: {
|
|
43
|
+
readonly skip: () => void;
|
|
44
|
+
}) => Promise<void>, timeoutMs?: number) => unknown;
|
|
45
|
+
readonly beforeAll: (body: () => Promise<void>, timeoutMs?: number) => unknown;
|
|
46
|
+
readonly afterEach: (body: () => void | Promise<void>) => unknown;
|
|
47
|
+
readonly expect: (actual: unknown, message?: string) => {
|
|
48
|
+
readonly toBe: (expected: unknown) => unknown;
|
|
49
|
+
readonly toEqual: (expected: unknown) => unknown;
|
|
50
|
+
readonly toContain: (expected: unknown) => unknown;
|
|
51
|
+
readonly toBeGreaterThan: (expected: number) => unknown;
|
|
52
|
+
readonly toBeGreaterThanOrEqual: (expected: number) => unknown;
|
|
53
|
+
readonly rejects: {
|
|
54
|
+
readonly toThrow: () => Promise<unknown>;
|
|
55
|
+
};
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
declare interface JournalRoutingState {
|
|
60
|
+
readonly generation: number;
|
|
61
|
+
readonly layout: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
39
64
|
/**
|
|
40
65
|
* Clear cluster state left behind by EARLIER runs of this suite.
|
|
41
66
|
*
|
|
@@ -70,9 +95,9 @@ export declare const purgeClusterState: (fixture: ClusterEngineFixture) => Promi
|
|
|
70
95
|
* `@voltro/sql-*` (or the workflow package) calls this from a thin
|
|
71
96
|
* `cluster.<dialect>.integration.test.ts` with its fixture.
|
|
72
97
|
*/
|
|
73
|
-
export declare const runClusterEngineSuite: (fixture: ClusterEngineFixture) => void;
|
|
98
|
+
export declare const runClusterEngineSuite: (fixture: ClusterEngineFixture, tests: ClusterSuiteTestApi) => void;
|
|
74
99
|
|
|
75
|
-
export declare const runJournalEncryptionSuite: (fixture: ClusterEngineFixture) => void;
|
|
100
|
+
export declare const runJournalEncryptionSuite: (fixture: ClusterEngineFixture, tests: ClusterSuiteTestApi) => void;
|
|
76
101
|
|
|
77
102
|
/**
|
|
78
103
|
* Runner-storage backend for `effect/cluster.SingleRunner`.
|
|
@@ -122,6 +147,15 @@ declare type ShardLockMode = 'auto' | 'row' | 'advisory';
|
|
|
122
147
|
export declare const tcpReachable: (host: string, port: number, timeoutMs?: number) => Promise<boolean>;
|
|
123
148
|
|
|
124
149
|
declare interface WorkflowEngineLayerOptions {
|
|
150
|
+
/** Client mode submits/polls durable work without registering an executor,
|
|
151
|
+
* runner heartbeat or shard owner. Requires shared SQL runner storage. */
|
|
152
|
+
readonly runnerMode?: 'worker' | 'client';
|
|
153
|
+
/** Opt into the persisted PostgreSQL routing generation and guarded journal.
|
|
154
|
+
* Requires a completed stop-all routing transition before split ownership. */
|
|
155
|
+
readonly journalRouting?: JournalRoutingState;
|
|
156
|
+
/** Explicit cluster-wide routing inventory and this runner's ownership.
|
|
157
|
+
* Omitted preserves existing routing. Does not migrate persisted addresses. */
|
|
158
|
+
readonly shardGroups?: WorkflowShardGroups;
|
|
125
159
|
/**
|
|
126
160
|
* The dialect's SqlClient layer (postgres / mysql / mariadb / mssql).
|
|
127
161
|
* For `runnerStorage: 'memory'` it's still required so the cluster's
|
|
@@ -204,4 +238,13 @@ declare interface WorkflowEngineLayerOptions {
|
|
|
204
238
|
readonly extraShardingLayers?: ReadonlyArray<Layer.Layer<never, never, Sharding.Sharding>>;
|
|
205
239
|
}
|
|
206
240
|
|
|
241
|
+
/** Explicit routing inventory shared by every runner in one cluster. Changing
|
|
242
|
+
* groups changes persistent addresses; this is not a journal migration. */
|
|
243
|
+
/** Group names are at most 32 ASCII characters: group + colon + a safe integer
|
|
244
|
+
* slot fits the existing SQL journal shard_id VARCHAR(50) on every dialect. */
|
|
245
|
+
declare interface WorkflowShardGroups {
|
|
246
|
+
readonly available: ReadonlyArray<string>;
|
|
247
|
+
readonly assigned: ReadonlyArray<string>;
|
|
248
|
+
}
|
|
249
|
+
|
|
207
250
|
export { }
|