@smthrs/run-store 0.0.0-stage → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -0
- package/LICENSE +21 -0
- package/README.md +195 -2
- package/dist/cjs/AttemptStore.d.ts +359 -0
- package/dist/cjs/AttemptStore.d.ts.map +1 -0
- package/dist/cjs/AttemptStore.js +731 -0
- package/dist/cjs/AttemptStore.js.map +7 -0
- package/dist/cjs/Heartbeat.d.ts +46 -0
- package/dist/cjs/Heartbeat.d.ts.map +1 -0
- package/dist/cjs/Heartbeat.js +28 -0
- package/dist/cjs/Heartbeat.js.map +7 -0
- package/dist/cjs/Migrations.d.ts +42 -0
- package/dist/cjs/Migrations.d.ts.map +1 -0
- package/dist/cjs/Migrations.js +55 -0
- package/dist/cjs/Migrations.js.map +7 -0
- package/dist/cjs/Ownership.d.ts +275 -0
- package/dist/cjs/Ownership.d.ts.map +1 -0
- package/dist/cjs/Ownership.js +129 -0
- package/dist/cjs/Ownership.js.map +7 -0
- package/dist/cjs/RunStore.d.ts +571 -0
- package/dist/cjs/RunStore.d.ts.map +1 -0
- package/dist/cjs/RunStore.js +1079 -0
- package/dist/cjs/RunStore.js.map +7 -0
- package/dist/cjs/RunStoreMetrics.d.ts +151 -0
- package/dist/cjs/RunStoreMetrics.d.ts.map +1 -0
- package/dist/cjs/RunStoreMetrics.js +91 -0
- package/dist/cjs/RunStoreMetrics.js.map +7 -0
- package/dist/cjs/index.d.ts +47 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +43 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/Boundary.d.ts +80 -0
- package/dist/cjs/internal/Boundary.d.ts.map +1 -0
- package/dist/cjs/internal/Boundary.js +92 -0
- package/dist/cjs/internal/Boundary.js.map +7 -0
- package/dist/cjs/internal/Companion.d.ts +44 -0
- package/dist/cjs/internal/Companion.d.ts.map +1 -0
- package/dist/cjs/internal/Companion.js +50 -0
- package/dist/cjs/internal/Companion.js.map +7 -0
- package/dist/cjs/internal/SpanOutcome.d.ts +37 -0
- package/dist/cjs/internal/SpanOutcome.d.ts.map +1 -0
- package/dist/cjs/internal/SpanOutcome.js +62 -0
- package/dist/cjs/internal/SpanOutcome.js.map +7 -0
- package/dist/cjs/migrations/0001_initial.d.ts +15 -0
- package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
- package/dist/cjs/migrations/0001_initial.js +110 -0
- package/dist/cjs/migrations/0001_initial.js.map +7 -0
- package/dist/cjs/migrations/0002_lineage.d.ts +27 -0
- package/dist/cjs/migrations/0002_lineage.d.ts.map +1 -0
- package/dist/cjs/migrations/0002_lineage.js +43 -0
- package/dist/cjs/migrations/0002_lineage.js.map +7 -0
- package/dist/cjs/migrations/0003_execution_revisions.d.ts +15 -0
- package/dist/cjs/migrations/0003_execution_revisions.d.ts.map +1 -0
- package/dist/cjs/migrations/0003_execution_revisions.js +77 -0
- package/dist/cjs/migrations/0003_execution_revisions.js.map +7 -0
- package/dist/cjs/migrations/0004_waiting_request.d.ts +32 -0
- package/dist/cjs/migrations/0004_waiting_request.d.ts.map +1 -0
- package/dist/cjs/migrations/0004_waiting_request.js +42 -0
- package/dist/cjs/migrations/0004_waiting_request.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/test/TestRunStore.d.ts +17 -0
- package/dist/cjs/test/TestRunStore.d.ts.map +1 -0
- package/dist/cjs/test/TestRunStore.js +42 -0
- package/dist/cjs/test/TestRunStore.js.map +7 -0
- package/dist/esm/AttemptStore.d.ts +359 -0
- package/dist/esm/AttemptStore.d.ts.map +1 -0
- package/dist/esm/AttemptStore.js +835 -0
- package/dist/esm/AttemptStore.js.map +1 -0
- package/dist/esm/Heartbeat.d.ts +46 -0
- package/dist/esm/Heartbeat.d.ts.map +1 -0
- package/dist/esm/Heartbeat.js +46 -0
- package/dist/esm/Heartbeat.js.map +1 -0
- package/dist/esm/Migrations.d.ts +42 -0
- package/dist/esm/Migrations.d.ts.map +1 -0
- package/dist/esm/Migrations.js +56 -0
- package/dist/esm/Migrations.js.map +1 -0
- package/dist/esm/Ownership.d.ts +275 -0
- package/dist/esm/Ownership.d.ts.map +1 -0
- package/dist/esm/Ownership.js +299 -0
- package/dist/esm/Ownership.js.map +1 -0
- package/dist/esm/RunStore.d.ts +571 -0
- package/dist/esm/RunStore.d.ts.map +1 -0
- package/dist/esm/RunStore.js +1287 -0
- package/dist/esm/RunStore.js.map +1 -0
- package/dist/esm/RunStoreMetrics.d.ts +151 -0
- package/dist/esm/RunStoreMetrics.d.ts.map +1 -0
- package/dist/esm/RunStoreMetrics.js +120 -0
- package/dist/esm/RunStoreMetrics.js.map +1 -0
- package/dist/esm/index.d.ts +47 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +47 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/Boundary.d.ts +80 -0
- package/dist/esm/internal/Boundary.d.ts.map +1 -0
- package/dist/esm/internal/Boundary.js +109 -0
- package/dist/esm/internal/Boundary.js.map +1 -0
- package/dist/esm/internal/Companion.d.ts +44 -0
- package/dist/esm/internal/Companion.d.ts.map +1 -0
- package/dist/esm/internal/Companion.js +53 -0
- package/dist/esm/internal/Companion.js.map +1 -0
- package/dist/esm/internal/SpanOutcome.d.ts +37 -0
- package/dist/esm/internal/SpanOutcome.d.ts.map +1 -0
- package/dist/esm/internal/SpanOutcome.js +48 -0
- package/dist/esm/internal/SpanOutcome.js.map +1 -0
- package/dist/esm/migrations/0001_initial.d.ts +15 -0
- package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
- package/dist/esm/migrations/0001_initial.js +88 -0
- package/dist/esm/migrations/0001_initial.js.map +1 -0
- package/dist/esm/migrations/0002_lineage.d.ts +27 -0
- package/dist/esm/migrations/0002_lineage.d.ts.map +1 -0
- package/dist/esm/migrations/0002_lineage.js +33 -0
- package/dist/esm/migrations/0002_lineage.js.map +1 -0
- package/dist/esm/migrations/0003_execution_revisions.d.ts +15 -0
- package/dist/esm/migrations/0003_execution_revisions.d.ts.map +1 -0
- package/dist/esm/migrations/0003_execution_revisions.js +58 -0
- package/dist/esm/migrations/0003_execution_revisions.js.map +1 -0
- package/dist/esm/migrations/0004_waiting_request.d.ts +32 -0
- package/dist/esm/migrations/0004_waiting_request.d.ts.map +1 -0
- package/dist/esm/migrations/0004_waiting_request.js +37 -0
- package/dist/esm/migrations/0004_waiting_request.js.map +1 -0
- package/dist/esm/test/TestRunStore.d.ts +17 -0
- package/dist/esm/test/TestRunStore.d.ts.map +1 -0
- package/dist/esm/test/TestRunStore.js +19 -0
- package/dist/esm/test/TestRunStore.js.map +1 -0
- package/docs/README.md +142 -0
- package/docs/api.md +901 -0
- package/docs/concepts/durable-values.md +140 -0
- package/docs/concepts/execution-revisions.md +53 -0
- package/docs/concepts/fencing.md +146 -0
- package/docs/concepts/leases.md +154 -0
- package/docs/concepts/liveness-evidence.md +142 -0
- package/docs/guides/cancel-a-run.md +156 -0
- package/docs/guides/claim-and-finish-a-run.md +175 -0
- package/docs/guides/compose-the-stores.md +142 -0
- package/docs/guides/observe-outcomes.md +182 -0
- package/docs/guides/record-step-attempts.md +170 -0
- package/docs/guides/recover-a-stalled-run.md +173 -0
- package/docs/guides/testing.md +141 -0
- package/docs/installation.md +109 -0
- package/docs/quickstart.md +201 -0
- package/docs/troubleshooting.md +255 -0
- package/package.json +183 -3
- package/src/AttemptStore.ts +1195 -0
- package/src/Heartbeat.ts +47 -0
- package/src/Migrations.ts +59 -0
- package/src/Ownership.ts +407 -0
- package/src/RunStore.ts +2063 -0
- package/src/RunStoreMetrics.ts +165 -0
- package/src/index.ts +51 -0
- package/src/internal/Boundary.ts +130 -0
- package/src/internal/Companion.ts +64 -0
- package/src/internal/SpanOutcome.ts +71 -0
- package/src/migrations/0001_initial.ts +114 -0
- package/src/migrations/0002_lineage.ts +38 -0
- package/src/migrations/0003_execution_revisions.ts +69 -0
- package/src/migrations/0004_waiting_request.ts +39 -0
- package/src/test/TestRunStore.ts +22 -0
|
@@ -0,0 +1,1079 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
var RunStore_exports = {};
|
|
30
|
+
__export(RunStore_exports, {
|
|
31
|
+
OwnershipTransition: () => OwnershipTransition,
|
|
32
|
+
RunStatus: () => RunStatus,
|
|
33
|
+
RunStore: () => RunStore,
|
|
34
|
+
RunStoreError: () => RunStoreError,
|
|
35
|
+
RunStoreErrorCode: () => RunStoreErrorCode,
|
|
36
|
+
TerminalRunStatus: () => TerminalRunStatus,
|
|
37
|
+
TransitionGuard: () => TransitionGuard,
|
|
38
|
+
companionStream: () => companionStream,
|
|
39
|
+
isTerminalRunStatus: () => isTerminalRunStatus,
|
|
40
|
+
layer: () => layer,
|
|
41
|
+
layerNoop: () => layerNoop,
|
|
42
|
+
layerWith: () => layerWith,
|
|
43
|
+
make: () => make,
|
|
44
|
+
makeNoop: () => makeNoop,
|
|
45
|
+
maximumRunJsonDepth: () => maximumRunJsonDepth,
|
|
46
|
+
maximumRunJsonNodes: () => maximumRunJsonNodes,
|
|
47
|
+
ownershipEventType: () => ownershipEventType
|
|
48
|
+
});
|
|
49
|
+
module.exports = __toCommonJS(RunStore_exports);
|
|
50
|
+
var Dialect = __toESM(require("@smthrs/database/Dialect"), 1);
|
|
51
|
+
var import_DurableWriter = require("@smthrs/database/DurableWriter");
|
|
52
|
+
var import_Consensus = require("@smthrs/journal/Consensus");
|
|
53
|
+
var import_JournalEvent = require("@smthrs/journal/JournalEvent");
|
|
54
|
+
var import_OwnerId = require("@smthrs/journal/OwnerId");
|
|
55
|
+
var SqlConsensus = __toESM(require("@smthrs/journal/SqlConsensus"), 1);
|
|
56
|
+
var ObservabilityMetric = __toESM(require("@smthrs/observability/Metric"), 1);
|
|
57
|
+
var import_effect = require("effect");
|
|
58
|
+
var SqlClient = __toESM(require("effect/unstable/sql/SqlClient"), 1);
|
|
59
|
+
var import_Heartbeat = require("./Heartbeat.js");
|
|
60
|
+
var Boundary = __toESM(require("./internal/Boundary.js"), 1);
|
|
61
|
+
var Companion = __toESM(require("./internal/Companion.js"), 1);
|
|
62
|
+
var import_SpanOutcome = require("./internal/SpanOutcome.js");
|
|
63
|
+
var RunStoreMetrics = __toESM(require("./RunStoreMetrics.js"), 1);
|
|
64
|
+
const UnknownFromJsonString = import_effect.Schema.fromJsonString(import_effect.Schema.Unknown);
|
|
65
|
+
const loneSurrogate = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;
|
|
66
|
+
const wellFormedIdentifier = (value) => value.length > 0 && !loneSurrogate.test(value);
|
|
67
|
+
const NonNegativeSafeInt = import_effect.Schema.Int.check(
|
|
68
|
+
import_effect.Schema.isGreaterThanOrEqualTo(0),
|
|
69
|
+
import_effect.Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER)
|
|
70
|
+
);
|
|
71
|
+
const durableIdentifier = import_effect.Schema.NonEmptyString.check(
|
|
72
|
+
import_effect.Schema.isMaxLength(Boundary.maximumIdentifierLength),
|
|
73
|
+
import_effect.Schema.makeFilter(
|
|
74
|
+
(value) => wellFormedIdentifier(value) && Boundary.isDurableText(value),
|
|
75
|
+
{ title: "durableIdentifier" }
|
|
76
|
+
)
|
|
77
|
+
);
|
|
78
|
+
const maximumRunJsonDepth = 128;
|
|
79
|
+
const maximumRunJsonNodes = 1e5;
|
|
80
|
+
const runJsonLimits = {
|
|
81
|
+
maxDepth: maximumRunJsonDepth,
|
|
82
|
+
maxMembers: maximumRunJsonNodes,
|
|
83
|
+
maxNodes: maximumRunJsonNodes
|
|
84
|
+
};
|
|
85
|
+
const RunStatus = import_effect.Schema.Literals([
|
|
86
|
+
"pending",
|
|
87
|
+
"running",
|
|
88
|
+
"suspended",
|
|
89
|
+
"completed",
|
|
90
|
+
"failed",
|
|
91
|
+
"cancelled"
|
|
92
|
+
]);
|
|
93
|
+
const TerminalRunStatus = import_effect.Schema.Literals(["completed", "failed", "cancelled"]);
|
|
94
|
+
const isTerminalRunStatus = (status) => status === "completed" || status === "failed" || status === "cancelled";
|
|
95
|
+
const RunStoreErrorCode = import_effect.Schema.Literals([
|
|
96
|
+
"invalid_run",
|
|
97
|
+
"not_found_row",
|
|
98
|
+
"constraint",
|
|
99
|
+
"decode_failed",
|
|
100
|
+
"persistence_failed"
|
|
101
|
+
]);
|
|
102
|
+
class RunStoreError extends import_effect.Schema.TaggedError()("@smthrs/run-store/RunStoreError", {
|
|
103
|
+
code: RunStoreErrorCode,
|
|
104
|
+
method: import_effect.Schema.String,
|
|
105
|
+
message: import_effect.Schema.String,
|
|
106
|
+
cause: import_effect.Schema.Unknown
|
|
107
|
+
}) {
|
|
108
|
+
}
|
|
109
|
+
const TransitionGuard = import_effect.Schema.Struct({
|
|
110
|
+
cancelRequested: import_effect.Schema.optional(import_effect.Schema.Literals(["absent", "present"]))
|
|
111
|
+
});
|
|
112
|
+
const OwnershipTransition = import_effect.Schema.Literals(["claimed", "activated", "released", "stolen", "expired"]);
|
|
113
|
+
const companionStream = Companion.stream;
|
|
114
|
+
const ownershipEventType = (transition) => `flows.consensus.${transition}`;
|
|
115
|
+
class RunStore extends import_effect.Context.Service()("@smthrs/run-store/RunStore") {
|
|
116
|
+
}
|
|
117
|
+
const claimed = (claimedAtMs) => ({ _tag: "Claimed", claimedAtMs });
|
|
118
|
+
const notFound = { _tag: "NotFound" };
|
|
119
|
+
const alreadyClaimed = { _tag: "AlreadyClaimed" };
|
|
120
|
+
const heartbeatFresh = { _tag: "HeartbeatFresh" };
|
|
121
|
+
const snapshotChanged = { _tag: "SnapshotChanged" };
|
|
122
|
+
const evidenceRequired = { _tag: "EvidenceRequired" };
|
|
123
|
+
const activated = { _tag: "Activated" };
|
|
124
|
+
const claimLost = { _tag: "ClaimLost" };
|
|
125
|
+
const abandoned = { _tag: "Abandoned" };
|
|
126
|
+
const recovered = { _tag: "Recovered" };
|
|
127
|
+
const claimFresh = { _tag: "ClaimFresh" };
|
|
128
|
+
const claimChanged = { _tag: "ClaimChanged" };
|
|
129
|
+
const livenessUnconfirmed = { _tag: "LivenessUnconfirmed" };
|
|
130
|
+
const updated = { _tag: "Updated" };
|
|
131
|
+
const fenceLost = { _tag: "FenceLost" };
|
|
132
|
+
const transitioned = { _tag: "Transitioned" };
|
|
133
|
+
const guardFailed = { _tag: "GuardFailed" };
|
|
134
|
+
const heartbeatStaleAfterMs = import_effect.Duration.toMillis(import_Heartbeat.heartbeatStaleAfter);
|
|
135
|
+
const heartbeatSkewAllowanceMs = import_effect.Duration.toMillis(import_Heartbeat.heartbeatSkewAllowance);
|
|
136
|
+
const terminalStatuses = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
|
|
137
|
+
const DatabaseRunRow = import_effect.Schema.Struct({
|
|
138
|
+
runId: durableIdentifier,
|
|
139
|
+
status: RunStatus,
|
|
140
|
+
createdAtMs: NonNegativeSafeInt,
|
|
141
|
+
startedAtMs: import_effect.Schema.NullOr(NonNegativeSafeInt),
|
|
142
|
+
finishedAtMs: import_effect.Schema.NullOr(NonNegativeSafeInt),
|
|
143
|
+
ownerHostId: import_effect.Schema.NullOr(durableIdentifier),
|
|
144
|
+
ownerPid: import_effect.Schema.NullOr(NonNegativeSafeInt),
|
|
145
|
+
ownerNonce: import_effect.Schema.NullOr(durableIdentifier),
|
|
146
|
+
heartbeatAtMs: import_effect.Schema.NullOr(NonNegativeSafeInt),
|
|
147
|
+
claimHostId: import_effect.Schema.NullOr(durableIdentifier),
|
|
148
|
+
claimPid: import_effect.Schema.NullOr(NonNegativeSafeInt),
|
|
149
|
+
claimNonce: import_effect.Schema.NullOr(durableIdentifier),
|
|
150
|
+
claimedAtMs: import_effect.Schema.NullOr(NonNegativeSafeInt),
|
|
151
|
+
parentRunId: import_effect.Schema.NullOr(durableIdentifier),
|
|
152
|
+
cancelRequestedAtMs: import_effect.Schema.NullOr(NonNegativeSafeInt),
|
|
153
|
+
lineageId: import_effect.Schema.NullOr(durableIdentifier),
|
|
154
|
+
roundOrdinal: import_effect.Schema.NullOr(NonNegativeSafeInt),
|
|
155
|
+
stateJson: import_effect.Schema.String
|
|
156
|
+
});
|
|
157
|
+
const runStoreError = (method, code, message, cause) => new RunStoreError({
|
|
158
|
+
code,
|
|
159
|
+
method,
|
|
160
|
+
message: `${code}: RunStore.${method}: ${message}`,
|
|
161
|
+
cause
|
|
162
|
+
});
|
|
163
|
+
const databaseFailure = (cause) => {
|
|
164
|
+
let current = cause;
|
|
165
|
+
for (let depth = 0; depth < 8 && typeof current === "object" && current !== null; depth++) {
|
|
166
|
+
if (current instanceof import_DurableWriter.DatabaseError) return current;
|
|
167
|
+
current = current.cause;
|
|
168
|
+
}
|
|
169
|
+
return void 0;
|
|
170
|
+
};
|
|
171
|
+
const persistenceError = (method, cause) => {
|
|
172
|
+
if (import_effect.Schema.is(RunStoreError)(cause)) return cause;
|
|
173
|
+
const database = cause instanceof import_DurableWriter.DatabaseError ? cause : databaseFailure(cause.cause);
|
|
174
|
+
const code = database?.code === "constraint" ? "constraint" : "persistence_failed";
|
|
175
|
+
return runStoreError(method, code, "database operation failed", {
|
|
176
|
+
category: code,
|
|
177
|
+
// Keep the classification for outer transaction retries, never driver
|
|
178
|
+
// causes that may contain executable state or bound SQL values.
|
|
179
|
+
cause: new import_DurableWriter.DatabaseError({ code: database?.code ?? "unknown" })
|
|
180
|
+
});
|
|
181
|
+
};
|
|
182
|
+
const invalidRunError = (method, causeOrField, detail = "violates the durable contract") => runStoreError(
|
|
183
|
+
method,
|
|
184
|
+
"invalid_run",
|
|
185
|
+
"run input is invalid",
|
|
186
|
+
typeof causeOrField === "string" ? { field: causeOrField, detail } : causeOrField
|
|
187
|
+
);
|
|
188
|
+
const stateAdmission = (value) => Boundary.admitJsonText(value, runJsonLimits);
|
|
189
|
+
const isJsonString = (value) => import_effect.Schema.decodeUnknownResult(UnknownFromJsonString)(value)._tag === "Success";
|
|
190
|
+
const validOwner = (value) => import_effect.Schema.is(import_OwnerId.OwnerId)(value) && Boundary.isDurableText(value.hostId) && Boundary.isDurableText(value.nonce) && Number.isSafeInteger(value.pid) && value.pid >= 0;
|
|
191
|
+
const validTimestamp = (value) => Number.isSafeInteger(value) && value >= 0;
|
|
192
|
+
const inertRecord = (input, allowed) => {
|
|
193
|
+
if (typeof input !== "object" || input === null) return void 0;
|
|
194
|
+
try {
|
|
195
|
+
const prototype = Object.getPrototypeOf(input);
|
|
196
|
+
if (prototype !== Object.prototype && prototype !== null) return void 0;
|
|
197
|
+
const fields = /* @__PURE__ */ Object.create(null);
|
|
198
|
+
for (const key of Reflect.ownKeys(input)) {
|
|
199
|
+
const descriptor = Object.getOwnPropertyDescriptor(input, key);
|
|
200
|
+
if (descriptor === void 0 || !descriptor.enumerable) continue;
|
|
201
|
+
if (!("value" in descriptor) || allowed !== void 0 && (typeof key !== "string" || !allowed.has(key))) {
|
|
202
|
+
return void 0;
|
|
203
|
+
}
|
|
204
|
+
if (typeof key === "string") fields[key] = descriptor.value;
|
|
205
|
+
}
|
|
206
|
+
return fields;
|
|
207
|
+
} catch {
|
|
208
|
+
return void 0;
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
const snapshotOwner = (method, field, input) => import_effect.Effect.suspend(() => {
|
|
212
|
+
const fields = inertRecord(input, /* @__PURE__ */ new Set(["hostId", "pid", "nonce"]));
|
|
213
|
+
if (fields === void 0) {
|
|
214
|
+
return import_effect.Effect.fail(invalidRunError(method, field, "must be an inert owner record"));
|
|
215
|
+
}
|
|
216
|
+
const owner = Object.freeze({
|
|
217
|
+
hostId: fields.hostId,
|
|
218
|
+
pid: fields.pid,
|
|
219
|
+
nonce: fields.nonce
|
|
220
|
+
});
|
|
221
|
+
return validOwner(owner) ? import_effect.Effect.succeed(owner) : import_effect.Effect.fail(invalidRunError(method, field, "must be a valid owner identity"));
|
|
222
|
+
});
|
|
223
|
+
const snapshotRunId = (method, input) => typeof input === "string" && wellFormedIdentifier(input) && Boundary.isDurableText(input) ? import_effect.Effect.succeed(input) : import_effect.Effect.fail(invalidRunError(method, "runId", "must be well-formed durable text"));
|
|
224
|
+
const snapshotTimestamp = (method, field, input) => typeof input === "number" && validTimestamp(input) ? import_effect.Effect.succeed(input) : import_effect.Effect.fail(
|
|
225
|
+
invalidRunError(method, field, "must be a non-negative safe integer")
|
|
226
|
+
);
|
|
227
|
+
const snapshotLeaseReading = (method, field, input, cause) => import_effect.Effect.gen(function* () {
|
|
228
|
+
const nowMs = yield* snapshotTimestamp(method, field, input);
|
|
229
|
+
const clockMs = yield* import_effect.Clock.currentTimeMillis.pipe(import_effect.Effect.map(Math.floor));
|
|
230
|
+
if (nowMs > clockMs + heartbeatSkewAllowanceMs) {
|
|
231
|
+
return yield* import_effect.Effect.fail(
|
|
232
|
+
invalidRunError(method, {
|
|
233
|
+
...cause,
|
|
234
|
+
field,
|
|
235
|
+
nowMs,
|
|
236
|
+
clockMs,
|
|
237
|
+
detail: "runs ahead of the store clock by more than the heartbeat skew allowance"
|
|
238
|
+
})
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
return nowMs;
|
|
242
|
+
});
|
|
243
|
+
const snapshotState = (method, input, cause) => {
|
|
244
|
+
const admitted = stateAdmission(input);
|
|
245
|
+
return admitted.ok ? import_effect.Effect.succeed(admitted.value) : import_effect.Effect.fail(
|
|
246
|
+
invalidRunError(
|
|
247
|
+
method,
|
|
248
|
+
cause,
|
|
249
|
+
admitted.complaint
|
|
250
|
+
)
|
|
251
|
+
);
|
|
252
|
+
};
|
|
253
|
+
const snapshotCreateOptions = (input) => import_effect.Effect.suspend(() => {
|
|
254
|
+
const value = input ?? {};
|
|
255
|
+
const fields = inertRecord(value, /* @__PURE__ */ new Set(["parentRunId", "lineageId", "roundOrdinal"]));
|
|
256
|
+
if (fields === void 0) {
|
|
257
|
+
return import_effect.Effect.fail(invalidRunError("create", "options", "must be an inert data record"));
|
|
258
|
+
}
|
|
259
|
+
const rawParent = fields.parentRunId;
|
|
260
|
+
const rawLineage = fields.lineageId;
|
|
261
|
+
const rawRound = fields.roundOrdinal;
|
|
262
|
+
const parentRunId = rawParent === void 0 ? null : rawParent;
|
|
263
|
+
const lineageId = rawLineage === void 0 ? null : rawLineage;
|
|
264
|
+
const roundOrdinal = rawRound === void 0 ? null : rawRound;
|
|
265
|
+
if (parentRunId !== null && (typeof parentRunId !== "string" || !wellFormedIdentifier(parentRunId) || !Boundary.isDurableText(parentRunId)) || lineageId !== null && (typeof lineageId !== "string" || !wellFormedIdentifier(lineageId) || !Boundary.isDurableText(lineageId)) || roundOrdinal !== null && (typeof roundOrdinal !== "number" || !validTimestamp(roundOrdinal)) || lineageId === null !== (roundOrdinal === null)) return import_effect.Effect.fail(invalidRunError("create", "options", "contains invalid ancestry or lineage"));
|
|
266
|
+
return import_effect.Effect.succeed(Object.freeze({ parentRunId, lineageId, roundOrdinal }));
|
|
267
|
+
});
|
|
268
|
+
const inertFields = (input, required) => {
|
|
269
|
+
const fields = inertRecord(input);
|
|
270
|
+
return fields !== void 0 && required.every((key) => Object.hasOwn(fields, key)) ? fields : void 0;
|
|
271
|
+
};
|
|
272
|
+
const snapshotExpected = (method, input) => import_effect.Effect.gen(function* () {
|
|
273
|
+
const fields = inertFields(input, ["status", "owner", "heartbeatAtMs"]);
|
|
274
|
+
if (fields === void 0) {
|
|
275
|
+
return yield* import_effect.Effect.fail(invalidRunError(method, "expected", "must be an inert snapshot record"));
|
|
276
|
+
}
|
|
277
|
+
const status = fields.status;
|
|
278
|
+
const rawOwner = fields.owner;
|
|
279
|
+
const heartbeatAtMs = fields.heartbeatAtMs;
|
|
280
|
+
if (!import_effect.Schema.is(RunStatus)(status) || heartbeatAtMs !== null && (typeof heartbeatAtMs !== "number" || !validTimestamp(heartbeatAtMs))) {
|
|
281
|
+
return yield* import_effect.Effect.fail(invalidRunError(method, "expected", "contains an invalid status or heartbeat"));
|
|
282
|
+
}
|
|
283
|
+
const owner = rawOwner === null ? null : yield* snapshotOwner(method, "expected.owner", rawOwner);
|
|
284
|
+
if (status === "running") {
|
|
285
|
+
if (owner === null || heartbeatAtMs === null) {
|
|
286
|
+
return yield* import_effect.Effect.fail(invalidRunError(method, "expected", "violates ownership invariants"));
|
|
287
|
+
}
|
|
288
|
+
return Object.freeze({ status, owner, heartbeatAtMs });
|
|
289
|
+
} else {
|
|
290
|
+
if (owner !== null || heartbeatAtMs !== null) {
|
|
291
|
+
return yield* import_effect.Effect.fail(invalidRunError(method, "expected", "violates ownership invariants"));
|
|
292
|
+
}
|
|
293
|
+
return Object.freeze({ status, owner, heartbeatAtMs });
|
|
294
|
+
}
|
|
295
|
+
});
|
|
296
|
+
const snapshotEvidence = (method, input) => import_effect.Effect.gen(function* () {
|
|
297
|
+
const fields = inertRecord(input, /* @__PURE__ */ new Set(["expectedOwner", "checkedAtMs", "kind"]));
|
|
298
|
+
if (fields === void 0) {
|
|
299
|
+
return yield* import_effect.Effect.fail(invalidRunError(method, "evidence", "must be an inert evidence record"));
|
|
300
|
+
}
|
|
301
|
+
const expectedOwner = yield* snapshotOwner(method, "evidence.expectedOwner", fields.expectedOwner);
|
|
302
|
+
const checkedAtMs = yield* snapshotTimestamp(method, "evidence.checkedAtMs", fields.checkedAtMs);
|
|
303
|
+
const kind = fields.kind;
|
|
304
|
+
if (kind !== "same-host-pid-dead" && kind !== "cross-host-unreachable-stale" && kind !== "lease-expired") return yield* import_effect.Effect.fail(invalidRunError(method, "evidence.kind"));
|
|
305
|
+
return Object.freeze({ expectedOwner, checkedAtMs, kind });
|
|
306
|
+
});
|
|
307
|
+
const snapshotGuard = (input) => import_effect.Effect.suspend(() => {
|
|
308
|
+
if (input === void 0) return import_effect.Effect.succeed(void 0);
|
|
309
|
+
const fields = inertRecord(input, /* @__PURE__ */ new Set(["cancelRequested"]));
|
|
310
|
+
if (fields === void 0) {
|
|
311
|
+
return import_effect.Effect.fail(invalidRunError("transitionOwned", "guard", "must be an inert exact guard"));
|
|
312
|
+
}
|
|
313
|
+
const cancelRequested = fields.cancelRequested;
|
|
314
|
+
if (cancelRequested !== void 0 && cancelRequested !== "absent" && cancelRequested !== "present") {
|
|
315
|
+
return import_effect.Effect.fail(invalidRunError("transitionOwned", "guard.cancelRequested"));
|
|
316
|
+
}
|
|
317
|
+
return import_effect.Effect.succeed(Object.freeze(cancelRequested === void 0 ? {} : { cancelRequested }));
|
|
318
|
+
});
|
|
319
|
+
const ownerFromColumns = (hostId, pid, nonce) => {
|
|
320
|
+
if (hostId === null && pid === null && nonce === null) return null;
|
|
321
|
+
if (hostId !== null && pid !== null && nonce !== null) {
|
|
322
|
+
return Object.freeze({ hostId, pid, nonce });
|
|
323
|
+
}
|
|
324
|
+
return void 0;
|
|
325
|
+
};
|
|
326
|
+
const rowMatchesClaim = (row, claimant, claimedAtMs) => row.claimHostId === claimant.hostId && row.claimPid === claimant.pid && row.claimNonce === claimant.nonce && row.claimedAtMs === claimedAtMs;
|
|
327
|
+
const rowMatchesSnapshot = (row, expected) => row.status === expected.status && row.ownerHostId === (expected.owner?.hostId ?? null) && row.ownerPid === (expected.owner?.pid ?? null) && row.ownerNonce === (expected.owner?.nonce ?? null) && row.heartbeatAtMs === expected.heartbeatAtMs;
|
|
328
|
+
const rowOwnedBy = (row, owner) => row.status === "running" && row.ownerHostId === owner.hostId && row.ownerPid === owner.pid && row.ownerNonce === owner.nonce;
|
|
329
|
+
const claimColumnsFree = (row) => row.claimHostId === null && row.claimPid === null && row.claimNonce === null && row.claimedAtMs === null;
|
|
330
|
+
const decodeRunRow = (method, runId, input) => import_effect.Schema.decodeUnknownEffect(DatabaseRunRow)(input).pipe(
|
|
331
|
+
import_effect.Effect.mapError(
|
|
332
|
+
() => runStoreError(method, "decode_failed", "could not decode flows_runs row", { runId, stage: "row-schema" })
|
|
333
|
+
),
|
|
334
|
+
import_effect.Effect.flatMap((row) => {
|
|
335
|
+
const owner = ownerFromColumns(row.ownerHostId, row.ownerPid, row.ownerNonce);
|
|
336
|
+
const claim = ownerFromColumns(row.claimHostId, row.claimPid, row.claimNonce);
|
|
337
|
+
const admittedState = stateAdmission(row.stateJson);
|
|
338
|
+
const stateJsonValid = isJsonString(row.stateJson);
|
|
339
|
+
const invalidOwner = owner === void 0 || owner === null && row.heartbeatAtMs !== null || owner !== null && row.heartbeatAtMs === null || (row.status === "running" ? owner === null : owner !== null);
|
|
340
|
+
const invalidClaim = claim === void 0 || claim === null && row.claimedAtMs !== null || claim !== null && row.claimedAtMs === null;
|
|
341
|
+
if (invalidOwner || invalidClaim || !admittedState.ok) {
|
|
342
|
+
return import_effect.Effect.fail(
|
|
343
|
+
runStoreError(method, "decode_failed", "flows_runs row violates durable invariants", {
|
|
344
|
+
runId,
|
|
345
|
+
status: row.status,
|
|
346
|
+
heartbeatAtMs: row.heartbeatAtMs,
|
|
347
|
+
claimedAtMs: row.claimedAtMs,
|
|
348
|
+
hasClaimColumns: row.claimHostId !== null || row.claimPid !== null || row.claimNonce !== null,
|
|
349
|
+
hasOwnerColumns: row.ownerHostId !== null || row.ownerPid !== null || row.ownerNonce !== null,
|
|
350
|
+
stateJsonValid
|
|
351
|
+
})
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
return import_effect.Effect.succeed(Object.freeze({
|
|
355
|
+
runId: row.runId,
|
|
356
|
+
status: row.status,
|
|
357
|
+
createdAtMs: row.createdAtMs,
|
|
358
|
+
startedAtMs: row.startedAtMs,
|
|
359
|
+
finishedAtMs: row.finishedAtMs,
|
|
360
|
+
owner,
|
|
361
|
+
heartbeatAtMs: row.heartbeatAtMs,
|
|
362
|
+
claim,
|
|
363
|
+
claimedAtMs: row.claimedAtMs,
|
|
364
|
+
parentRunId: row.parentRunId,
|
|
365
|
+
cancelRequestedAtMs: row.cancelRequestedAtMs,
|
|
366
|
+
lineageId: row.lineageId,
|
|
367
|
+
roundOrdinal: row.roundOrdinal,
|
|
368
|
+
stateJson: row.stateJson
|
|
369
|
+
}));
|
|
370
|
+
})
|
|
371
|
+
);
|
|
372
|
+
const lineageMembership = (sql, runId) => sql`(
|
|
373
|
+
lineage_id = (SELECT COALESCE(lineage_id, run_id) FROM flows_runs WHERE run_id = ${runId})
|
|
374
|
+
OR (
|
|
375
|
+
run_id = (SELECT COALESCE(lineage_id, run_id) FROM flows_runs WHERE run_id = ${runId})
|
|
376
|
+
AND (
|
|
377
|
+
lineage_id IS NULL
|
|
378
|
+
OR lineage_id = (SELECT COALESCE(lineage_id, run_id) FROM flows_runs WHERE run_id = ${runId})
|
|
379
|
+
)
|
|
380
|
+
)
|
|
381
|
+
)`;
|
|
382
|
+
const classifyCancellationMiss = (method, runId, rows) => import_effect.Effect.gen(function* () {
|
|
383
|
+
let outcome = notFound;
|
|
384
|
+
for (const row of rows) {
|
|
385
|
+
const status = yield* import_effect.Schema.decodeUnknownEffect(RunStatus)(row.status).pipe(
|
|
386
|
+
import_effect.Effect.mapError((cause) => runStoreError(method, "decode_failed", "could not decode flows_runs status", cause))
|
|
387
|
+
);
|
|
388
|
+
if (isTerminalRunStatus(status)) {
|
|
389
|
+
if (outcome._tag !== "AlreadyRequested") outcome = { _tag: "Terminal", status };
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
if (row.requestedAtMs === null) {
|
|
393
|
+
return yield* import_effect.Effect.fail(
|
|
394
|
+
runStoreError(method, "persistence_failed", "guarded cancellation update missed a live unrequested row", {
|
|
395
|
+
runId,
|
|
396
|
+
stage: "write-invariant"
|
|
397
|
+
})
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
outcome = { _tag: "AlreadyRequested", requestedAtMs: Number(row.requestedAtMs) };
|
|
401
|
+
}
|
|
402
|
+
return outcome;
|
|
403
|
+
});
|
|
404
|
+
const selectRun = (sql, runId, mode = "single") => sql`
|
|
405
|
+
SELECT
|
|
406
|
+
run_id AS "runId",
|
|
407
|
+
status AS "status",
|
|
408
|
+
created_at_ms AS "createdAtMs",
|
|
409
|
+
started_at_ms AS "startedAtMs",
|
|
410
|
+
finished_at_ms AS "finishedAtMs",
|
|
411
|
+
owner_host_id AS "ownerHostId",
|
|
412
|
+
owner_pid AS "ownerPid",
|
|
413
|
+
owner_nonce AS "ownerNonce",
|
|
414
|
+
heartbeat_at_ms AS "heartbeatAtMs",
|
|
415
|
+
claim_host_id AS "claimHostId",
|
|
416
|
+
claim_pid AS "claimPid",
|
|
417
|
+
claim_nonce AS "claimNonce",
|
|
418
|
+
claimed_at_ms AS "claimedAtMs",
|
|
419
|
+
parent_run_id AS "parentRunId",
|
|
420
|
+
cancel_requested_at_ms AS "cancelRequestedAtMs",
|
|
421
|
+
lineage_id AS "lineageId",
|
|
422
|
+
round_ordinal AS "roundOrdinal",
|
|
423
|
+
state_json AS "stateJson"
|
|
424
|
+
FROM flows_runs
|
|
425
|
+
WHERE ${mode === "latest" ? sql`run_id = COALESCE(
|
|
426
|
+
(SELECT run_id FROM flows_runs
|
|
427
|
+
WHERE lineage_id = (SELECT COALESCE(lineage_id, run_id) FROM flows_runs WHERE run_id = ${runId})
|
|
428
|
+
ORDER BY round_ordinal DESC LIMIT 1),
|
|
429
|
+
(SELECT run_id FROM flows_runs WHERE run_id = ${runId})
|
|
430
|
+
)` : mode === "lineage" ? lineageMembership(sql, runId) : sql`run_id = ${runId}`}
|
|
431
|
+
ORDER BY COALESCE(round_ordinal, 0), run_id
|
|
432
|
+
`;
|
|
433
|
+
const classifyClaimLoss = (row, nowMs) => {
|
|
434
|
+
if (row === void 0) return notFound;
|
|
435
|
+
if (row.claimHostId !== null) return alreadyClaimed;
|
|
436
|
+
if (row.status === "running" && row.heartbeatAtMs !== null && row.heartbeatAtMs >= nowMs - heartbeatStaleAfterMs) {
|
|
437
|
+
return heartbeatFresh;
|
|
438
|
+
}
|
|
439
|
+
return snapshotChanged;
|
|
440
|
+
};
|
|
441
|
+
const evidenceMatches = (expected, claimant, checkedAtMs, evidence) => expected.status === "running" && expected.owner !== null && (0, import_Consensus.matchesEvidence)(expected.owner, claimant, checkedAtMs, evidence);
|
|
442
|
+
const make = import_effect.Effect.gen(
|
|
443
|
+
function* () {
|
|
444
|
+
const sql = yield* import_effect.Effect.service(SqlClient.SqlClient);
|
|
445
|
+
const writer = yield* import_DurableWriter.DurableWriter;
|
|
446
|
+
const consensus = yield* import_Consensus.Consensus;
|
|
447
|
+
const write = (method, effect) => writer.write(effect).pipe(import_effect.Effect.mapError((cause) => persistenceError(method, cause)));
|
|
448
|
+
const recordTransition = (runId, transition, owner, grantedAtMs) => Companion.append(runId, ownershipEventType(transition), {
|
|
449
|
+
owner: { hostId: owner.hostId, pid: owner.pid, nonce: owner.nonce },
|
|
450
|
+
grantedAtMs
|
|
451
|
+
});
|
|
452
|
+
const read = (method, effect) => effect.pipe(import_effect.Effect.mapError((cause) => persistenceError(method, (0, import_DurableWriter.fromSqlError)(cause))));
|
|
453
|
+
const create = import_effect.Effect.fn("RunStore.create")(
|
|
454
|
+
(runIdInput, stateInput, optionsInput) => import_effect.Effect.gen(function* () {
|
|
455
|
+
const runId = yield* snapshotRunId("create", runIdInput);
|
|
456
|
+
if ((0, import_JournalEvent.isCompanionRunId)(runId)) {
|
|
457
|
+
return yield* import_effect.Effect.fail(invalidRunError("create", "runId", "names a reserved journal companion stream"));
|
|
458
|
+
}
|
|
459
|
+
const { lineageId, parentRunId, roundOrdinal } = yield* snapshotCreateOptions(optionsInput);
|
|
460
|
+
const stateJson = yield* snapshotState("create", stateInput, {
|
|
461
|
+
runId,
|
|
462
|
+
parentRunId,
|
|
463
|
+
lineageId,
|
|
464
|
+
roundOrdinal,
|
|
465
|
+
stateJsonLength: typeof stateInput === "string" ? stateInput.length : null,
|
|
466
|
+
stateJsonValid: isJsonString(stateInput)
|
|
467
|
+
});
|
|
468
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId });
|
|
469
|
+
const createdAtMs = yield* import_effect.Clock.currentTimeMillis.pipe(import_effect.Effect.map(Math.floor));
|
|
470
|
+
yield* write(
|
|
471
|
+
"create",
|
|
472
|
+
sql`
|
|
473
|
+
INSERT INTO flows_runs (
|
|
474
|
+
run_id,
|
|
475
|
+
status,
|
|
476
|
+
created_at_ms,
|
|
477
|
+
started_at_ms,
|
|
478
|
+
finished_at_ms,
|
|
479
|
+
owner_host_id,
|
|
480
|
+
owner_pid,
|
|
481
|
+
owner_nonce,
|
|
482
|
+
heartbeat_at_ms,
|
|
483
|
+
claim_host_id,
|
|
484
|
+
claim_pid,
|
|
485
|
+
claim_nonce,
|
|
486
|
+
claimed_at_ms,
|
|
487
|
+
parent_run_id,
|
|
488
|
+
lineage_id,
|
|
489
|
+
round_ordinal,
|
|
490
|
+
state_json
|
|
491
|
+
) VALUES (
|
|
492
|
+
${runId},
|
|
493
|
+
'pending',
|
|
494
|
+
${createdAtMs},
|
|
495
|
+
NULL,
|
|
496
|
+
NULL,
|
|
497
|
+
NULL,
|
|
498
|
+
NULL,
|
|
499
|
+
NULL,
|
|
500
|
+
NULL,
|
|
501
|
+
NULL,
|
|
502
|
+
NULL,
|
|
503
|
+
NULL,
|
|
504
|
+
NULL,
|
|
505
|
+
${parentRunId},
|
|
506
|
+
${lineageId},
|
|
507
|
+
${roundOrdinal},
|
|
508
|
+
${stateJson}
|
|
509
|
+
)
|
|
510
|
+
`.pipe(import_effect.Effect.asVoid)
|
|
511
|
+
);
|
|
512
|
+
}).pipe(import_SpanOutcome.observeExit)
|
|
513
|
+
);
|
|
514
|
+
const get = import_effect.Effect.fn("RunStore.get")(
|
|
515
|
+
(runIdInput) => import_effect.Effect.gen(function* () {
|
|
516
|
+
const runId = yield* snapshotRunId("get", runIdInput);
|
|
517
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId });
|
|
518
|
+
const rows = yield* read("get", selectRun(sql, runId));
|
|
519
|
+
return rows[0] === void 0 ? yield* import_effect.Effect.fail(
|
|
520
|
+
runStoreError("get", "not_found_row", `run ${runId} was not found`, { runId })
|
|
521
|
+
) : yield* decodeRunRow("get", runId, rows[0]);
|
|
522
|
+
}).pipe(import_SpanOutcome.observeExit)
|
|
523
|
+
);
|
|
524
|
+
const lineage = import_effect.Effect.fn("RunStore.lineage")(
|
|
525
|
+
(runIdInput) => import_effect.Effect.gen(function* () {
|
|
526
|
+
const runId = yield* snapshotRunId("lineage", runIdInput);
|
|
527
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId });
|
|
528
|
+
const rows = yield* read("lineage", selectRun(sql, runId, "lineage"));
|
|
529
|
+
return yield* import_effect.Effect.forEach(rows, (row) => decodeRunRow("lineage", row.runId, row));
|
|
530
|
+
}).pipe(import_SpanOutcome.observeExit)
|
|
531
|
+
);
|
|
532
|
+
const latestRound = import_effect.Effect.fn("RunStore.latestRound")(
|
|
533
|
+
(runIdInput) => import_effect.Effect.gen(function* () {
|
|
534
|
+
const runId = yield* snapshotRunId("latestRound", runIdInput);
|
|
535
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId });
|
|
536
|
+
const rows = yield* read("latestRound", selectRun(sql, runId, "latest"));
|
|
537
|
+
return rows[0] === void 0 ? yield* import_effect.Effect.fail(runStoreError("latestRound", "not_found_row", `run ${runId} was not found`, { runId })) : yield* decodeRunRow("latestRound", rows[0].runId, rows[0]);
|
|
538
|
+
}).pipe(import_SpanOutcome.observeExit)
|
|
539
|
+
);
|
|
540
|
+
const requestCancel = import_effect.Effect.fn("RunStore.requestCancel")(
|
|
541
|
+
(runIdInput, nowMsInput) => import_effect.Effect.gen(function* () {
|
|
542
|
+
const runId = yield* snapshotRunId("requestCancel", runIdInput);
|
|
543
|
+
const nowMs = yield* snapshotTimestamp("requestCancel", "nowMs", nowMsInput);
|
|
544
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId });
|
|
545
|
+
return yield* write(
|
|
546
|
+
"requestCancel",
|
|
547
|
+
import_effect.Effect.gen(function* () {
|
|
548
|
+
const rows = yield* sql`
|
|
549
|
+
UPDATE flows_runs
|
|
550
|
+
SET cancel_requested_at_ms = ${nowMs}
|
|
551
|
+
WHERE run_id = ${runId}
|
|
552
|
+
AND cancel_requested_at_ms IS NULL
|
|
553
|
+
AND status NOT IN ('completed', 'failed', 'cancelled')
|
|
554
|
+
RETURNING cancel_requested_at_ms AS "requestedAtMs"
|
|
555
|
+
`;
|
|
556
|
+
if (rows[0] !== void 0) {
|
|
557
|
+
return { _tag: "CancelRequested", requestedAtMs: Number(rows[0].requestedAtMs) };
|
|
558
|
+
}
|
|
559
|
+
const current = yield* sql`
|
|
560
|
+
SELECT cancel_requested_at_ms AS "requestedAtMs", status AS "status"
|
|
561
|
+
FROM flows_runs WHERE run_id = ${runId}
|
|
562
|
+
`;
|
|
563
|
+
return yield* classifyCancellationMiss("requestCancel", runId, current);
|
|
564
|
+
})
|
|
565
|
+
);
|
|
566
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)())
|
|
567
|
+
);
|
|
568
|
+
const requestCancelLineage = import_effect.Effect.fn("RunStore.requestCancelLineage")(
|
|
569
|
+
(runIdInput, nowMsInput) => import_effect.Effect.gen(function* () {
|
|
570
|
+
const runId = yield* snapshotRunId("requestCancelLineage", runIdInput);
|
|
571
|
+
const nowMs = yield* snapshotTimestamp("requestCancelLineage", "nowMs", nowMsInput);
|
|
572
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId });
|
|
573
|
+
return yield* write(
|
|
574
|
+
"requestCancelLineage",
|
|
575
|
+
import_effect.Effect.gen(function* () {
|
|
576
|
+
const rows = yield* sql`
|
|
577
|
+
UPDATE flows_runs
|
|
578
|
+
SET cancel_requested_at_ms = ${nowMs}
|
|
579
|
+
WHERE ${lineageMembership(sql, runId)}
|
|
580
|
+
AND cancel_requested_at_ms IS NULL
|
|
581
|
+
AND status NOT IN ('completed', 'failed', 'cancelled')
|
|
582
|
+
RETURNING cancel_requested_at_ms AS "requestedAtMs"
|
|
583
|
+
`;
|
|
584
|
+
if (rows[0] !== void 0) {
|
|
585
|
+
return { _tag: "CancelRequested", requestedAtMs: Number(rows[0].requestedAtMs) };
|
|
586
|
+
}
|
|
587
|
+
const members = yield* sql`
|
|
588
|
+
SELECT cancel_requested_at_ms AS "requestedAtMs", status AS "status"
|
|
589
|
+
FROM flows_runs
|
|
590
|
+
WHERE ${lineageMembership(sql, runId)}
|
|
591
|
+
ORDER BY COALESCE(round_ordinal, 0), run_id
|
|
592
|
+
`;
|
|
593
|
+
return yield* classifyCancellationMiss("requestCancelLineage", runId, members);
|
|
594
|
+
})
|
|
595
|
+
);
|
|
596
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)())
|
|
597
|
+
);
|
|
598
|
+
const claim = import_effect.Effect.fn("RunStore.claim")(
|
|
599
|
+
(runIdInput, expectedInput, claimantInput, nowMsInput) => import_effect.Effect.gen(function* () {
|
|
600
|
+
const runId = yield* snapshotRunId("claim", runIdInput);
|
|
601
|
+
const expected = yield* snapshotExpected("claim", expectedInput);
|
|
602
|
+
const claimant = yield* snapshotOwner("claim", "claimant", claimantInput);
|
|
603
|
+
const nowMs = yield* snapshotLeaseReading(
|
|
604
|
+
"claim",
|
|
605
|
+
"nowMs",
|
|
606
|
+
nowMsInput,
|
|
607
|
+
{ runId }
|
|
608
|
+
);
|
|
609
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, claimantHostId: claimant.hostId });
|
|
610
|
+
return yield* write(
|
|
611
|
+
"claim",
|
|
612
|
+
import_effect.Effect.gen(function* () {
|
|
613
|
+
const row = (yield* selectRun(sql, runId))[0];
|
|
614
|
+
const admissible = row !== void 0 && (row.status === "pending" || row.status === "suspended") && rowMatchesSnapshot(row, expected) && claimColumnsFree(row);
|
|
615
|
+
if (!admissible) return classifyClaimLoss(row, nowMs);
|
|
616
|
+
const grant = yield* consensus.claim(runId, claimant, nowMs);
|
|
617
|
+
if (grant._tag === "Rejected") {
|
|
618
|
+
return classifyClaimLoss(row, nowMs);
|
|
619
|
+
}
|
|
620
|
+
yield* sql`
|
|
621
|
+
UPDATE flows_runs
|
|
622
|
+
SET
|
|
623
|
+
claim_host_id = ${claimant.hostId},
|
|
624
|
+
claim_pid = ${claimant.pid},
|
|
625
|
+
claim_nonce = ${claimant.nonce},
|
|
626
|
+
claimed_at_ms = ${grant.grantedAtMs}
|
|
627
|
+
WHERE run_id = ${runId}
|
|
628
|
+
`;
|
|
629
|
+
yield* recordTransition(runId, "claimed", claimant, grant.grantedAtMs);
|
|
630
|
+
return claimed(grant.grantedAtMs);
|
|
631
|
+
})
|
|
632
|
+
);
|
|
633
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)(
|
|
634
|
+
(outcome) => RunStoreMetrics.claim[outcome._tag],
|
|
635
|
+
import_effect.Metric.withAttributes(RunStoreMetrics.claims, { op: "claim" })
|
|
636
|
+
))
|
|
637
|
+
);
|
|
638
|
+
const claimAndOwn = import_effect.Effect.fn("RunStore.claimAndOwn")(
|
|
639
|
+
(runIdInput, expectedInput, ownerInput, nowMsInput, evidenceInput) => import_effect.Effect.gen(function* () {
|
|
640
|
+
const runId = yield* snapshotRunId("claimAndOwn", runIdInput);
|
|
641
|
+
const expected = yield* snapshotExpected("claimAndOwn", expectedInput);
|
|
642
|
+
const owner = yield* snapshotOwner("claimAndOwn", "owner", ownerInput);
|
|
643
|
+
const nowMs = yield* snapshotLeaseReading(
|
|
644
|
+
"claimAndOwn",
|
|
645
|
+
"nowMs",
|
|
646
|
+
nowMsInput,
|
|
647
|
+
{ runId }
|
|
648
|
+
);
|
|
649
|
+
const evidence = evidenceInput === void 0 ? void 0 : yield* snapshotEvidence("claimAndOwn", evidenceInput);
|
|
650
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, ownerHostId: owner.hostId });
|
|
651
|
+
return yield* import_effect.Effect.suspend(() => {
|
|
652
|
+
if (expected.status === "running" && !(0, import_Consensus.sameOwner)(expected.owner, owner) && !(evidence !== void 0 && evidenceMatches(expected, owner, nowMs, evidence))) {
|
|
653
|
+
return read("claimAndOwn", selectRun(sql, runId)).pipe(
|
|
654
|
+
import_effect.Effect.map((current) => {
|
|
655
|
+
const row = current[0];
|
|
656
|
+
const loss = classifyClaimLoss(row, nowMs);
|
|
657
|
+
return loss === snapshotChanged && row !== void 0 && rowMatchesSnapshot(row, expected) ? evidenceRequired : loss;
|
|
658
|
+
})
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
return write(
|
|
662
|
+
"claimAndOwn",
|
|
663
|
+
import_effect.Effect.gen(function* () {
|
|
664
|
+
const row = (yield* selectRun(sql, runId))[0];
|
|
665
|
+
const admissible = row !== void 0 && (row.status === "pending" || row.status === "suspended" || row.status === "running") && rowMatchesSnapshot(row, expected) && claimColumnsFree(row) && (row.status !== "running" || row.heartbeatAtMs === null || row.heartbeatAtMs < nowMs - heartbeatStaleAfterMs);
|
|
666
|
+
if (!admissible) return classifyClaimLoss(row, nowMs);
|
|
667
|
+
const expectedOwner = expected.owner;
|
|
668
|
+
let grant;
|
|
669
|
+
let transition = "claimed";
|
|
670
|
+
if (expectedOwner === null) {
|
|
671
|
+
grant = yield* consensus.claim(runId, owner, nowMs);
|
|
672
|
+
} else if ((0, import_Consensus.sameOwner)(expectedOwner, owner)) {
|
|
673
|
+
yield* consensus.release(runId, owner);
|
|
674
|
+
yield* recordTransition(runId, "released", owner, null);
|
|
675
|
+
grant = yield* consensus.claim(runId, owner, nowMs);
|
|
676
|
+
} else {
|
|
677
|
+
grant = yield* consensus.steal(runId, owner, nowMs, evidence);
|
|
678
|
+
transition = "stolen";
|
|
679
|
+
}
|
|
680
|
+
if (grant._tag === "Rejected") return classifyClaimLoss(row, nowMs);
|
|
681
|
+
const activation = yield* consensus.activate(runId, owner, grant.grantedAtMs, nowMs);
|
|
682
|
+
if (activation._tag === "Lost") {
|
|
683
|
+
yield* consensus.release(runId, owner);
|
|
684
|
+
return classifyClaimLoss(row, nowMs);
|
|
685
|
+
}
|
|
686
|
+
yield* recordTransition(runId, transition, owner, grant.grantedAtMs);
|
|
687
|
+
yield* recordTransition(runId, "activated", owner, grant.grantedAtMs);
|
|
688
|
+
yield* sql`
|
|
689
|
+
UPDATE flows_runs
|
|
690
|
+
SET
|
|
691
|
+
status = 'running',
|
|
692
|
+
started_at_ms = COALESCE(started_at_ms, ${nowMs}),
|
|
693
|
+
finished_at_ms = NULL,
|
|
694
|
+
owner_host_id = ${owner.hostId},
|
|
695
|
+
owner_pid = ${owner.pid},
|
|
696
|
+
owner_nonce = ${owner.nonce},
|
|
697
|
+
heartbeat_at_ms = ${nowMs}
|
|
698
|
+
WHERE run_id = ${runId}
|
|
699
|
+
`;
|
|
700
|
+
return activated;
|
|
701
|
+
})
|
|
702
|
+
);
|
|
703
|
+
});
|
|
704
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)(
|
|
705
|
+
(outcome) => RunStoreMetrics.claimAndOwn[outcome._tag],
|
|
706
|
+
import_effect.Metric.withAttributes(RunStoreMetrics.claims, { op: "claim_and_own" })
|
|
707
|
+
))
|
|
708
|
+
);
|
|
709
|
+
const activate = import_effect.Effect.fn("RunStore.activate")(
|
|
710
|
+
(runIdInput, claimantInput, claimedAtMsInput, expectedInput) => import_effect.Effect.gen(function* () {
|
|
711
|
+
const runId = yield* snapshotRunId("activate", runIdInput);
|
|
712
|
+
const claimant = yield* snapshotOwner("activate", "claimant", claimantInput);
|
|
713
|
+
const claimedAtMs = yield* snapshotTimestamp(
|
|
714
|
+
"activate",
|
|
715
|
+
"claimedAtMs",
|
|
716
|
+
claimedAtMsInput
|
|
717
|
+
);
|
|
718
|
+
const expected = yield* snapshotExpected("activate", expectedInput);
|
|
719
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, claimantHostId: claimant.hostId });
|
|
720
|
+
const activatedAtMs = yield* import_effect.Clock.currentTimeMillis.pipe(import_effect.Effect.map(Math.floor));
|
|
721
|
+
return yield* write(
|
|
722
|
+
"activate",
|
|
723
|
+
import_effect.Effect.gen(function* () {
|
|
724
|
+
const row = (yield* selectRun(sql, runId))[0];
|
|
725
|
+
if (row === void 0 || !rowMatchesClaim(row, claimant, claimedAtMs)) return claimLost;
|
|
726
|
+
const clearClaim = sql`
|
|
727
|
+
UPDATE flows_runs
|
|
728
|
+
SET
|
|
729
|
+
claim_host_id = NULL,
|
|
730
|
+
claim_pid = NULL,
|
|
731
|
+
claim_nonce = NULL,
|
|
732
|
+
claimed_at_ms = NULL
|
|
733
|
+
WHERE run_id = ${runId}
|
|
734
|
+
AND claim_host_id = ${claimant.hostId}
|
|
735
|
+
AND claim_pid = ${claimant.pid}
|
|
736
|
+
AND claim_nonce = ${claimant.nonce}
|
|
737
|
+
AND claimed_at_ms = ${claimedAtMs}
|
|
738
|
+
`;
|
|
739
|
+
if (!rowMatchesSnapshot(row, expected)) {
|
|
740
|
+
yield* consensus.release(runId, claimant);
|
|
741
|
+
yield* recordTransition(runId, "released", claimant, claimedAtMs);
|
|
742
|
+
yield* clearClaim;
|
|
743
|
+
return snapshotChanged;
|
|
744
|
+
}
|
|
745
|
+
const activation = yield* consensus.activate(runId, claimant, claimedAtMs, activatedAtMs);
|
|
746
|
+
if (activation._tag === "Lost") {
|
|
747
|
+
yield* clearClaim;
|
|
748
|
+
return claimLost;
|
|
749
|
+
}
|
|
750
|
+
yield* sql`
|
|
751
|
+
UPDATE flows_runs
|
|
752
|
+
SET
|
|
753
|
+
status = 'running',
|
|
754
|
+
started_at_ms = COALESCE(started_at_ms, ${activatedAtMs}),
|
|
755
|
+
finished_at_ms = NULL,
|
|
756
|
+
owner_host_id = ${claimant.hostId},
|
|
757
|
+
owner_pid = ${claimant.pid},
|
|
758
|
+
owner_nonce = ${claimant.nonce},
|
|
759
|
+
heartbeat_at_ms = ${activatedAtMs},
|
|
760
|
+
claim_host_id = NULL,
|
|
761
|
+
claim_pid = NULL,
|
|
762
|
+
claim_nonce = NULL,
|
|
763
|
+
claimed_at_ms = NULL
|
|
764
|
+
WHERE run_id = ${runId}
|
|
765
|
+
`;
|
|
766
|
+
yield* recordTransition(runId, "activated", claimant, claimedAtMs);
|
|
767
|
+
return activated;
|
|
768
|
+
})
|
|
769
|
+
);
|
|
770
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)(
|
|
771
|
+
(outcome) => RunStoreMetrics.activate[outcome._tag],
|
|
772
|
+
import_effect.Metric.withAttributes(RunStoreMetrics.claims, { op: "activate" })
|
|
773
|
+
))
|
|
774
|
+
);
|
|
775
|
+
const abandonClaim = import_effect.Effect.fn("RunStore.abandonClaim")(
|
|
776
|
+
(runIdInput, claimantInput, claimedAtMsInput) => import_effect.Effect.gen(function* () {
|
|
777
|
+
const runId = yield* snapshotRunId("abandonClaim", runIdInput);
|
|
778
|
+
const claimant = yield* snapshotOwner("abandonClaim", "claimant", claimantInput);
|
|
779
|
+
const claimedAtMs = yield* snapshotTimestamp("abandonClaim", "claimedAtMs", claimedAtMsInput);
|
|
780
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, claimantHostId: claimant.hostId });
|
|
781
|
+
return yield* write(
|
|
782
|
+
"abandonClaim",
|
|
783
|
+
import_effect.Effect.gen(function* () {
|
|
784
|
+
const rows = yield* sql`
|
|
785
|
+
UPDATE flows_runs
|
|
786
|
+
SET
|
|
787
|
+
claim_host_id = NULL,
|
|
788
|
+
claim_pid = NULL,
|
|
789
|
+
claim_nonce = NULL,
|
|
790
|
+
claimed_at_ms = NULL
|
|
791
|
+
WHERE run_id = ${runId}
|
|
792
|
+
AND claim_host_id = ${claimant.hostId}
|
|
793
|
+
AND claim_pid = ${claimant.pid}
|
|
794
|
+
AND claim_nonce = ${claimant.nonce}
|
|
795
|
+
AND claimed_at_ms = ${claimedAtMs}
|
|
796
|
+
RETURNING run_id AS "runId"
|
|
797
|
+
`;
|
|
798
|
+
if (rows.length === 0) return claimLost;
|
|
799
|
+
yield* consensus.release(runId, claimant);
|
|
800
|
+
yield* recordTransition(runId, "released", claimant, claimedAtMs);
|
|
801
|
+
return abandoned;
|
|
802
|
+
})
|
|
803
|
+
);
|
|
804
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)(
|
|
805
|
+
(outcome) => RunStoreMetrics.abandonClaim[outcome._tag],
|
|
806
|
+
import_effect.Metric.withAttributes(RunStoreMetrics.claims, { op: "abandon_claim" })
|
|
807
|
+
))
|
|
808
|
+
);
|
|
809
|
+
const recoverClaim = import_effect.Effect.fn("RunStore.recoverClaim")(
|
|
810
|
+
(runIdInput, staleClaimantInput, claimedAtMsInput, observerInput, nowMsInput, evidenceInput) => import_effect.Effect.gen(function* () {
|
|
811
|
+
const runId = yield* snapshotRunId("recoverClaim", runIdInput);
|
|
812
|
+
const staleClaimant = yield* snapshotOwner("recoverClaim", "staleClaimant", staleClaimantInput);
|
|
813
|
+
const claimedAtMs = yield* snapshotTimestamp(
|
|
814
|
+
"recoverClaim",
|
|
815
|
+
"claimedAtMs",
|
|
816
|
+
claimedAtMsInput
|
|
817
|
+
);
|
|
818
|
+
const observer = yield* snapshotOwner("recoverClaim", "observer", observerInput);
|
|
819
|
+
const nowMs = yield* snapshotLeaseReading("recoverClaim", "nowMs", nowMsInput, { runId, claimedAtMs });
|
|
820
|
+
const evidence = yield* snapshotEvidence("recoverClaim", evidenceInput);
|
|
821
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, observerHostId: observer.hostId });
|
|
822
|
+
return yield* import_effect.Effect.suspend(() => {
|
|
823
|
+
if (!(0, import_Consensus.matchesEvidence)(staleClaimant, observer, nowMs, evidence)) {
|
|
824
|
+
return import_effect.Effect.succeed(livenessUnconfirmed);
|
|
825
|
+
}
|
|
826
|
+
return write(
|
|
827
|
+
"recoverClaim",
|
|
828
|
+
import_effect.Effect.gen(function* () {
|
|
829
|
+
const row = (yield* selectRun(sql, runId))[0];
|
|
830
|
+
if (row === void 0) return notFound;
|
|
831
|
+
if (!rowMatchesClaim(row, staleClaimant, claimedAtMs)) return claimChanged;
|
|
832
|
+
if (claimedAtMs >= nowMs - heartbeatStaleAfterMs) return claimFresh;
|
|
833
|
+
const outcome = yield* consensus.recover(runId, staleClaimant, claimedAtMs, observer, nowMs, evidence);
|
|
834
|
+
if (outcome._tag === "Rejected") {
|
|
835
|
+
return claimChanged;
|
|
836
|
+
}
|
|
837
|
+
yield* sql`
|
|
838
|
+
UPDATE flows_runs
|
|
839
|
+
SET
|
|
840
|
+
claim_host_id = NULL,
|
|
841
|
+
claim_pid = NULL,
|
|
842
|
+
claim_nonce = NULL,
|
|
843
|
+
claimed_at_ms = NULL
|
|
844
|
+
WHERE run_id = ${runId}
|
|
845
|
+
`;
|
|
846
|
+
yield* recordTransition(runId, "expired", staleClaimant, claimedAtMs);
|
|
847
|
+
return recovered;
|
|
848
|
+
})
|
|
849
|
+
);
|
|
850
|
+
});
|
|
851
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)(
|
|
852
|
+
(outcome) => RunStoreMetrics.recoverClaim[outcome._tag],
|
|
853
|
+
import_effect.Metric.withAttributes(RunStoreMetrics.claims, { op: "recover_claim" })
|
|
854
|
+
))
|
|
855
|
+
);
|
|
856
|
+
const renewLease = (method) => import_effect.Effect.fn(`RunStore.${method}`)(
|
|
857
|
+
(runIdInput, ownerInput, nowMsInput) => import_effect.Effect.gen(function* () {
|
|
858
|
+
const runId = yield* snapshotRunId(method, runIdInput);
|
|
859
|
+
const owner = yield* snapshotOwner(method, "owner", ownerInput);
|
|
860
|
+
const nowMs = yield* snapshotLeaseReading(
|
|
861
|
+
method,
|
|
862
|
+
"nowMs",
|
|
863
|
+
nowMsInput,
|
|
864
|
+
{ runId }
|
|
865
|
+
);
|
|
866
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, ownerHostId: owner.hostId });
|
|
867
|
+
return yield* write(
|
|
868
|
+
method,
|
|
869
|
+
import_effect.Effect.gen(function* () {
|
|
870
|
+
const outcome = yield* consensus[method](runId, owner, nowMs);
|
|
871
|
+
if (outcome._tag === "Lost") {
|
|
872
|
+
const current2 = yield* selectRun(sql, runId);
|
|
873
|
+
return current2.length === 0 ? notFound : fenceLost;
|
|
874
|
+
}
|
|
875
|
+
const rows = yield* sql`
|
|
876
|
+
UPDATE flows_runs
|
|
877
|
+
SET heartbeat_at_ms = ${Dialect.greatest(sql)}(heartbeat_at_ms, ${outcome.heartbeatAtMs})
|
|
878
|
+
WHERE run_id = ${runId}
|
|
879
|
+
AND status = 'running'
|
|
880
|
+
AND owner_host_id = ${owner.hostId}
|
|
881
|
+
AND owner_pid = ${owner.pid}
|
|
882
|
+
AND owner_nonce = ${owner.nonce}
|
|
883
|
+
RETURNING run_id AS "runId"
|
|
884
|
+
`;
|
|
885
|
+
if (rows.length > 0) return updated;
|
|
886
|
+
const current = yield* selectRun(sql, runId);
|
|
887
|
+
return current.length === 0 ? notFound : fenceLost;
|
|
888
|
+
})
|
|
889
|
+
);
|
|
890
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)((outcome) => RunStoreMetrics.heartbeat[outcome._tag], RunStoreMetrics.heartbeats))
|
|
891
|
+
);
|
|
892
|
+
const heartbeat = renewLease("heartbeat");
|
|
893
|
+
const reconfirm = renewLease("reconfirm");
|
|
894
|
+
const transitionOwned = import_effect.Effect.fn("RunStore.transitionOwned")(
|
|
895
|
+
(runIdInput, ownerInput, toStatusInput, stateInput, guardInput) => import_effect.Effect.gen(function* () {
|
|
896
|
+
const runId = yield* snapshotRunId("transitionOwned", runIdInput);
|
|
897
|
+
const owner = yield* snapshotOwner("transitionOwned", "owner", ownerInput);
|
|
898
|
+
if (!import_effect.Schema.is(RunStatus)(toStatusInput) || toStatusInput === "pending") {
|
|
899
|
+
return yield* import_effect.Effect.fail(invalidRunError("transitionOwned", "toStatus"));
|
|
900
|
+
}
|
|
901
|
+
const toStatus = toStatusInput;
|
|
902
|
+
const state = stateInput === void 0 ? null : yield* snapshotState("transitionOwned", stateInput, {
|
|
903
|
+
runId,
|
|
904
|
+
stateJsonLength: typeof stateInput === "string" ? stateInput.length : null,
|
|
905
|
+
stateJsonValid: isJsonString(stateInput)
|
|
906
|
+
});
|
|
907
|
+
const guard = yield* snapshotGuard(guardInput);
|
|
908
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, ownerHostId: owner.hostId, to: toStatus });
|
|
909
|
+
const transitionedAtMs = yield* import_effect.Clock.currentTimeMillis.pipe(import_effect.Effect.map(Math.floor));
|
|
910
|
+
const outcome = yield* write(
|
|
911
|
+
"transitionOwned",
|
|
912
|
+
import_effect.Effect.gen(function* () {
|
|
913
|
+
const held = yield* consensus.guard(runId, owner).pipe(
|
|
914
|
+
import_effect.Effect.as(true),
|
|
915
|
+
import_effect.Effect.catch((cause) => cause.code === "fence_lost" ? import_effect.Effect.succeed(false) : import_effect.Effect.fail(cause))
|
|
916
|
+
);
|
|
917
|
+
const row = (yield* selectRun(sql, runId))[0];
|
|
918
|
+
if (row === void 0) return notFound;
|
|
919
|
+
if (!held || !rowOwnedBy(row, owner)) return fenceLost;
|
|
920
|
+
const guardBlocked = guard?.cancelRequested === "absent" && row.cancelRequestedAtMs !== null || guard?.cancelRequested === "present" && row.cancelRequestedAtMs === null;
|
|
921
|
+
if (guardBlocked) return guardFailed;
|
|
922
|
+
if (toStatus === "running") {
|
|
923
|
+
yield* sql`
|
|
924
|
+
UPDATE flows_runs
|
|
925
|
+
SET
|
|
926
|
+
status = 'running',
|
|
927
|
+
finished_at_ms = NULL,
|
|
928
|
+
state_json = COALESCE(${state}, state_json)
|
|
929
|
+
WHERE run_id = ${runId}
|
|
930
|
+
`;
|
|
931
|
+
return transitioned;
|
|
932
|
+
}
|
|
933
|
+
yield* sql`
|
|
934
|
+
UPDATE flows_runs
|
|
935
|
+
SET
|
|
936
|
+
status = ${toStatus},
|
|
937
|
+
finished_at_ms = ${terminalStatuses.has(toStatus) ? transitionedAtMs : null},
|
|
938
|
+
owner_host_id = NULL,
|
|
939
|
+
owner_pid = NULL,
|
|
940
|
+
owner_nonce = NULL,
|
|
941
|
+
heartbeat_at_ms = NULL,
|
|
942
|
+
claim_host_id = NULL,
|
|
943
|
+
claim_pid = NULL,
|
|
944
|
+
claim_nonce = NULL,
|
|
945
|
+
claimed_at_ms = NULL,
|
|
946
|
+
state_json = COALESCE(${state}, state_json)
|
|
947
|
+
WHERE run_id = ${runId}
|
|
948
|
+
`;
|
|
949
|
+
yield* consensus.release(runId, owner);
|
|
950
|
+
yield* recordTransition(runId, "released", owner, null);
|
|
951
|
+
if (terminalStatuses.has(toStatus)) {
|
|
952
|
+
yield* (0, import_DurableWriter.afterCommit)(import_effect.Metric.update(ObservabilityMetric.runThroughput, 1), sql);
|
|
953
|
+
}
|
|
954
|
+
return transitioned;
|
|
955
|
+
})
|
|
956
|
+
);
|
|
957
|
+
return outcome;
|
|
958
|
+
}).pipe(
|
|
959
|
+
(0, import_SpanOutcome.observeOutcome)((outcome) => import_effect.Metric.withAttributes(RunStoreMetrics.transition[outcome._tag], {
|
|
960
|
+
to: toStatusInput
|
|
961
|
+
}), import_effect.Metric.withAttributes(RunStoreMetrics.transitions, { to: toStatusInput }))
|
|
962
|
+
)
|
|
963
|
+
);
|
|
964
|
+
const steal = import_effect.Effect.fn("RunStore.steal")(
|
|
965
|
+
(runIdInput, expectedInput, claimantInput, nowMsInput, evidenceInput) => import_effect.Effect.gen(function* () {
|
|
966
|
+
const runId = yield* snapshotRunId("steal", runIdInput);
|
|
967
|
+
const expected = yield* snapshotExpected("steal", expectedInput);
|
|
968
|
+
const claimant = yield* snapshotOwner("steal", "claimant", claimantInput);
|
|
969
|
+
const nowMs = yield* snapshotLeaseReading(
|
|
970
|
+
"steal",
|
|
971
|
+
"nowMs",
|
|
972
|
+
nowMsInput,
|
|
973
|
+
{ runId }
|
|
974
|
+
);
|
|
975
|
+
const evidence = yield* snapshotEvidence("steal", evidenceInput);
|
|
976
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, claimantHostId: claimant.hostId });
|
|
977
|
+
return yield* import_effect.Effect.suspend(() => {
|
|
978
|
+
if (!evidenceMatches(expected, claimant, nowMs, evidence)) {
|
|
979
|
+
return import_effect.Effect.succeed(livenessUnconfirmed);
|
|
980
|
+
}
|
|
981
|
+
return write(
|
|
982
|
+
"steal",
|
|
983
|
+
import_effect.Effect.gen(function* () {
|
|
984
|
+
const row = (yield* selectRun(sql, runId))[0];
|
|
985
|
+
const admissible = row !== void 0 && rowMatchesSnapshot(row, expected) && claimColumnsFree(row) && row.heartbeatAtMs !== null && row.heartbeatAtMs < nowMs - heartbeatStaleAfterMs;
|
|
986
|
+
if (!admissible) return classifyClaimLoss(row, nowMs);
|
|
987
|
+
const grant = yield* consensus.steal(runId, claimant, nowMs, evidence);
|
|
988
|
+
if (grant._tag === "Rejected") {
|
|
989
|
+
return classifyClaimLoss(row, nowMs);
|
|
990
|
+
}
|
|
991
|
+
yield* sql`
|
|
992
|
+
UPDATE flows_runs
|
|
993
|
+
SET
|
|
994
|
+
claim_host_id = ${claimant.hostId},
|
|
995
|
+
claim_pid = ${claimant.pid},
|
|
996
|
+
claim_nonce = ${claimant.nonce},
|
|
997
|
+
claimed_at_ms = ${grant.grantedAtMs}
|
|
998
|
+
WHERE run_id = ${runId}
|
|
999
|
+
`;
|
|
1000
|
+
yield* recordTransition(runId, "stolen", claimant, grant.grantedAtMs);
|
|
1001
|
+
return claimed(grant.grantedAtMs);
|
|
1002
|
+
})
|
|
1003
|
+
);
|
|
1004
|
+
});
|
|
1005
|
+
}).pipe((0, import_SpanOutcome.observeOutcome)(
|
|
1006
|
+
(outcome) => RunStoreMetrics.steal[outcome._tag],
|
|
1007
|
+
import_effect.Metric.withAttributes(RunStoreMetrics.claims, { op: "steal" })
|
|
1008
|
+
))
|
|
1009
|
+
);
|
|
1010
|
+
const acknowledgeCancel = import_effect.Effect.fn("RunStore.acknowledgeCancel")(
|
|
1011
|
+
(id, ownerInput, time) => import_effect.Effect.gen(function* () {
|
|
1012
|
+
const runId = yield* snapshotRunId("acknowledgeCancel", id);
|
|
1013
|
+
const owner = yield* snapshotOwner("acknowledgeCancel", "owner", ownerInput);
|
|
1014
|
+
const observedAtMs = yield* snapshotTimestamp("acknowledgeCancel", "nowMs", time);
|
|
1015
|
+
yield* import_effect.Effect.annotateCurrentSpan({ runId, ownerHostId: owner.hostId });
|
|
1016
|
+
const acknowledgement = JSON.stringify({ observedAtMs, owner });
|
|
1017
|
+
return yield* write(
|
|
1018
|
+
"acknowledgeCancel",
|
|
1019
|
+
sql`
|
|
1020
|
+
UPDATE flows_runs
|
|
1021
|
+
SET cancel_acknowledgement_json = COALESCE(cancel_acknowledgement_json, ${acknowledgement})
|
|
1022
|
+
WHERE run_id = ${runId} AND status = 'running' AND cancel_requested_at_ms IS NOT NULL
|
|
1023
|
+
AND owner_host_id = ${owner.hostId} AND owner_pid = ${owner.pid} AND owner_nonce = ${owner.nonce}
|
|
1024
|
+
RETURNING run_id
|
|
1025
|
+
`.pipe(import_effect.Effect.map((rows) => rows.length > 0))
|
|
1026
|
+
);
|
|
1027
|
+
}).pipe(import_SpanOutcome.observeExit)
|
|
1028
|
+
);
|
|
1029
|
+
return RunStore.of({
|
|
1030
|
+
create,
|
|
1031
|
+
get,
|
|
1032
|
+
lineage,
|
|
1033
|
+
latestRound,
|
|
1034
|
+
requestCancel,
|
|
1035
|
+
acknowledgeCancel,
|
|
1036
|
+
requestCancelLineage,
|
|
1037
|
+
claim,
|
|
1038
|
+
claimAndOwn,
|
|
1039
|
+
activate,
|
|
1040
|
+
abandonClaim,
|
|
1041
|
+
recoverClaim,
|
|
1042
|
+
heartbeat,
|
|
1043
|
+
reconfirm,
|
|
1044
|
+
transitionOwned,
|
|
1045
|
+
steal
|
|
1046
|
+
});
|
|
1047
|
+
}
|
|
1048
|
+
);
|
|
1049
|
+
const makeNoop = (overrides = {}) => {
|
|
1050
|
+
const unavailable = (method) => import_effect.Effect.fail(runStoreError(method, "persistence_failed", "no run store in this environment", method));
|
|
1051
|
+
return RunStore.of({
|
|
1052
|
+
create: import_effect.Effect.fn("RunStore.create")(() => unavailable("create")),
|
|
1053
|
+
get: import_effect.Effect.fn("RunStore.get")(() => unavailable("get")),
|
|
1054
|
+
lineage: import_effect.Effect.fn("RunStore.lineage")(() => unavailable("lineage")),
|
|
1055
|
+
latestRound: import_effect.Effect.fn("RunStore.latestRound")(() => unavailable("latestRound")),
|
|
1056
|
+
requestCancel: import_effect.Effect.fn("RunStore.requestCancel")(() => import_effect.Effect.succeed(notFound)),
|
|
1057
|
+
acknowledgeCancel: import_effect.Effect.fn("RunStore.acknowledgeCancel")(() => import_effect.Effect.succeed(false)),
|
|
1058
|
+
requestCancelLineage: import_effect.Effect.fn("RunStore.requestCancelLineage")(() => unavailable("requestCancelLineage")),
|
|
1059
|
+
claim: import_effect.Effect.fn("RunStore.claim")(() => import_effect.Effect.succeed(notFound)),
|
|
1060
|
+
claimAndOwn: import_effect.Effect.fn("RunStore.claimAndOwn")(() => import_effect.Effect.succeed(notFound)),
|
|
1061
|
+
activate: import_effect.Effect.fn("RunStore.activate")(() => import_effect.Effect.succeed(claimLost)),
|
|
1062
|
+
abandonClaim: import_effect.Effect.fn("RunStore.abandonClaim")(() => import_effect.Effect.succeed(claimLost)),
|
|
1063
|
+
recoverClaim: import_effect.Effect.fn("RunStore.recoverClaim")(() => import_effect.Effect.succeed(notFound)),
|
|
1064
|
+
heartbeat: import_effect.Effect.fn("RunStore.heartbeat")(() => import_effect.Effect.succeed(notFound)),
|
|
1065
|
+
reconfirm: import_effect.Effect.fn("RunStore.reconfirm")(() => import_effect.Effect.succeed(notFound)),
|
|
1066
|
+
transitionOwned: import_effect.Effect.fn("RunStore.transitionOwned")(() => import_effect.Effect.succeed(notFound)),
|
|
1067
|
+
steal: import_effect.Effect.fn("RunStore.steal")(() => import_effect.Effect.succeed(notFound)),
|
|
1068
|
+
...overrides
|
|
1069
|
+
});
|
|
1070
|
+
};
|
|
1071
|
+
const layerNoop = (overrides = {}) => import_effect.Layer.succeed(RunStore)(makeNoop(overrides));
|
|
1072
|
+
const layerWith = import_effect.Layer.effect(
|
|
1073
|
+
RunStore,
|
|
1074
|
+
make
|
|
1075
|
+
);
|
|
1076
|
+
const layer = layerWith.pipe(
|
|
1077
|
+
import_effect.Layer.provide(SqlConsensus.layer)
|
|
1078
|
+
);
|
|
1079
|
+
//# sourceMappingURL=RunStore.js.map
|