@smthrs/run-store 0.0.0-stage → 1.0.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -0
- package/LICENSE +21 -0
- package/README.md +195 -2
- package/dist/cjs/AttemptStore.d.ts +359 -0
- package/dist/cjs/AttemptStore.d.ts.map +1 -0
- package/dist/cjs/AttemptStore.js +731 -0
- package/dist/cjs/AttemptStore.js.map +7 -0
- package/dist/cjs/Heartbeat.d.ts +46 -0
- package/dist/cjs/Heartbeat.d.ts.map +1 -0
- package/dist/cjs/Heartbeat.js +28 -0
- package/dist/cjs/Heartbeat.js.map +7 -0
- package/dist/cjs/Migrations.d.ts +42 -0
- package/dist/cjs/Migrations.d.ts.map +1 -0
- package/dist/cjs/Migrations.js +55 -0
- package/dist/cjs/Migrations.js.map +7 -0
- package/dist/cjs/Ownership.d.ts +275 -0
- package/dist/cjs/Ownership.d.ts.map +1 -0
- package/dist/cjs/Ownership.js +129 -0
- package/dist/cjs/Ownership.js.map +7 -0
- package/dist/cjs/RunStore.d.ts +571 -0
- package/dist/cjs/RunStore.d.ts.map +1 -0
- package/dist/cjs/RunStore.js +1079 -0
- package/dist/cjs/RunStore.js.map +7 -0
- package/dist/cjs/RunStoreMetrics.d.ts +151 -0
- package/dist/cjs/RunStoreMetrics.d.ts.map +1 -0
- package/dist/cjs/RunStoreMetrics.js +91 -0
- package/dist/cjs/RunStoreMetrics.js.map +7 -0
- package/dist/cjs/index.d.ts +47 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +43 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/Boundary.d.ts +80 -0
- package/dist/cjs/internal/Boundary.d.ts.map +1 -0
- package/dist/cjs/internal/Boundary.js +92 -0
- package/dist/cjs/internal/Boundary.js.map +7 -0
- package/dist/cjs/internal/Companion.d.ts +44 -0
- package/dist/cjs/internal/Companion.d.ts.map +1 -0
- package/dist/cjs/internal/Companion.js +50 -0
- package/dist/cjs/internal/Companion.js.map +7 -0
- package/dist/cjs/internal/SpanOutcome.d.ts +37 -0
- package/dist/cjs/internal/SpanOutcome.d.ts.map +1 -0
- package/dist/cjs/internal/SpanOutcome.js +62 -0
- package/dist/cjs/internal/SpanOutcome.js.map +7 -0
- package/dist/cjs/migrations/0001_initial.d.ts +15 -0
- package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
- package/dist/cjs/migrations/0001_initial.js +110 -0
- package/dist/cjs/migrations/0001_initial.js.map +7 -0
- package/dist/cjs/migrations/0002_lineage.d.ts +27 -0
- package/dist/cjs/migrations/0002_lineage.d.ts.map +1 -0
- package/dist/cjs/migrations/0002_lineage.js +43 -0
- package/dist/cjs/migrations/0002_lineage.js.map +7 -0
- package/dist/cjs/migrations/0003_execution_revisions.d.ts +15 -0
- package/dist/cjs/migrations/0003_execution_revisions.d.ts.map +1 -0
- package/dist/cjs/migrations/0003_execution_revisions.js +77 -0
- package/dist/cjs/migrations/0003_execution_revisions.js.map +7 -0
- package/dist/cjs/migrations/0004_waiting_request.d.ts +32 -0
- package/dist/cjs/migrations/0004_waiting_request.d.ts.map +1 -0
- package/dist/cjs/migrations/0004_waiting_request.js +42 -0
- package/dist/cjs/migrations/0004_waiting_request.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/test/TestRunStore.d.ts +17 -0
- package/dist/cjs/test/TestRunStore.d.ts.map +1 -0
- package/dist/cjs/test/TestRunStore.js +42 -0
- package/dist/cjs/test/TestRunStore.js.map +7 -0
- package/dist/esm/AttemptStore.d.ts +359 -0
- package/dist/esm/AttemptStore.d.ts.map +1 -0
- package/dist/esm/AttemptStore.js +835 -0
- package/dist/esm/AttemptStore.js.map +1 -0
- package/dist/esm/Heartbeat.d.ts +46 -0
- package/dist/esm/Heartbeat.d.ts.map +1 -0
- package/dist/esm/Heartbeat.js +46 -0
- package/dist/esm/Heartbeat.js.map +1 -0
- package/dist/esm/Migrations.d.ts +42 -0
- package/dist/esm/Migrations.d.ts.map +1 -0
- package/dist/esm/Migrations.js +56 -0
- package/dist/esm/Migrations.js.map +1 -0
- package/dist/esm/Ownership.d.ts +275 -0
- package/dist/esm/Ownership.d.ts.map +1 -0
- package/dist/esm/Ownership.js +299 -0
- package/dist/esm/Ownership.js.map +1 -0
- package/dist/esm/RunStore.d.ts +571 -0
- package/dist/esm/RunStore.d.ts.map +1 -0
- package/dist/esm/RunStore.js +1287 -0
- package/dist/esm/RunStore.js.map +1 -0
- package/dist/esm/RunStoreMetrics.d.ts +151 -0
- package/dist/esm/RunStoreMetrics.d.ts.map +1 -0
- package/dist/esm/RunStoreMetrics.js +120 -0
- package/dist/esm/RunStoreMetrics.js.map +1 -0
- package/dist/esm/index.d.ts +47 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +47 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/Boundary.d.ts +80 -0
- package/dist/esm/internal/Boundary.d.ts.map +1 -0
- package/dist/esm/internal/Boundary.js +109 -0
- package/dist/esm/internal/Boundary.js.map +1 -0
- package/dist/esm/internal/Companion.d.ts +44 -0
- package/dist/esm/internal/Companion.d.ts.map +1 -0
- package/dist/esm/internal/Companion.js +53 -0
- package/dist/esm/internal/Companion.js.map +1 -0
- package/dist/esm/internal/SpanOutcome.d.ts +37 -0
- package/dist/esm/internal/SpanOutcome.d.ts.map +1 -0
- package/dist/esm/internal/SpanOutcome.js +48 -0
- package/dist/esm/internal/SpanOutcome.js.map +1 -0
- package/dist/esm/migrations/0001_initial.d.ts +15 -0
- package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
- package/dist/esm/migrations/0001_initial.js +88 -0
- package/dist/esm/migrations/0001_initial.js.map +1 -0
- package/dist/esm/migrations/0002_lineage.d.ts +27 -0
- package/dist/esm/migrations/0002_lineage.d.ts.map +1 -0
- package/dist/esm/migrations/0002_lineage.js +33 -0
- package/dist/esm/migrations/0002_lineage.js.map +1 -0
- package/dist/esm/migrations/0003_execution_revisions.d.ts +15 -0
- package/dist/esm/migrations/0003_execution_revisions.d.ts.map +1 -0
- package/dist/esm/migrations/0003_execution_revisions.js +58 -0
- package/dist/esm/migrations/0003_execution_revisions.js.map +1 -0
- package/dist/esm/migrations/0004_waiting_request.d.ts +32 -0
- package/dist/esm/migrations/0004_waiting_request.d.ts.map +1 -0
- package/dist/esm/migrations/0004_waiting_request.js +37 -0
- package/dist/esm/migrations/0004_waiting_request.js.map +1 -0
- package/dist/esm/test/TestRunStore.d.ts +17 -0
- package/dist/esm/test/TestRunStore.d.ts.map +1 -0
- package/dist/esm/test/TestRunStore.js +19 -0
- package/dist/esm/test/TestRunStore.js.map +1 -0
- package/docs/README.md +142 -0
- package/docs/api.md +901 -0
- package/docs/concepts/durable-values.md +140 -0
- package/docs/concepts/execution-revisions.md +53 -0
- package/docs/concepts/fencing.md +146 -0
- package/docs/concepts/leases.md +154 -0
- package/docs/concepts/liveness-evidence.md +142 -0
- package/docs/guides/cancel-a-run.md +156 -0
- package/docs/guides/claim-and-finish-a-run.md +175 -0
- package/docs/guides/compose-the-stores.md +142 -0
- package/docs/guides/observe-outcomes.md +182 -0
- package/docs/guides/record-step-attempts.md +170 -0
- package/docs/guides/recover-a-stalled-run.md +173 -0
- package/docs/guides/testing.md +141 -0
- package/docs/installation.md +109 -0
- package/docs/quickstart.md +201 -0
- package/docs/troubleshooting.md +255 -0
- package/package.json +183 -3
- package/src/AttemptStore.ts +1195 -0
- package/src/Heartbeat.ts +47 -0
- package/src/Migrations.ts +59 -0
- package/src/Ownership.ts +407 -0
- package/src/RunStore.ts +2063 -0
- package/src/RunStoreMetrics.ts +165 -0
- package/src/index.ts +51 -0
- package/src/internal/Boundary.ts +130 -0
- package/src/internal/Companion.ts +64 -0
- package/src/internal/SpanOutcome.ts +71 -0
- package/src/migrations/0001_initial.ts +114 -0
- package/src/migrations/0002_lineage.ts +38 -0
- package/src/migrations/0003_execution_revisions.ts +69 -0
- package/src/migrations/0004_waiting_request.ts +39 -0
- package/src/test/TestRunStore.ts +22 -0
|
@@ -0,0 +1,1195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable storage for individual step attempts.
|
|
3
|
+
*
|
|
4
|
+
* Attempt metadata is deliberately opaque to this module. Its shape belongs to
|
|
5
|
+
* the step executor, and is persisted unchanged across attempt state changes.
|
|
6
|
+
*
|
|
7
|
+
* The running-state and owner fences mean stale attempts and repeated terminal
|
|
8
|
+
* transitions never overwrite the winning row.
|
|
9
|
+
*
|
|
10
|
+
* @since 0.1.0
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import * as Dialect from "@smthrs/database/Dialect"
|
|
14
|
+
import { DatabaseError, DurableWriter, fromSqlError } from "@smthrs/database/DurableWriter"
|
|
15
|
+
import type { OwnerId } from "@smthrs/journal/OwnerId"
|
|
16
|
+
import * as Context from "effect/Context"
|
|
17
|
+
import * as Effect from "effect/Effect"
|
|
18
|
+
import * as Layer from "effect/Layer"
|
|
19
|
+
import * as Option from "effect/Option"
|
|
20
|
+
import * as Schema from "effect/Schema"
|
|
21
|
+
import * as SqlClient from "effect/unstable/sql/SqlClient"
|
|
22
|
+
import * as SqlError from "effect/unstable/sql/SqlError"
|
|
23
|
+
import * as Boundary from "./internal/Boundary.ts"
|
|
24
|
+
import { observeExit, observeOutcome } from "./internal/SpanOutcome.ts"
|
|
25
|
+
|
|
26
|
+
const NonNegativeSafeInt = Schema.Int.check(
|
|
27
|
+
Schema.isGreaterThanOrEqualTo(0),
|
|
28
|
+
Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER)
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
const durableIdentifier = Schema.NonEmptyString.check(
|
|
32
|
+
Schema.isMaxLength(Boundary.maximumIdentifierLength),
|
|
33
|
+
Schema.makeFilter((value: string) => Boundary.isDurableText(value), { title: "durableIdentifier" })
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Absolute ceiling for {@link Options.maxCheckpointBytes}.
|
|
38
|
+
*
|
|
39
|
+
* This bounds only the configured checkpoint policy; metadata, errors, and
|
|
40
|
+
* outcomes have no byte ceiling.
|
|
41
|
+
*
|
|
42
|
+
* @category constants
|
|
43
|
+
* @since 1.0.0-rc.0
|
|
44
|
+
*/
|
|
45
|
+
export const maximumCheckpointBytes = 16 * 1024 * 1024
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Maximum nesting admitted for durable attempt JSON.
|
|
49
|
+
*
|
|
50
|
+
* @category constants
|
|
51
|
+
* @since 1.0.0-rc.0
|
|
52
|
+
*/
|
|
53
|
+
export const maximumJsonDepth = 128
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Maximum values and members admitted for durable attempt JSON.
|
|
57
|
+
*
|
|
58
|
+
* @category constants
|
|
59
|
+
* @since 1.0.0-rc.0
|
|
60
|
+
*/
|
|
61
|
+
export const maximumJsonNodes = 100_000
|
|
62
|
+
|
|
63
|
+
const jsonLimits = (maxBytes?: number | undefined): Boundary.JsonLimits => ({
|
|
64
|
+
...(maxBytes === undefined ? {} : { maxBytes, maxStringBytes: maxBytes }),
|
|
65
|
+
maxDepth: maximumJsonDepth,
|
|
66
|
+
maxMembers: maximumJsonNodes,
|
|
67
|
+
maxNodes: maximumJsonNodes
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Strict JSON value accepted as executable attempt state.
|
|
72
|
+
*
|
|
73
|
+
* The store still takes an inert snapshot at effect start; this schema is the
|
|
74
|
+
* public declaration contract shared by `Attempt`, `FinishAttempt`, and
|
|
75
|
+
* `AttemptPatch`.
|
|
76
|
+
*
|
|
77
|
+
* @category schemas
|
|
78
|
+
* @since 1.0.0-rc.0
|
|
79
|
+
*/
|
|
80
|
+
export const JsonValue = Schema.declare<Boundary.Json>(
|
|
81
|
+
(value): value is Boundary.Json => Boundary.admitJson(value, jsonLimits()).ok,
|
|
82
|
+
{ identifier: "@smthrs/run-store/JsonValue" }
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Strict JSON value accepted as executable attempt state.
|
|
87
|
+
*
|
|
88
|
+
* @category models
|
|
89
|
+
* @since 1.0.0-rc.0
|
|
90
|
+
*/
|
|
91
|
+
export type JsonValue = typeof JsonValue.Type
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Stable error codes returned by attempt persistence operations.
|
|
95
|
+
*
|
|
96
|
+
* @category models
|
|
97
|
+
* @since 0.1.0
|
|
98
|
+
*/
|
|
99
|
+
export const AttemptStoreErrorCode = Schema.Literals([
|
|
100
|
+
"invalid_attempt",
|
|
101
|
+
"constraint",
|
|
102
|
+
"decode_failed",
|
|
103
|
+
"persistence_failed",
|
|
104
|
+
"unknown"
|
|
105
|
+
])
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Stable error codes returned by attempt persistence operations.
|
|
109
|
+
*
|
|
110
|
+
* @category models
|
|
111
|
+
* @since 0.1.0
|
|
112
|
+
*/
|
|
113
|
+
export type AttemptStoreErrorCode = typeof AttemptStoreErrorCode.Type
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Error raised by attempt persistence operations.
|
|
117
|
+
*
|
|
118
|
+
* The identity string equals the defining module path, which is how every
|
|
119
|
+
* tagged identity in this package is named.
|
|
120
|
+
*
|
|
121
|
+
* @category errors
|
|
122
|
+
* @since 0.1.0
|
|
123
|
+
*/
|
|
124
|
+
export class AttemptStoreError extends Schema.TaggedError<AttemptStoreError>()(
|
|
125
|
+
"@smthrs/run-store/AttemptStoreError",
|
|
126
|
+
{
|
|
127
|
+
code: AttemptStoreErrorCode,
|
|
128
|
+
method: Schema.String,
|
|
129
|
+
message: Schema.String,
|
|
130
|
+
cause: Schema.optional(Schema.Unknown)
|
|
131
|
+
}
|
|
132
|
+
) {}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Identifies one execution of a content-addressed step within a run.
|
|
136
|
+
*
|
|
137
|
+
* @category models
|
|
138
|
+
* @since 0.1.0
|
|
139
|
+
*/
|
|
140
|
+
export const AttemptId = Schema.Struct({
|
|
141
|
+
runId: durableIdentifier,
|
|
142
|
+
stepKeyDigest: durableIdentifier,
|
|
143
|
+
attempt: NonNegativeSafeInt
|
|
144
|
+
})
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Identifies one execution of a content-addressed step within a run.
|
|
148
|
+
*
|
|
149
|
+
* @category models
|
|
150
|
+
* @since 0.1.0
|
|
151
|
+
*/
|
|
152
|
+
export type AttemptId = typeof AttemptId.Type
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* A durable attempt row.
|
|
156
|
+
*
|
|
157
|
+
* @category models
|
|
158
|
+
* @since 0.1.0
|
|
159
|
+
*/
|
|
160
|
+
export const Attempt = Schema.Struct({
|
|
161
|
+
...AttemptId.fields,
|
|
162
|
+
state: durableIdentifier,
|
|
163
|
+
startedAtMs: NonNegativeSafeInt,
|
|
164
|
+
finishedAtMs: Schema.optionalKey(NonNegativeSafeInt),
|
|
165
|
+
heartbeatAtMs: Schema.optionalKey(NonNegativeSafeInt),
|
|
166
|
+
checkpoint: Schema.optionalKey(JsonValue),
|
|
167
|
+
error: Schema.optionalKey(JsonValue),
|
|
168
|
+
outcome: Schema.optionalKey(JsonValue),
|
|
169
|
+
meta: JsonValue
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* A durable attempt row.
|
|
174
|
+
*
|
|
175
|
+
* @category models
|
|
176
|
+
* @since 0.1.0
|
|
177
|
+
*/
|
|
178
|
+
export type Attempt = typeof Attempt.Type
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Input used to finish an existing attempt. `error`, `outcome`, and `meta`
|
|
182
|
+
* follow the same rule as {@link AttemptPatch}: an omitted field is left as
|
|
183
|
+
* recorded, so a terminal transition never erases a value written mid-flight
|
|
184
|
+
* by `put` or `patch`. Supplying one replaces it atomically with the terminal
|
|
185
|
+
* state, which lets an executor durably record what it discovered while
|
|
186
|
+
* handling a failure.
|
|
187
|
+
*
|
|
188
|
+
* @category models
|
|
189
|
+
* @since 0.1.0
|
|
190
|
+
*/
|
|
191
|
+
export const FinishAttempt = Schema.Struct({
|
|
192
|
+
...AttemptId.fields,
|
|
193
|
+
state: durableIdentifier,
|
|
194
|
+
finishedAtMs: NonNegativeSafeInt,
|
|
195
|
+
error: Schema.optionalKey(JsonValue),
|
|
196
|
+
outcome: Schema.optionalKey(JsonValue),
|
|
197
|
+
meta: Schema.optionalKey(JsonValue)
|
|
198
|
+
})
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Input used to finish an existing attempt.
|
|
202
|
+
*
|
|
203
|
+
* @category models
|
|
204
|
+
* @since 0.1.0
|
|
205
|
+
*/
|
|
206
|
+
export type FinishAttempt = typeof FinishAttempt.Type
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Result of starting an owner-fenced attempt.
|
|
210
|
+
*
|
|
211
|
+
* @category models
|
|
212
|
+
* @since 0.1.0
|
|
213
|
+
*/
|
|
214
|
+
export type PutResult =
|
|
215
|
+
| { readonly _tag: "Inserted" }
|
|
216
|
+
| { readonly _tag: "Upserted" }
|
|
217
|
+
| { readonly _tag: "ExistingSame" }
|
|
218
|
+
| { readonly _tag: "Conflict" }
|
|
219
|
+
| { readonly _tag: "FenceLost" }
|
|
220
|
+
| { readonly _tag: "RunNotFound" }
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Fields a patch may rewrite.
|
|
224
|
+
*
|
|
225
|
+
* A patch never touches `state`, `started_at_ms`, or `finished_at_ms`: those
|
|
226
|
+
* are the lifecycle, and only `put`/`heartbeat`/`finish` move them. An
|
|
227
|
+
* omitted field is left as recorded.
|
|
228
|
+
*
|
|
229
|
+
* @category models
|
|
230
|
+
* @since 0.1.0
|
|
231
|
+
*/
|
|
232
|
+
export const AttemptPatch = Schema.Struct({
|
|
233
|
+
checkpoint: Schema.optionalKey(JsonValue),
|
|
234
|
+
error: Schema.optionalKey(JsonValue),
|
|
235
|
+
outcome: Schema.optionalKey(JsonValue),
|
|
236
|
+
meta: Schema.optionalKey(JsonValue)
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Fields a patch may rewrite.
|
|
241
|
+
*
|
|
242
|
+
* @category models
|
|
243
|
+
* @since 0.1.0
|
|
244
|
+
*/
|
|
245
|
+
export type AttemptPatch = typeof AttemptPatch.Type
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Result of an owner-fenced attempt patch.
|
|
249
|
+
*
|
|
250
|
+
* @category models
|
|
251
|
+
* @since 0.1.0
|
|
252
|
+
*/
|
|
253
|
+
export type PatchResult =
|
|
254
|
+
| { readonly _tag: "Patched" }
|
|
255
|
+
| { readonly _tag: "NotFound" }
|
|
256
|
+
| { readonly _tag: "FenceLost" }
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Store-wide policy.
|
|
260
|
+
*
|
|
261
|
+
* Defaults treat only `running` attempts as in progress, cap checkpoints at
|
|
262
|
+
* 1 MiB, and make attempt insertion first-writer-wins.
|
|
263
|
+
*
|
|
264
|
+
* @category models
|
|
265
|
+
* @since 0.1.0
|
|
266
|
+
*/
|
|
267
|
+
export interface Options {
|
|
268
|
+
/**
|
|
269
|
+
* States the store treats as "attempt still in progress". `heartbeat` and
|
|
270
|
+
* `finish` fence on membership, and `finish` refuses them as targets.
|
|
271
|
+
* Defaults to `["running"]`.
|
|
272
|
+
*/
|
|
273
|
+
readonly inProgressStates?: ReadonlyArray<string> | undefined
|
|
274
|
+
/**
|
|
275
|
+
* Largest encoded checkpoint accepted, in bytes. Defaults to 1 MiB and may
|
|
276
|
+
* not exceed {@link maximumCheckpointBytes}.
|
|
277
|
+
*/
|
|
278
|
+
readonly maxCheckpointBytes?: number | undefined
|
|
279
|
+
/**
|
|
280
|
+
* `"insert"` (the default) is first-writer-wins: a re-put with different
|
|
281
|
+
* content reports `Conflict`. `"upsert"` overwrites it and reports
|
|
282
|
+
* `Upserted` only while the existing row is still in progress. A terminal
|
|
283
|
+
* row remains immutable. Both modes keep the run-ownership fence.
|
|
284
|
+
*/
|
|
285
|
+
readonly putMode?: "insert" | "upsert" | undefined
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Result of a fenced attempt heartbeat.
|
|
290
|
+
*
|
|
291
|
+
* @category models
|
|
292
|
+
* @since 0.1.0
|
|
293
|
+
*/
|
|
294
|
+
export type HeartbeatResult =
|
|
295
|
+
| { readonly _tag: "Updated" }
|
|
296
|
+
| { readonly _tag: "FenceLost" }
|
|
297
|
+
| { readonly _tag: "NotFound" }
|
|
298
|
+
| { readonly _tag: "StateChanged" }
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Result of a fenced terminal attempt transition.
|
|
302
|
+
*
|
|
303
|
+
* @category models
|
|
304
|
+
* @since 0.1.0
|
|
305
|
+
*/
|
|
306
|
+
export type FinishResult =
|
|
307
|
+
| { readonly _tag: "Finished" }
|
|
308
|
+
| { readonly _tag: "FenceLost" }
|
|
309
|
+
| { readonly _tag: "NotFound" }
|
|
310
|
+
| { readonly _tag: "StateChanged" }
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* Attempt persistence operations.
|
|
314
|
+
*
|
|
315
|
+
* @category models
|
|
316
|
+
* @since 0.1.0
|
|
317
|
+
*/
|
|
318
|
+
export interface Service {
|
|
319
|
+
readonly put: (attempt: Attempt, owner: OwnerId) => Effect.Effect<PutResult, AttemptStoreError>
|
|
320
|
+
readonly get: (id: AttemptId) => Effect.Effect<Option.Option<Attempt>, AttemptStoreError>
|
|
321
|
+
readonly heartbeat: (
|
|
322
|
+
runId: string,
|
|
323
|
+
stepKeyDigest: string,
|
|
324
|
+
attempt: number,
|
|
325
|
+
owner: OwnerId,
|
|
326
|
+
nowMs: number,
|
|
327
|
+
checkpoint?: JsonValue
|
|
328
|
+
) => Effect.Effect<HeartbeatResult, AttemptStoreError>
|
|
329
|
+
readonly finish: (attempt: FinishAttempt, owner: OwnerId) => Effect.Effect<FinishResult, AttemptStoreError>
|
|
330
|
+
/**
|
|
331
|
+
* Rewrites opaque fields without competing for the attempt lifecycle: a
|
|
332
|
+
* patch never moves `state`, so executors record response text, worktree
|
|
333
|
+
* pointers, or cache flags on running *and* terminal rows. It is fenced on
|
|
334
|
+
* run ownership like every other write — `outcome` is replayed verbatim as
|
|
335
|
+
* the attempt's result, so a delayed patch from an owner that lost the run
|
|
336
|
+
* (or from any writer after the run reached a terminal status and its
|
|
337
|
+
* owner columns were cleared) reports `FenceLost` instead of rewriting the
|
|
338
|
+
* winning row. Prior art: Temporal conditions every persistence write on
|
|
339
|
+
* the shard `rangeID` (`reference/temporal/service/history/shard/`); there
|
|
340
|
+
* is no unfenced write surface.
|
|
341
|
+
*/
|
|
342
|
+
readonly patch: (
|
|
343
|
+
id: AttemptId,
|
|
344
|
+
patch: AttemptPatch,
|
|
345
|
+
owner: OwnerId
|
|
346
|
+
) => Effect.Effect<PatchResult, AttemptStoreError>
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Service tag for durable step attempts.
|
|
351
|
+
*
|
|
352
|
+
* The identity string equals the defining module path, which is how every
|
|
353
|
+
* service in this package is named.
|
|
354
|
+
*
|
|
355
|
+
* @category services
|
|
356
|
+
* @since 0.1.0
|
|
357
|
+
*/
|
|
358
|
+
export class AttemptStore extends Context.Service<AttemptStore, Service>()("@smthrs/run-store/AttemptStore") {}
|
|
359
|
+
|
|
360
|
+
const AttemptRow = Schema.Struct({
|
|
361
|
+
run_id: durableIdentifier,
|
|
362
|
+
step_key_digest: durableIdentifier,
|
|
363
|
+
attempt: NonNegativeSafeInt,
|
|
364
|
+
state: durableIdentifier,
|
|
365
|
+
started_at_ms: NonNegativeSafeInt,
|
|
366
|
+
finished_at_ms: Schema.NullOr(NonNegativeSafeInt),
|
|
367
|
+
heartbeat_at_ms: Schema.NullOr(NonNegativeSafeInt),
|
|
368
|
+
checkpoint_json: Schema.NullOr(Schema.String),
|
|
369
|
+
error_json: Schema.NullOr(Schema.String),
|
|
370
|
+
outcome_json: Schema.NullOr(Schema.String),
|
|
371
|
+
meta_json: Schema.String
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
type AttemptRow = typeof AttemptRow.Type
|
|
375
|
+
|
|
376
|
+
interface RunFenceRow {
|
|
377
|
+
readonly status: string
|
|
378
|
+
readonly owner_host_id: string | null
|
|
379
|
+
readonly owner_pid: number | null
|
|
380
|
+
readonly owner_nonce: string | null
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
type AttemptStoreMethod = "makeWith" | keyof Service
|
|
384
|
+
|
|
385
|
+
const attemptStoreError = (
|
|
386
|
+
method: AttemptStoreMethod,
|
|
387
|
+
code: AttemptStoreErrorCode,
|
|
388
|
+
detail: string,
|
|
389
|
+
cause?: unknown
|
|
390
|
+
): AttemptStoreError =>
|
|
391
|
+
new AttemptStoreError({
|
|
392
|
+
code,
|
|
393
|
+
method,
|
|
394
|
+
message: `${code}: AttemptStore.${method}: ${detail}`,
|
|
395
|
+
cause
|
|
396
|
+
})
|
|
397
|
+
|
|
398
|
+
const error = (method: AttemptStoreMethod, code: AttemptStoreErrorCode, detail: string): AttemptStoreError =>
|
|
399
|
+
attemptStoreError(method, code, detail)
|
|
400
|
+
|
|
401
|
+
const encode = (
|
|
402
|
+
method: AttemptStoreMethod,
|
|
403
|
+
value: unknown,
|
|
404
|
+
field: string,
|
|
405
|
+
maxBytes?: number | undefined
|
|
406
|
+
): Effect.Effect<string, AttemptStoreError> =>
|
|
407
|
+
Effect.suspend(() => {
|
|
408
|
+
const admitted = Boundary.admitJson(value, jsonLimits(maxBytes))
|
|
409
|
+
return admitted.ok
|
|
410
|
+
? Effect.succeed(JSON.stringify(admitted.value))
|
|
411
|
+
: Effect.fail(error(method, "invalid_attempt", `${field} ${admitted.complaint}`))
|
|
412
|
+
})
|
|
413
|
+
|
|
414
|
+
const encodeOptionalWith =
|
|
415
|
+
(method: AttemptStoreMethod) =>
|
|
416
|
+
(value: unknown | undefined, field: string): Effect.Effect<string | null, AttemptStoreError> =>
|
|
417
|
+
value === undefined ? Effect.succeed(null) : encode(method, value, field)
|
|
418
|
+
|
|
419
|
+
const defaultMaxCheckpointBytes = 1024 * 1024
|
|
420
|
+
|
|
421
|
+
const defaultInProgressStates: ReadonlyArray<string> = ["running"]
|
|
422
|
+
|
|
423
|
+
const encodeCheckpointWith =
|
|
424
|
+
(method: AttemptStoreMethod, maxBytes: number) =>
|
|
425
|
+
(value: unknown | undefined): Effect.Effect<string | null, AttemptStoreError> =>
|
|
426
|
+
value === undefined ? Effect.succeed(null) : encode(method, value, "checkpoint", maxBytes)
|
|
427
|
+
|
|
428
|
+
const decode = (
|
|
429
|
+
method: AttemptStoreMethod,
|
|
430
|
+
value: string | null,
|
|
431
|
+
field: string,
|
|
432
|
+
maxBytes?: number | undefined
|
|
433
|
+
): Effect.Effect<JsonValue | undefined, AttemptStoreError> =>
|
|
434
|
+
Effect.suspend(() => {
|
|
435
|
+
if (value === null) return Effect.succeed(undefined)
|
|
436
|
+
const admitted = Boundary.admitJsonText(value, jsonLimits(maxBytes))
|
|
437
|
+
return admitted.ok
|
|
438
|
+
? Effect.succeed(admitted.json)
|
|
439
|
+
: Effect.fail(error(method, "decode_failed", `${field} ${admitted.complaint}`))
|
|
440
|
+
})
|
|
441
|
+
|
|
442
|
+
const decodeRequired = (
|
|
443
|
+
method: AttemptStoreMethod,
|
|
444
|
+
value: string,
|
|
445
|
+
field: string,
|
|
446
|
+
maxBytes?: number | undefined
|
|
447
|
+
): Effect.Effect<JsonValue, AttemptStoreError> =>
|
|
448
|
+
// A non-null SQL column cannot take `decode`'s missing-value branch.
|
|
449
|
+
decode(method, value, field, maxBytes) as Effect.Effect<JsonValue, AttemptStoreError>
|
|
450
|
+
|
|
451
|
+
const validateId = (method: AttemptStoreMethod, id: AttemptId): Effect.Effect<void, AttemptStoreError> =>
|
|
452
|
+
Boundary.isDurableText(id.runId) &&
|
|
453
|
+
Boundary.isDurableText(id.stepKeyDigest) &&
|
|
454
|
+
Number.isSafeInteger(id.attempt) &&
|
|
455
|
+
id.attempt >= 0
|
|
456
|
+
? Effect.void
|
|
457
|
+
: Effect.fail(
|
|
458
|
+
error(method, "invalid_attempt", "attempt identity violates the durable identifier contract")
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
const inspectRecord = (
|
|
462
|
+
input: unknown,
|
|
463
|
+
required: ReadonlyArray<string>,
|
|
464
|
+
optional: ReadonlyArray<string>,
|
|
465
|
+
field: string
|
|
466
|
+
): Readonly<Record<string, unknown>> => {
|
|
467
|
+
if (typeof input !== "object" || input === null) throw new TypeError(field)
|
|
468
|
+
const prototype = Object.getPrototypeOf(input)
|
|
469
|
+
if (prototype !== Object.prototype && prototype !== null) throw new TypeError(field)
|
|
470
|
+
const allowed = new Set([...required, ...optional])
|
|
471
|
+
const output = Object.create(null) as Record<string, unknown>
|
|
472
|
+
for (const name of required) {
|
|
473
|
+
const descriptor = Object.getOwnPropertyDescriptor(input, name)
|
|
474
|
+
if (descriptor === undefined || !("value" in descriptor) || !descriptor.enumerable) throw new TypeError(field)
|
|
475
|
+
output[name] = descriptor.value
|
|
476
|
+
}
|
|
477
|
+
for (const name of optional) {
|
|
478
|
+
const descriptor = Object.getOwnPropertyDescriptor(input, name)
|
|
479
|
+
if (descriptor === undefined) continue
|
|
480
|
+
if (!("value" in descriptor) || !descriptor.enumerable) throw new TypeError(field)
|
|
481
|
+
if (descriptor.value !== undefined) output[name] = descriptor.value
|
|
482
|
+
}
|
|
483
|
+
for (const key of Reflect.ownKeys(input)) {
|
|
484
|
+
if (typeof key !== "string" || !allowed.has(key)) {
|
|
485
|
+
if (Object.getOwnPropertyDescriptor(input, key)?.enumerable) throw new TypeError(field)
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
return output
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
/**
|
|
492
|
+
* Captures the fencing owner once, as a frozen inert record. Every span, SQL
|
|
493
|
+
* predicate, and fence comparison reads this copy, so an accessor, a proxy, or
|
|
494
|
+
* a mutation during an await cannot put two different owners into one write.
|
|
495
|
+
*/
|
|
496
|
+
const snapshotOwner = (method: AttemptStoreMethod, input: OwnerId): Effect.Effect<OwnerId, AttemptStoreError> =>
|
|
497
|
+
Effect.suspend(() => {
|
|
498
|
+
try {
|
|
499
|
+
const values = inspectRecord(input, ["hostId", "pid", "nonce"], [], "owner")
|
|
500
|
+
const owner = Object.freeze({ hostId: values.hostId, pid: values.pid, nonce: values.nonce })
|
|
501
|
+
if (
|
|
502
|
+
Boundary.isDurableText(owner.hostId) && Boundary.isDurableText(owner.nonce) &&
|
|
503
|
+
Number.isSafeInteger(owner.pid) && (owner.pid as number) >= 0
|
|
504
|
+
) {
|
|
505
|
+
return Effect.succeed(owner as OwnerId)
|
|
506
|
+
}
|
|
507
|
+
} catch {
|
|
508
|
+
// Falls through to the one refusal below.
|
|
509
|
+
}
|
|
510
|
+
return Effect.fail(error(method, "invalid_attempt", "owner must be an inert, valid owner identity"))
|
|
511
|
+
})
|
|
512
|
+
|
|
513
|
+
const snapshotJson = (
|
|
514
|
+
method: AttemptStoreMethod,
|
|
515
|
+
value: unknown,
|
|
516
|
+
field: string,
|
|
517
|
+
maxBytes?: number | undefined
|
|
518
|
+
): { readonly ok: true; readonly value: JsonValue } | { readonly ok: false; readonly error: AttemptStoreError } => {
|
|
519
|
+
const admitted = Boundary.admitJson(value, jsonLimits(maxBytes))
|
|
520
|
+
return admitted.ok
|
|
521
|
+
? { ok: true, value: admitted.value }
|
|
522
|
+
: { ok: false, error: error(method, "invalid_attempt", `${field} ${admitted.complaint}`) }
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* Structural admission for an attempt identity. `Attempt` and `FinishAttempt`
|
|
527
|
+
* extend `AttemptId`, so a stored attempt must pass wherever an id is typed:
|
|
528
|
+
* extra own data fields are ignored, while a non-plain prototype, any
|
|
529
|
+
* enumerable accessor, or a missing or inherited required field is refused.
|
|
530
|
+
*/
|
|
531
|
+
const inspectIdRecord = (input: unknown, field: string): Readonly<Record<string, unknown>> => {
|
|
532
|
+
if (typeof input !== "object" || input === null) throw new TypeError(field)
|
|
533
|
+
const prototype = Object.getPrototypeOf(input)
|
|
534
|
+
if (prototype !== Object.prototype && prototype !== null) throw new TypeError(field)
|
|
535
|
+
for (const key of Reflect.ownKeys(input)) {
|
|
536
|
+
const descriptor = Object.getOwnPropertyDescriptor(input, key)
|
|
537
|
+
if (descriptor !== undefined && descriptor.enumerable && !("value" in descriptor)) throw new TypeError(field)
|
|
538
|
+
}
|
|
539
|
+
const output = Object.create(null) as Record<string, unknown>
|
|
540
|
+
for (const name of ["runId", "stepKeyDigest", "attempt"]) {
|
|
541
|
+
const descriptor = Object.getOwnPropertyDescriptor(input, name)
|
|
542
|
+
if (descriptor === undefined || !("value" in descriptor) || !descriptor.enumerable) throw new TypeError(field)
|
|
543
|
+
output[name] = descriptor.value
|
|
544
|
+
}
|
|
545
|
+
return output
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
const snapshotId = (method: AttemptStoreMethod, input: AttemptId): Effect.Effect<AttemptId, AttemptStoreError> =>
|
|
549
|
+
Effect.suspend(() => {
|
|
550
|
+
try {
|
|
551
|
+
const values = inspectIdRecord(input, "attempt id")
|
|
552
|
+
const id = Object.freeze({
|
|
553
|
+
runId: values.runId,
|
|
554
|
+
stepKeyDigest: values.stepKeyDigest,
|
|
555
|
+
attempt: values.attempt
|
|
556
|
+
}) as AttemptId
|
|
557
|
+
return Effect.as(validateId(method, id), id)
|
|
558
|
+
} catch {
|
|
559
|
+
return Effect.fail(error(method, "invalid_attempt", "attempt id must be an inert data record"))
|
|
560
|
+
}
|
|
561
|
+
})
|
|
562
|
+
|
|
563
|
+
const snapshotAttempt = (method: AttemptStoreMethod, input: Attempt): Effect.Effect<Attempt, AttemptStoreError> =>
|
|
564
|
+
Effect.suspend(() => {
|
|
565
|
+
try {
|
|
566
|
+
const values = inspectRecord(
|
|
567
|
+
input,
|
|
568
|
+
["runId", "stepKeyDigest", "attempt", "state", "startedAtMs", "meta"],
|
|
569
|
+
["finishedAtMs", "heartbeatAtMs", "checkpoint", "error", "outcome"],
|
|
570
|
+
"attempt"
|
|
571
|
+
)
|
|
572
|
+
const checkpoint = values.checkpoint === undefined
|
|
573
|
+
? undefined
|
|
574
|
+
: snapshotJson(method, values.checkpoint, "checkpoint", maximumCheckpointBytes)
|
|
575
|
+
const attemptError = values.error === undefined ? undefined : snapshotJson(method, values.error, "error")
|
|
576
|
+
const outcome = values.outcome === undefined ? undefined : snapshotJson(method, values.outcome, "outcome")
|
|
577
|
+
const meta = snapshotJson(method, values.meta, "meta")
|
|
578
|
+
if (checkpoint !== undefined && !checkpoint.ok) return Effect.fail(checkpoint.error)
|
|
579
|
+
if (attemptError !== undefined && !attemptError.ok) return Effect.fail(attemptError.error)
|
|
580
|
+
if (outcome !== undefined && !outcome.ok) return Effect.fail(outcome.error)
|
|
581
|
+
if (!meta.ok) return Effect.fail(meta.error)
|
|
582
|
+
const candidate = Object.freeze({
|
|
583
|
+
runId: values.runId,
|
|
584
|
+
stepKeyDigest: values.stepKeyDigest,
|
|
585
|
+
attempt: values.attempt,
|
|
586
|
+
state: values.state,
|
|
587
|
+
startedAtMs: values.startedAtMs,
|
|
588
|
+
...(values.finishedAtMs === undefined ? {} : { finishedAtMs: values.finishedAtMs }),
|
|
589
|
+
...(values.heartbeatAtMs === undefined ? {} : { heartbeatAtMs: values.heartbeatAtMs }),
|
|
590
|
+
...(checkpoint === undefined ? {} : { checkpoint: checkpoint.value }),
|
|
591
|
+
...(attemptError === undefined ? {} : { error: attemptError.value }),
|
|
592
|
+
...(outcome === undefined ? {} : { outcome: outcome.value }),
|
|
593
|
+
meta: meta.value
|
|
594
|
+
})
|
|
595
|
+
return Schema.decodeUnknownEffect(Attempt)(candidate).pipe(
|
|
596
|
+
Effect.mapError(() => error(method, "invalid_attempt", "attempt violates the persistence contract"))
|
|
597
|
+
)
|
|
598
|
+
} catch {
|
|
599
|
+
return Effect.fail(error(method, "invalid_attempt", "attempt must be an inert data record"))
|
|
600
|
+
}
|
|
601
|
+
})
|
|
602
|
+
|
|
603
|
+
const snapshotFinish = (
|
|
604
|
+
method: AttemptStoreMethod,
|
|
605
|
+
input: FinishAttempt
|
|
606
|
+
): Effect.Effect<FinishAttempt, AttemptStoreError> =>
|
|
607
|
+
Effect.suspend(() => {
|
|
608
|
+
try {
|
|
609
|
+
const values = inspectRecord(
|
|
610
|
+
input,
|
|
611
|
+
["runId", "stepKeyDigest", "attempt", "state", "finishedAtMs"],
|
|
612
|
+
["error", "outcome", "meta"],
|
|
613
|
+
"finished attempt"
|
|
614
|
+
)
|
|
615
|
+
const attemptError = values.error === undefined ? undefined : snapshotJson(method, values.error, "error")
|
|
616
|
+
const outcome = values.outcome === undefined ? undefined : snapshotJson(method, values.outcome, "outcome")
|
|
617
|
+
const meta = values.meta === undefined ? undefined : snapshotJson(method, values.meta, "meta")
|
|
618
|
+
if (attemptError !== undefined && !attemptError.ok) return Effect.fail(attemptError.error)
|
|
619
|
+
if (outcome !== undefined && !outcome.ok) return Effect.fail(outcome.error)
|
|
620
|
+
if (meta !== undefined && !meta.ok) return Effect.fail(meta.error)
|
|
621
|
+
return Schema.decodeUnknownEffect(FinishAttempt)(Object.freeze({
|
|
622
|
+
runId: values.runId,
|
|
623
|
+
stepKeyDigest: values.stepKeyDigest,
|
|
624
|
+
attempt: values.attempt,
|
|
625
|
+
state: values.state,
|
|
626
|
+
finishedAtMs: values.finishedAtMs,
|
|
627
|
+
...(attemptError === undefined ? {} : { error: attemptError.value }),
|
|
628
|
+
...(outcome === undefined ? {} : { outcome: outcome.value }),
|
|
629
|
+
...(meta === undefined ? {} : { meta: meta.value })
|
|
630
|
+
})).pipe(Effect.mapError(() => error(method, "invalid_attempt", "finished attempt violates the contract")))
|
|
631
|
+
} catch {
|
|
632
|
+
return Effect.fail(error(method, "invalid_attempt", "finished attempt must be an inert data record"))
|
|
633
|
+
}
|
|
634
|
+
})
|
|
635
|
+
|
|
636
|
+
const snapshotPatch = (
|
|
637
|
+
method: AttemptStoreMethod,
|
|
638
|
+
input: AttemptPatch
|
|
639
|
+
): Effect.Effect<AttemptPatch, AttemptStoreError> =>
|
|
640
|
+
Effect.suspend(() => {
|
|
641
|
+
try {
|
|
642
|
+
const values = inspectRecord(input, [], ["checkpoint", "error", "outcome", "meta"], "attempt patch")
|
|
643
|
+
const checkpoint = values.checkpoint === undefined
|
|
644
|
+
? undefined
|
|
645
|
+
: snapshotJson(method, values.checkpoint, "checkpoint", maximumCheckpointBytes)
|
|
646
|
+
const attemptError = values.error === undefined ? undefined : snapshotJson(method, values.error, "error")
|
|
647
|
+
const outcome = values.outcome === undefined ? undefined : snapshotJson(method, values.outcome, "outcome")
|
|
648
|
+
const meta = values.meta === undefined ? undefined : snapshotJson(method, values.meta, "meta")
|
|
649
|
+
if (checkpoint !== undefined && !checkpoint.ok) return Effect.fail(checkpoint.error)
|
|
650
|
+
if (attemptError !== undefined && !attemptError.ok) return Effect.fail(attemptError.error)
|
|
651
|
+
if (outcome !== undefined && !outcome.ok) return Effect.fail(outcome.error)
|
|
652
|
+
if (meta !== undefined && !meta.ok) return Effect.fail(meta.error)
|
|
653
|
+
return Effect.succeed(Object.freeze({
|
|
654
|
+
...(checkpoint === undefined ? {} : { checkpoint: checkpoint.value }),
|
|
655
|
+
...(attemptError === undefined ? {} : { error: attemptError.value }),
|
|
656
|
+
...(outcome === undefined ? {} : { outcome: outcome.value }),
|
|
657
|
+
...(meta === undefined ? {} : { meta: meta.value })
|
|
658
|
+
}))
|
|
659
|
+
} catch {
|
|
660
|
+
return Effect.fail(error(method, "invalid_attempt", "attempt patch must be an inert data record"))
|
|
661
|
+
}
|
|
662
|
+
})
|
|
663
|
+
|
|
664
|
+
interface ResolvedOptions {
|
|
665
|
+
readonly inProgressStates: ReadonlyArray<string>
|
|
666
|
+
readonly maxCheckpointBytes: number
|
|
667
|
+
readonly putMode: "insert" | "upsert"
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
const snapshotOptions = (
|
|
671
|
+
method: AttemptStoreMethod,
|
|
672
|
+
input: Options
|
|
673
|
+
): Effect.Effect<ResolvedOptions, AttemptStoreError> =>
|
|
674
|
+
Effect.suspend(() => {
|
|
675
|
+
try {
|
|
676
|
+
const values = inspectRecord(
|
|
677
|
+
input,
|
|
678
|
+
[],
|
|
679
|
+
["inProgressStates", "maxCheckpointBytes", "putMode"],
|
|
680
|
+
"attempt store options"
|
|
681
|
+
)
|
|
682
|
+
const rawStates = values.inProgressStates ?? defaultInProgressStates
|
|
683
|
+
const admitted = Boundary.admitJson(rawStates, {
|
|
684
|
+
...jsonLimits(64 * 1024),
|
|
685
|
+
maxDepth: 2,
|
|
686
|
+
maxMembers: 256,
|
|
687
|
+
maxNodes: 257
|
|
688
|
+
})
|
|
689
|
+
if (!admitted.ok || !Array.isArray(admitted.value)) {
|
|
690
|
+
return Effect.fail(error(method, "invalid_attempt", "inProgressStates must be an inert string array"))
|
|
691
|
+
}
|
|
692
|
+
const states = admitted.value
|
|
693
|
+
if (
|
|
694
|
+
states.length === 0 ||
|
|
695
|
+
states.some((state) => !Boundary.isDurableText(state)) ||
|
|
696
|
+
new Set(states).size !== states.length
|
|
697
|
+
) {
|
|
698
|
+
return Effect.fail(
|
|
699
|
+
error(method, "invalid_attempt", "inProgressStates must contain unique durable state names")
|
|
700
|
+
)
|
|
701
|
+
}
|
|
702
|
+
const maxCheckpointBytes = values.maxCheckpointBytes ?? defaultMaxCheckpointBytes
|
|
703
|
+
if (
|
|
704
|
+
typeof maxCheckpointBytes !== "number" || !Number.isSafeInteger(maxCheckpointBytes) ||
|
|
705
|
+
maxCheckpointBytes <= 0 ||
|
|
706
|
+
maxCheckpointBytes > maximumCheckpointBytes
|
|
707
|
+
) {
|
|
708
|
+
return Effect.fail(
|
|
709
|
+
error(method, "invalid_attempt", `maxCheckpointBytes must be between 1 and ${maximumCheckpointBytes}`)
|
|
710
|
+
)
|
|
711
|
+
}
|
|
712
|
+
const putMode = values.putMode ?? "insert"
|
|
713
|
+
if (putMode !== "insert" && putMode !== "upsert") {
|
|
714
|
+
return Effect.fail(error(method, "invalid_attempt", "putMode must be insert or upsert"))
|
|
715
|
+
}
|
|
716
|
+
return Effect.succeed(Object.freeze({
|
|
717
|
+
inProgressStates: Object.freeze([...states]) as ReadonlyArray<string>,
|
|
718
|
+
maxCheckpointBytes,
|
|
719
|
+
putMode
|
|
720
|
+
}))
|
|
721
|
+
} catch {
|
|
722
|
+
return Effect.fail(error(method, "invalid_attempt", "attempt store options must be an inert data record"))
|
|
723
|
+
}
|
|
724
|
+
})
|
|
725
|
+
|
|
726
|
+
const ownsRunningRun = (row: RunFenceRow, owner: OwnerId): boolean =>
|
|
727
|
+
row.status === "running" &&
|
|
728
|
+
row.owner_host_id === owner.hostId &&
|
|
729
|
+
row.owner_pid === owner.pid &&
|
|
730
|
+
row.owner_nonce === owner.nonce
|
|
731
|
+
|
|
732
|
+
const sameAttempt = (stored: Attempt, expected: Attempt): boolean =>
|
|
733
|
+
stored.state === expected.state &&
|
|
734
|
+
stored.startedAtMs === expected.startedAtMs &&
|
|
735
|
+
stored.finishedAtMs === expected.finishedAtMs &&
|
|
736
|
+
stored.heartbeatAtMs === expected.heartbeatAtMs &&
|
|
737
|
+
Boundary.sameJson(stored.checkpoint, expected.checkpoint) &&
|
|
738
|
+
Boundary.sameJson(stored.error, expected.error) &&
|
|
739
|
+
Boundary.sameJson(stored.outcome, expected.outcome) &&
|
|
740
|
+
Boundary.sameJson(stored.meta, expected.meta)
|
|
741
|
+
|
|
742
|
+
/**
|
|
743
|
+
* Curried so each operation names itself once at construction and every
|
|
744
|
+
* failure travels through the one mapper, rather than through a per-call-site
|
|
745
|
+
* closure that only that operation's failing test could ever reach.
|
|
746
|
+
*/
|
|
747
|
+
const mapPersistenceError = (method: AttemptStoreMethod) => (cause: unknown): AttemptStoreError => {
|
|
748
|
+
if (Schema.is(AttemptStoreError)(cause)) {
|
|
749
|
+
return cause
|
|
750
|
+
}
|
|
751
|
+
const sqlError = SqlError.isSqlError(cause)
|
|
752
|
+
const databaseError = Schema.is(DatabaseError)(cause)
|
|
753
|
+
const constraint = sqlError
|
|
754
|
+
? cause.reason instanceof SqlError.ConstraintError || cause.reason instanceof SqlError.UniqueViolation
|
|
755
|
+
: databaseError && cause.code === "constraint"
|
|
756
|
+
const code = constraint ? "constraint" : "persistence_failed"
|
|
757
|
+
// Causes reach logs and telemetry. Driver failures may retain bound attempt
|
|
758
|
+
// payloads, so publish only a stable, payload-free descriptor.
|
|
759
|
+
return attemptStoreError(
|
|
760
|
+
method,
|
|
761
|
+
code,
|
|
762
|
+
"attempt persistence failed",
|
|
763
|
+
{
|
|
764
|
+
category: code,
|
|
765
|
+
reason: sqlError ? cause.reason._tag : databaseError ? cause.code : "unknown",
|
|
766
|
+
// The outer writer must still recognize a nested savepoint's conflict.
|
|
767
|
+
...(sqlError || databaseError
|
|
768
|
+
? { cause: new DatabaseError({ code: sqlError ? fromSqlError(cause).code : cause.code }) }
|
|
769
|
+
: {})
|
|
770
|
+
}
|
|
771
|
+
)
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
const decodeRow = (method: AttemptStoreMethod, input: unknown): Effect.Effect<Attempt, AttemptStoreError> =>
|
|
775
|
+
Schema.decodeUnknownEffect(AttemptRow)(input).pipe(
|
|
776
|
+
Effect.mapError(() => error(method, "decode_failed", "could not decode flows_attempts row")),
|
|
777
|
+
Effect.flatMap((row) =>
|
|
778
|
+
Effect.all({
|
|
779
|
+
checkpoint: decode(method, row.checkpoint_json, "checkpoint_json", maximumCheckpointBytes),
|
|
780
|
+
error: decode(method, row.error_json, "error_json"),
|
|
781
|
+
outcome: decode(method, row.outcome_json, "outcome_json"),
|
|
782
|
+
meta: decodeRequired(method, row.meta_json, "meta_json")
|
|
783
|
+
}).pipe(
|
|
784
|
+
Effect.map(({ checkpoint, error: attemptError, outcome, meta }) =>
|
|
785
|
+
Object.freeze({
|
|
786
|
+
runId: row.run_id,
|
|
787
|
+
stepKeyDigest: row.step_key_digest,
|
|
788
|
+
attempt: row.attempt,
|
|
789
|
+
state: row.state,
|
|
790
|
+
startedAtMs: row.started_at_ms,
|
|
791
|
+
...(row.finished_at_ms === null ? {} : { finishedAtMs: row.finished_at_ms }),
|
|
792
|
+
...(row.heartbeat_at_ms === null ? {} : { heartbeatAtMs: row.heartbeat_at_ms }),
|
|
793
|
+
...(checkpoint === undefined ? {} : { checkpoint }),
|
|
794
|
+
...(attemptError === undefined ? {} : { error: attemptError }),
|
|
795
|
+
...(outcome === undefined ? {} : { outcome }),
|
|
796
|
+
meta
|
|
797
|
+
})
|
|
798
|
+
)
|
|
799
|
+
)
|
|
800
|
+
)
|
|
801
|
+
)
|
|
802
|
+
|
|
803
|
+
/**
|
|
804
|
+
* Builds the SQL-backed attempt store under an explicit policy.
|
|
805
|
+
*
|
|
806
|
+
* @category constructors
|
|
807
|
+
* @since 0.1.0
|
|
808
|
+
*/
|
|
809
|
+
export const makeWith = (
|
|
810
|
+
options: Options = {}
|
|
811
|
+
): Effect.Effect<Service, AttemptStoreError, DurableWriter | SqlClient.SqlClient> =>
|
|
812
|
+
Effect.gen(function*() {
|
|
813
|
+
const sql = yield* Effect.service(SqlClient.SqlClient)
|
|
814
|
+
const writer = yield* DurableWriter
|
|
815
|
+
|
|
816
|
+
const configured = yield* snapshotOptions("makeWith", options)
|
|
817
|
+
const inProgressStates = configured.inProgressStates
|
|
818
|
+
const maxCheckpointBytes = configured.maxCheckpointBytes
|
|
819
|
+
const upsert = configured.putMode === "upsert"
|
|
820
|
+
const putEncodeOptional = encodeOptionalWith("put")
|
|
821
|
+
const putEncodeCheckpoint = encodeCheckpointWith("put", maxCheckpointBytes)
|
|
822
|
+
const heartbeatEncodeCheckpoint = encodeCheckpointWith("heartbeat", maxCheckpointBytes)
|
|
823
|
+
const finishEncodeOptional = encodeOptionalWith("finish")
|
|
824
|
+
const patchEncodeOptional = encodeOptionalWith("patch")
|
|
825
|
+
const patchEncodeCheckpoint = encodeCheckpointWith("patch", maxCheckpointBytes)
|
|
826
|
+
const inProgress = sql.in("state", [...inProgressStates])
|
|
827
|
+
|
|
828
|
+
const put: Service["put"] = Effect.fn("AttemptStore.put")((input, ownerInput) =>
|
|
829
|
+
Effect.gen(function*() {
|
|
830
|
+
const owner = yield* snapshotOwner("put", ownerInput)
|
|
831
|
+
const attempt = yield* snapshotAttempt("put", input)
|
|
832
|
+
yield* Effect.annotateCurrentSpan({
|
|
833
|
+
runId: attempt.runId,
|
|
834
|
+
stepKeyDigest: attempt.stepKeyDigest,
|
|
835
|
+
attempt: attempt.attempt,
|
|
836
|
+
ownerHostId: owner.hostId
|
|
837
|
+
})
|
|
838
|
+
const checkpoint = yield* putEncodeCheckpoint(attempt.checkpoint)
|
|
839
|
+
const attemptError = yield* putEncodeOptional(attempt.error, "error")
|
|
840
|
+
const outcome = yield* putEncodeOptional(attempt.outcome, "outcome")
|
|
841
|
+
const meta = yield* encode("put", attempt.meta, "meta")
|
|
842
|
+
return yield* writer.write(
|
|
843
|
+
Effect.gen(function*() {
|
|
844
|
+
const inserted = yield* sql<{ readonly attempt: number }>`
|
|
845
|
+
INSERT INTO flows_attempts (
|
|
846
|
+
run_id, step_key_digest, attempt, state, started_at_ms, finished_at_ms,
|
|
847
|
+
heartbeat_at_ms, checkpoint_json, error_json, outcome_json, meta_json
|
|
848
|
+
)
|
|
849
|
+
SELECT
|
|
850
|
+
${attempt.runId}, ${attempt.stepKeyDigest}, ${attempt.attempt}, ${attempt.state},
|
|
851
|
+
${attempt.startedAtMs}, ${attempt.finishedAtMs ?? null}, ${attempt.heartbeatAtMs ?? null},
|
|
852
|
+
${checkpoint}, ${attemptError}, ${outcome}, ${meta}
|
|
853
|
+
WHERE EXISTS (
|
|
854
|
+
SELECT 1 FROM flows_runs
|
|
855
|
+
WHERE run_id = ${attempt.runId}
|
|
856
|
+
AND status = 'running'
|
|
857
|
+
AND owner_host_id = ${owner.hostId}
|
|
858
|
+
AND owner_pid = ${owner.pid}
|
|
859
|
+
AND owner_nonce = ${owner.nonce}
|
|
860
|
+
)
|
|
861
|
+
ON CONFLICT (run_id, step_key_digest, attempt) DO NOTHING
|
|
862
|
+
RETURNING attempt
|
|
863
|
+
`
|
|
864
|
+
if (inserted.length > 0) {
|
|
865
|
+
return { _tag: "Inserted" } as const
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
if (upsert) {
|
|
869
|
+
const replaced = yield* sql<{ readonly attempt: number }>`
|
|
870
|
+
UPDATE flows_attempts
|
|
871
|
+
SET
|
|
872
|
+
state = ${attempt.state},
|
|
873
|
+
started_at_ms = ${attempt.startedAtMs},
|
|
874
|
+
finished_at_ms = ${attempt.finishedAtMs ?? null},
|
|
875
|
+
heartbeat_at_ms = ${attempt.heartbeatAtMs ?? null},
|
|
876
|
+
checkpoint_json = ${checkpoint},
|
|
877
|
+
error_json = ${attemptError},
|
|
878
|
+
outcome_json = ${outcome},
|
|
879
|
+
meta_json = ${meta}
|
|
880
|
+
WHERE run_id = ${attempt.runId}
|
|
881
|
+
AND step_key_digest = ${attempt.stepKeyDigest}
|
|
882
|
+
AND attempt = ${attempt.attempt}
|
|
883
|
+
AND ${inProgress}
|
|
884
|
+
AND EXISTS (
|
|
885
|
+
SELECT 1 FROM flows_runs
|
|
886
|
+
WHERE run_id = ${attempt.runId}
|
|
887
|
+
AND status = 'running'
|
|
888
|
+
AND owner_host_id = ${owner.hostId}
|
|
889
|
+
AND owner_pid = ${owner.pid}
|
|
890
|
+
AND owner_nonce = ${owner.nonce}
|
|
891
|
+
)
|
|
892
|
+
RETURNING attempt
|
|
893
|
+
`
|
|
894
|
+
if (replaced.length > 0) {
|
|
895
|
+
return { _tag: "Upserted" } as const
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
const runRows = yield* sql<RunFenceRow>`
|
|
900
|
+
SELECT status, owner_host_id, owner_pid, owner_nonce
|
|
901
|
+
FROM flows_runs WHERE run_id = ${attempt.runId}
|
|
902
|
+
`
|
|
903
|
+
if (runRows.length === 0) {
|
|
904
|
+
return { _tag: "RunNotFound" } as const
|
|
905
|
+
}
|
|
906
|
+
if (!ownsRunningRun(runRows[0]!, owner)) {
|
|
907
|
+
return { _tag: "FenceLost" } as const
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
const rows = yield* sql<AttemptRow>`
|
|
911
|
+
SELECT run_id, step_key_digest, attempt, state, started_at_ms, finished_at_ms,
|
|
912
|
+
heartbeat_at_ms, checkpoint_json, error_json, outcome_json, meta_json
|
|
913
|
+
FROM flows_attempts
|
|
914
|
+
WHERE run_id = ${attempt.runId}
|
|
915
|
+
AND step_key_digest = ${attempt.stepKeyDigest}
|
|
916
|
+
AND attempt = ${attempt.attempt}
|
|
917
|
+
`
|
|
918
|
+
/* v8 ignore next -- the owned run and conflicting row are read in the same serialized write transaction */
|
|
919
|
+
if (rows.length === 0) {
|
|
920
|
+
return yield* Effect.fail(error("put", "unknown", "attempt disappeared during put"))
|
|
921
|
+
}
|
|
922
|
+
const stored = yield* decodeRow("put", rows[0]!)
|
|
923
|
+
return sameAttempt(stored, attempt)
|
|
924
|
+
? { _tag: "ExistingSame" } as const
|
|
925
|
+
: { _tag: "Conflict" } as const
|
|
926
|
+
})
|
|
927
|
+
).pipe(Effect.mapError(mapPersistenceError("put")))
|
|
928
|
+
}).pipe(observeOutcome<PutResult>())
|
|
929
|
+
)
|
|
930
|
+
|
|
931
|
+
const get: Service["get"] = Effect.fn("AttemptStore.get")((input) =>
|
|
932
|
+
Effect.gen(function*() {
|
|
933
|
+
const id = yield* snapshotId("get", input)
|
|
934
|
+
yield* Effect.annotateCurrentSpan({
|
|
935
|
+
runId: id.runId,
|
|
936
|
+
stepKeyDigest: id.stepKeyDigest,
|
|
937
|
+
attempt: id.attempt
|
|
938
|
+
})
|
|
939
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
940
|
+
SELECT run_id, step_key_digest, attempt, state, started_at_ms, finished_at_ms,
|
|
941
|
+
heartbeat_at_ms, checkpoint_json, error_json, outcome_json, meta_json
|
|
942
|
+
FROM flows_attempts
|
|
943
|
+
WHERE run_id = ${id.runId} AND step_key_digest = ${id.stepKeyDigest} AND attempt = ${id.attempt}
|
|
944
|
+
`.pipe(Effect.mapError(mapPersistenceError("get")))
|
|
945
|
+
return rows.length === 0 ? Option.none() : yield* Effect.map(decodeRow("get", rows[0]!), Option.some)
|
|
946
|
+
}).pipe(observeExit)
|
|
947
|
+
)
|
|
948
|
+
|
|
949
|
+
const heartbeat: Service["heartbeat"] = Effect.fn("AttemptStore.heartbeat")((
|
|
950
|
+
runId,
|
|
951
|
+
stepKeyDigest,
|
|
952
|
+
attempt,
|
|
953
|
+
ownerInput,
|
|
954
|
+
nowMs,
|
|
955
|
+
checkpointValue
|
|
956
|
+
) =>
|
|
957
|
+
Effect.gen(function*() {
|
|
958
|
+
const owner = yield* snapshotOwner("heartbeat", ownerInput)
|
|
959
|
+
yield* validateId("heartbeat", { runId, stepKeyDigest, attempt })
|
|
960
|
+
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
|
|
961
|
+
return yield* Effect.fail(
|
|
962
|
+
error("heartbeat", "invalid_attempt", "nowMs must be a non-negative safe integer")
|
|
963
|
+
)
|
|
964
|
+
}
|
|
965
|
+
yield* Effect.annotateCurrentSpan({ runId, stepKeyDigest, attempt, ownerHostId: owner.hostId })
|
|
966
|
+
const checkpoint = yield* heartbeatEncodeCheckpoint(checkpointValue)
|
|
967
|
+
return yield* writer.write(
|
|
968
|
+
Effect.gen(function*() {
|
|
969
|
+
const updated = yield* sql<{ readonly attempt: number }>`
|
|
970
|
+
UPDATE flows_attempts
|
|
971
|
+
SET
|
|
972
|
+
heartbeat_at_ms = ${Dialect.greatest(sql)}(COALESCE(heartbeat_at_ms, ${nowMs}), ${nowMs}),
|
|
973
|
+
checkpoint_json = COALESCE(${checkpoint}, checkpoint_json)
|
|
974
|
+
WHERE run_id = ${runId}
|
|
975
|
+
AND step_key_digest = ${stepKeyDigest}
|
|
976
|
+
AND attempt = ${attempt}
|
|
977
|
+
AND ${inProgress}
|
|
978
|
+
AND EXISTS (
|
|
979
|
+
SELECT 1 FROM flows_runs
|
|
980
|
+
WHERE run_id = ${runId}
|
|
981
|
+
AND status = 'running'
|
|
982
|
+
AND owner_host_id = ${owner.hostId}
|
|
983
|
+
AND owner_pid = ${owner.pid}
|
|
984
|
+
AND owner_nonce = ${owner.nonce}
|
|
985
|
+
)
|
|
986
|
+
RETURNING attempt
|
|
987
|
+
`
|
|
988
|
+
if (updated.length > 0) {
|
|
989
|
+
return { _tag: "Updated" } as const
|
|
990
|
+
}
|
|
991
|
+
const found = yield* sql<Pick<AttemptRow, "state">>`
|
|
992
|
+
SELECT state FROM flows_attempts
|
|
993
|
+
WHERE run_id = ${runId} AND step_key_digest = ${stepKeyDigest} AND attempt = ${attempt}
|
|
994
|
+
`
|
|
995
|
+
if (found.length === 0) {
|
|
996
|
+
return { _tag: "NotFound" } as const
|
|
997
|
+
}
|
|
998
|
+
const runRows = yield* sql<RunFenceRow>`
|
|
999
|
+
SELECT status, owner_host_id, owner_pid, owner_nonce
|
|
1000
|
+
FROM flows_runs WHERE run_id = ${runId}
|
|
1001
|
+
`
|
|
1002
|
+
if (runRows.length === 0 || !ownsRunningRun(runRows[0]!, owner)) {
|
|
1003
|
+
return { _tag: "FenceLost" } as const
|
|
1004
|
+
}
|
|
1005
|
+
return { _tag: "StateChanged" } as const
|
|
1006
|
+
})
|
|
1007
|
+
).pipe(Effect.mapError(mapPersistenceError("heartbeat")))
|
|
1008
|
+
}).pipe(observeOutcome<HeartbeatResult>())
|
|
1009
|
+
)
|
|
1010
|
+
|
|
1011
|
+
const finish: Service["finish"] = Effect.fn("AttemptStore.finish")((input, ownerInput) =>
|
|
1012
|
+
Effect.gen(function*() {
|
|
1013
|
+
const owner = yield* snapshotOwner("finish", ownerInput)
|
|
1014
|
+
const attempt = yield* snapshotFinish("finish", input)
|
|
1015
|
+
yield* Effect.annotateCurrentSpan({
|
|
1016
|
+
runId: attempt.runId,
|
|
1017
|
+
stepKeyDigest: attempt.stepKeyDigest,
|
|
1018
|
+
attempt: attempt.attempt,
|
|
1019
|
+
ownerHostId: owner.hostId
|
|
1020
|
+
})
|
|
1021
|
+
if (
|
|
1022
|
+
inProgressStates.includes(attempt.state) ||
|
|
1023
|
+
!Boundary.isDurableText(attempt.state)
|
|
1024
|
+
) {
|
|
1025
|
+
return yield* Effect.fail(
|
|
1026
|
+
error("finish", "invalid_attempt", "finish requires a terminal state and valid timestamp")
|
|
1027
|
+
)
|
|
1028
|
+
}
|
|
1029
|
+
const attemptError = yield* finishEncodeOptional(attempt.error, "error")
|
|
1030
|
+
const outcome = yield* finishEncodeOptional(attempt.outcome, "outcome")
|
|
1031
|
+
const meta = yield* finishEncodeOptional(attempt.meta, "meta")
|
|
1032
|
+
return yield* writer.write(
|
|
1033
|
+
Effect.gen(function*() {
|
|
1034
|
+
const updated = yield* sql<{ readonly attempt: number }>`
|
|
1035
|
+
UPDATE flows_attempts
|
|
1036
|
+
SET
|
|
1037
|
+
state = ${attempt.state},
|
|
1038
|
+
finished_at_ms = ${attempt.finishedAtMs},
|
|
1039
|
+
error_json = COALESCE(${attemptError}, error_json),
|
|
1040
|
+
outcome_json = COALESCE(${outcome}, outcome_json),
|
|
1041
|
+
meta_json = COALESCE(${meta}, meta_json)
|
|
1042
|
+
WHERE run_id = ${attempt.runId}
|
|
1043
|
+
AND step_key_digest = ${attempt.stepKeyDigest}
|
|
1044
|
+
AND attempt = ${attempt.attempt}
|
|
1045
|
+
AND ${inProgress}
|
|
1046
|
+
AND EXISTS (
|
|
1047
|
+
SELECT 1 FROM flows_runs
|
|
1048
|
+
WHERE run_id = ${attempt.runId}
|
|
1049
|
+
AND status = 'running'
|
|
1050
|
+
AND owner_host_id = ${owner.hostId}
|
|
1051
|
+
AND owner_pid = ${owner.pid}
|
|
1052
|
+
AND owner_nonce = ${owner.nonce}
|
|
1053
|
+
)
|
|
1054
|
+
RETURNING attempt
|
|
1055
|
+
`
|
|
1056
|
+
if (updated.length > 0) {
|
|
1057
|
+
return { _tag: "Finished" } as const
|
|
1058
|
+
}
|
|
1059
|
+
const found = yield* sql<Pick<AttemptRow, "state">>`
|
|
1060
|
+
SELECT state FROM flows_attempts
|
|
1061
|
+
WHERE run_id = ${attempt.runId}
|
|
1062
|
+
AND step_key_digest = ${attempt.stepKeyDigest}
|
|
1063
|
+
AND attempt = ${attempt.attempt}
|
|
1064
|
+
`
|
|
1065
|
+
if (found.length === 0) {
|
|
1066
|
+
return { _tag: "NotFound" } as const
|
|
1067
|
+
}
|
|
1068
|
+
const runRows = yield* sql<RunFenceRow>`
|
|
1069
|
+
SELECT status, owner_host_id, owner_pid, owner_nonce
|
|
1070
|
+
FROM flows_runs WHERE run_id = ${attempt.runId}
|
|
1071
|
+
`
|
|
1072
|
+
if (runRows.length === 0 || !ownsRunningRun(runRows[0]!, owner)) {
|
|
1073
|
+
return { _tag: "FenceLost" } as const
|
|
1074
|
+
}
|
|
1075
|
+
return { _tag: "StateChanged" } as const
|
|
1076
|
+
})
|
|
1077
|
+
).pipe(Effect.mapError(mapPersistenceError("finish")))
|
|
1078
|
+
}).pipe(observeOutcome<FinishResult>())
|
|
1079
|
+
)
|
|
1080
|
+
|
|
1081
|
+
const patch: Service["patch"] = Effect.fn("AttemptStore.patch")((idInput, patchInput, ownerInput) =>
|
|
1082
|
+
Effect.gen(function*() {
|
|
1083
|
+
const owner = yield* snapshotOwner("patch", ownerInput)
|
|
1084
|
+
const id = yield* snapshotId("patch", idInput)
|
|
1085
|
+
const fields = yield* snapshotPatch("patch", patchInput)
|
|
1086
|
+
yield* Effect.annotateCurrentSpan({
|
|
1087
|
+
runId: id.runId,
|
|
1088
|
+
stepKeyDigest: id.stepKeyDigest,
|
|
1089
|
+
attempt: id.attempt,
|
|
1090
|
+
ownerHostId: owner.hostId
|
|
1091
|
+
})
|
|
1092
|
+
const checkpoint = yield* patchEncodeCheckpoint(fields.checkpoint)
|
|
1093
|
+
const attemptError = yield* patchEncodeOptional(fields.error, "error")
|
|
1094
|
+
const outcome = yield* patchEncodeOptional(fields.outcome, "outcome")
|
|
1095
|
+
const meta = yield* patchEncodeOptional(fields.meta, "meta")
|
|
1096
|
+
return yield* writer.write(
|
|
1097
|
+
Effect.gen(function*() {
|
|
1098
|
+
// Unlike `heartbeat`/`finish` there is no state predicate: a patch
|
|
1099
|
+
// may touch a terminal row (evidence quarantine does), so the run
|
|
1100
|
+
// fence is the only gate. After a terminal run transition the
|
|
1101
|
+
// owner columns are cleared and every patch is refused.
|
|
1102
|
+
const updated = yield* sql<{ readonly attempt: number }>`
|
|
1103
|
+
UPDATE flows_attempts
|
|
1104
|
+
SET
|
|
1105
|
+
checkpoint_json = COALESCE(${checkpoint}, checkpoint_json),
|
|
1106
|
+
error_json = COALESCE(${attemptError}, error_json),
|
|
1107
|
+
outcome_json = COALESCE(${outcome}, outcome_json),
|
|
1108
|
+
meta_json = COALESCE(${meta}, meta_json)
|
|
1109
|
+
WHERE run_id = ${id.runId}
|
|
1110
|
+
AND step_key_digest = ${id.stepKeyDigest}
|
|
1111
|
+
AND attempt = ${id.attempt}
|
|
1112
|
+
AND EXISTS (
|
|
1113
|
+
SELECT 1 FROM flows_runs
|
|
1114
|
+
WHERE run_id = ${id.runId}
|
|
1115
|
+
AND status = 'running'
|
|
1116
|
+
AND owner_host_id = ${owner.hostId}
|
|
1117
|
+
AND owner_pid = ${owner.pid}
|
|
1118
|
+
AND owner_nonce = ${owner.nonce}
|
|
1119
|
+
)
|
|
1120
|
+
RETURNING attempt
|
|
1121
|
+
`
|
|
1122
|
+
if (updated.length > 0) {
|
|
1123
|
+
return { _tag: "Patched" } as const
|
|
1124
|
+
}
|
|
1125
|
+
const found = yield* sql<Pick<AttemptRow, "state">>`
|
|
1126
|
+
SELECT state FROM flows_attempts
|
|
1127
|
+
WHERE run_id = ${id.runId} AND step_key_digest = ${id.stepKeyDigest} AND attempt = ${id.attempt}
|
|
1128
|
+
`
|
|
1129
|
+
return found.length === 0
|
|
1130
|
+
? { _tag: "NotFound" } as const
|
|
1131
|
+
: { _tag: "FenceLost" } as const
|
|
1132
|
+
})
|
|
1133
|
+
).pipe(Effect.mapError(mapPersistenceError("patch")))
|
|
1134
|
+
}).pipe(observeOutcome<PatchResult>())
|
|
1135
|
+
)
|
|
1136
|
+
|
|
1137
|
+
return { put, get, heartbeat, finish, patch }
|
|
1138
|
+
})
|
|
1139
|
+
|
|
1140
|
+
/**
|
|
1141
|
+
* Builds the SQL-backed attempt store with default policy.
|
|
1142
|
+
*
|
|
1143
|
+
* @category constructors
|
|
1144
|
+
* @since 0.1.0
|
|
1145
|
+
*/
|
|
1146
|
+
export const make: Effect.Effect<Service, never, DurableWriter | SqlClient.SqlClient> = Effect.orDie(makeWith())
|
|
1147
|
+
|
|
1148
|
+
/**
|
|
1149
|
+
* Creates an attempt store from an implementation.
|
|
1150
|
+
*
|
|
1151
|
+
* @category constructors
|
|
1152
|
+
* @since 0.1.0
|
|
1153
|
+
*/
|
|
1154
|
+
export const makeNoop = (overrides: Partial<Service> = {}): Service => {
|
|
1155
|
+
const unavailable = (method: Exclude<AttemptStoreMethod, "makeWith">) =>
|
|
1156
|
+
Effect.fail(attemptStoreError(method, "unknown", "the store is unavailable in this environment"))
|
|
1157
|
+
return {
|
|
1158
|
+
put: Effect.fn("AttemptStore.put")(() => unavailable("put")),
|
|
1159
|
+
get: Effect.fn("AttemptStore.get")(() => unavailable("get")),
|
|
1160
|
+
heartbeat: Effect.fn("AttemptStore.heartbeat")(() => unavailable("heartbeat")),
|
|
1161
|
+
finish: Effect.fn("AttemptStore.finish")(() => unavailable("finish")),
|
|
1162
|
+
patch: Effect.fn("AttemptStore.patch")(() => unavailable("patch")),
|
|
1163
|
+
...overrides
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
/**
|
|
1168
|
+
* Provides a no-op attempt store.
|
|
1169
|
+
*
|
|
1170
|
+
* @category layers
|
|
1171
|
+
* @since 0.1.0
|
|
1172
|
+
*/
|
|
1173
|
+
export const layerNoop = (overrides: Partial<Service> = {}): Layer.Layer<AttemptStore> =>
|
|
1174
|
+
Layer.succeed(AttemptStore)(makeNoop(overrides))
|
|
1175
|
+
|
|
1176
|
+
/**
|
|
1177
|
+
* Provides the SQL-backed attempt store.
|
|
1178
|
+
*
|
|
1179
|
+
* @category layers
|
|
1180
|
+
* @since 0.1.0
|
|
1181
|
+
*/
|
|
1182
|
+
export const layer: Layer.Layer<AttemptStore, never, DurableWriter | SqlClient.SqlClient> = Layer.effect(AttemptStore)(
|
|
1183
|
+
make
|
|
1184
|
+
)
|
|
1185
|
+
|
|
1186
|
+
/**
|
|
1187
|
+
* Provides the SQL-backed attempt store under an explicit policy.
|
|
1188
|
+
*
|
|
1189
|
+
* @category layers
|
|
1190
|
+
* @since 0.1.0
|
|
1191
|
+
*/
|
|
1192
|
+
export const layerWith = (
|
|
1193
|
+
options: Options
|
|
1194
|
+
): Layer.Layer<AttemptStore, AttemptStoreError, DurableWriter | SqlClient.SqlClient> =>
|
|
1195
|
+
Layer.effect(AttemptStore)(makeWith(options))
|