@fungi.computer/watchdog 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -41
- package/dist/d1.js +1 -0
- package/dist/effect.d.ts +120 -8
- package/dist/effect.js +34 -11
- package/dist/index.d.ts +7 -5
- package/dist/index.js +2 -2
- package/dist/internal/async-runtime.d.ts +3 -1
- package/dist/internal/async-runtime.js +12 -3
- package/dist/internal/async-sql.d.ts +2 -1
- package/dist/internal/async-sql.js +18 -9
- package/dist/internal/async-store.d.ts +40 -7
- package/dist/internal/async-store.js +177 -24
- package/dist/internal/promise-watchdog-host.js +11 -2
- package/dist/internal/recovery.d.ts +9 -0
- package/dist/internal/recovery.js +20 -0
- package/dist/internal/retry.d.ts +19 -0
- package/dist/internal/retry.js +32 -0
- package/dist/internal/runtime.js +184 -82
- package/dist/internal/sqlite-store.d.ts +50 -9
- package/dist/internal/sqlite-store.js +200 -59
- package/dist/internal/watchdog-error.d.ts +2 -2
- package/dist/postgres.js +1 -0
- package/docs/api/d1.md +4 -3
- package/docs/api/effect.md +267 -29
- package/docs/api/index.md +48 -11
- package/docs/api/postgres.md +3 -3
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -7,8 +7,8 @@ asks your liveness probe what happened and recovers the work.
|
|
|
7
7
|
|
|
8
8
|
Watchdog never looks inside a job. The payload, queue name and lane are opaque
|
|
9
9
|
values you choose. Your code decides what a job means and how to run it.
|
|
10
|
-
Watchdog owns the lifecycle: queued, running, settled. Claims, claim
|
|
11
|
-
the SQL that moves rows between states stay private to the package.
|
|
10
|
+
Watchdog owns the lifecycle: queued, running, completing, settled. Claims, claim
|
|
11
|
+
tokens and the SQL that moves rows between states stay private to the package.
|
|
12
12
|
|
|
13
13
|
Use it when you need work to survive a restart, run once per claim, and settle
|
|
14
14
|
with an honest outcome, without writing your own state machine. It runs on any
|
|
@@ -23,8 +23,8 @@ npm install @fungi.computer/watchdog
|
|
|
23
23
|
|
|
24
24
|
Watchdog is ESM-only. The SQLite entrypoint runs anywhere you have a synchronous
|
|
25
25
|
SQLite handle, such as Cloudflare Durable Object storage or `node:sqlite` in
|
|
26
|
-
Node 24.
|
|
27
|
-
installs for you.
|
|
26
|
+
Node 24. Its runtime dependencies include Cairn, `effect`, `pg`, `@types/pg` and
|
|
27
|
+
`@cloudflare/workers-types`, which npm installs for you.
|
|
28
28
|
|
|
29
29
|
Entrypoints:
|
|
30
30
|
|
|
@@ -95,13 +95,15 @@ In a Durable Object, `ctx.storage` is already the SQLite owner.
|
|
|
95
95
|
## Core concepts
|
|
96
96
|
|
|
97
97
|
**Jobs.** A job has a `jobId`, a `queue`, a `lane`, an integer `priority`, an
|
|
98
|
-
opaque JSON `payload` and an optional `recovery: { maxRecoveries }`.
|
|
99
|
-
queue names are 1 to 256 characters; lanes are 1 to 512 characters.
|
|
100
|
-
construction can reserve one lane for concurrent delivery; otherwise
|
|
101
|
-
stored without a scheduling policy. A job moves through
|
|
98
|
+
opaque JSON `payload` and an optional `recovery: { maxRecoveries, windowMs? }`.
|
|
99
|
+
IDs and queue names are 1 to 256 characters; lanes are 1 to 512 characters.
|
|
100
|
+
Owner-local construction can reserve one lane for concurrent delivery; otherwise
|
|
101
|
+
the lane is stored without a scheduling policy. A job moves through four states:
|
|
102
102
|
|
|
103
103
|
- `queued`: accepted and waiting.
|
|
104
104
|
- `running`: claimed by one tick and handed to your executor.
|
|
105
|
+
- `completing`: execution decided its outcome; settlement and its projection are
|
|
106
|
+
pending.
|
|
105
107
|
- `settled`: finished, with an `outcome` of `completed`, `failed`, `cancelled`,
|
|
106
108
|
`outcome_unknown` or `interrupted`, and an optional `failureReason`.
|
|
107
109
|
|
|
@@ -125,25 +127,50 @@ as `execution_failed` or `execution_interrupted`.
|
|
|
125
127
|
- `isAlive(job)`: answers whether a running job's execution still exists.
|
|
126
128
|
- `wake.recompute()`: arms your next timer or alarm after durable state changes.
|
|
127
129
|
|
|
128
|
-
**
|
|
129
|
-
`
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
130
|
+
**Completion recovery.** An exact claim first records its outcome as
|
|
131
|
+
`completing`. A separate transaction settles that decision and invokes the owner
|
|
132
|
+
projection. If this transaction fails, the decision remains durable. Each tick
|
|
133
|
+
retries one pending completion before taking new work. Failure surfaces
|
|
134
|
+
immediately; the host owns bounded wake backoff. A restart never executes
|
|
135
|
+
completing work again, probes its liveness or charges its recovery budget. Queue
|
|
136
|
+
heads report it as `running` until settlement commits; `read(jobId)` reports
|
|
137
|
+
`completing`.
|
|
138
|
+
|
|
139
|
+
**Disappearance recovery.** After pending completion, a tick checks the oldest
|
|
140
|
+
running claim. If `isAlive` returns `false`, the execution is gone. A job with
|
|
141
|
+
recovery budget left goes back to `queued`. Otherwise it settles as
|
|
142
|
+
`interrupted`. If `isAlive` throws, Watchdog does not guess. The claim stays as
|
|
143
|
+
it is, and an otherwise idle tick reports
|
|
144
|
+
`{ status: "liveness_unknown", jobId, reason }`.
|
|
145
|
+
|
|
146
|
+
An optional positive `windowMs` resets the count when that much time has elapsed
|
|
147
|
+
since the first charged recovery in the current window. Clock rollback does not
|
|
148
|
+
reset it. Without a window, the count remains a lifetime budget for that job.
|
|
149
|
+
|
|
150
|
+
**Effect owner shutdown.** The Effect runtime's `close` stops dispatch and
|
|
151
|
+
interrupts its owned execution. After the executor scope ends, `isDrained(job)`
|
|
152
|
+
must prove that foreign cleanup completed. Only then does Watchdog requeue its
|
|
153
|
+
exact claim without charging recovery. An absent or false drain answer leaves
|
|
154
|
+
the claim running and fails with `drain_failed`; reopening treats it as an
|
|
155
|
+
unexpected disappearance. Durable cancellation wins over the handoff. An
|
|
156
|
+
already-decided completion continues settlement. Further ticks return `stopped`.
|
|
157
|
+
The production caller is Shiitake, whose run scope supplies bounded model/tool
|
|
158
|
+
cleanup evidence. Promise runtimes do not expose this lifecycle operation.
|
|
133
159
|
|
|
134
160
|
**Cancellation.** `cancel({ jobId, cancellationId })` records a durable
|
|
135
161
|
cancellation. A queued job settles as `cancelled` on its next tick. A job the
|
|
136
162
|
current runtime is executing is interrupted and settles as `cancelled`.
|
|
137
163
|
Replaying the same `cancellationId` returns `replayed`. A different one returns
|
|
138
|
-
`conflict`.
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
164
|
+
`conflict`. Once execution has durably decided, cancellation is a no-op and
|
|
165
|
+
returns `{ status: "decided", outcome }`, including after settlement. Pass
|
|
166
|
+
`onlyIfQueued: true` to refuse cancellation once a job has started.
|
|
167
|
+
|
|
168
|
+
**Lane refusal.** `failLane({ lane, reason })` terminally settles queued,
|
|
169
|
+
running and completing work in that lane as `failed`, retaining each job and its
|
|
170
|
+
bounded owner reason code. It handles at most 256 jobs per call, running jobs
|
|
171
|
+
first, and returns `{ settled, more }`. Repeat only while `more` is true; 256 is
|
|
172
|
+
a transition batch, not a cap on accepted work. Already settled history and
|
|
173
|
+
other lanes stay intact. Once exhausted, another call returns
|
|
147
174
|
`{ settled: 0, more: false }` and fires no projector. A later enqueue is new
|
|
148
175
|
work, not a permanent refusal marker.
|
|
149
176
|
|
|
@@ -171,8 +198,25 @@ ingest, claim, cancellation, settlement, lane refusal, recovery and purge. Use
|
|
|
171
198
|
it to keep a per-queue status row in step with Watchdog. A throw from `project`
|
|
172
199
|
fails the transition.
|
|
173
200
|
|
|
174
|
-
**Purge.** `purge(queue)` deletes every queued job in one queue. Running
|
|
175
|
-
settled jobs stay.
|
|
201
|
+
**Purge.** `purge(queue)` deletes every queued job in one queue. Running,
|
|
202
|
+
completing and settled jobs stay.
|
|
203
|
+
|
|
204
|
+
## Scheduled work and delivery attempts
|
|
205
|
+
|
|
206
|
+
A queued job may carry `notBefore`, an absolute millisecond deadline. Watchdog
|
|
207
|
+
retains it as `not_before`; future queue heads hold their successors while
|
|
208
|
+
independent queues proceed. `wakeState(now)` returns `{ pending, nextWakeAt }`
|
|
209
|
+
from the same rows, including shared leases. Cancellation bypasses a future due
|
|
210
|
+
time. Hosts schedule now when `pending` is true, otherwise at `nextWakeAt`.
|
|
211
|
+
|
|
212
|
+
An executor may supply a code-owned `retryPolicy` with bounded backoff and
|
|
213
|
+
either an attempt ceiling or absolute deadline. Watchdog increments
|
|
214
|
+
`attempt_count` and records the fallback due time before execution. The executor
|
|
215
|
+
receives the reserved attempt, returns `retry` to retain the same work, and
|
|
216
|
+
receives an exhausted context without another receiver attempt when the budget
|
|
217
|
+
ends. Delivery disappearance retains that charge and does not borrow Agent
|
|
218
|
+
recovery quotas. The native Effect executor's `replaySafe` predicate is for work
|
|
219
|
+
whose entire effect is one owner transaction, such as extension timers.
|
|
176
220
|
|
|
177
221
|
## Concurrent owner-local delivery
|
|
178
222
|
|
|
@@ -185,34 +229,40 @@ partition. Both slots require the queue head across lanes, ordered by running
|
|
|
185
229
|
first, then priority descending and enqueue sequence ascending. Cancelled queued
|
|
186
230
|
jobs can settle past a busy queue without executing.
|
|
187
231
|
|
|
188
|
-
The Effect runtime's `readSettled(limit, afterSequence)` exposes the
|
|
189
|
-
partition in enqueue order. A prefix fence withholds later settlements
|
|
190
|
-
earlier
|
|
191
|
-
page slot; they remain readable by
|
|
192
|
-
partitions. These owner-local
|
|
193
|
-
not change the D1 or
|
|
232
|
+
The Effect runtime's `readSettled(limit, afterSequence, lane?)` exposes the
|
|
233
|
+
serial partition in enqueue order. A prefix fence withholds later settlements
|
|
234
|
+
until all earlier jobs in the selected partition and optional lane settle or are
|
|
235
|
+
purged. Concurrent receipts never consume a page slot; they remain readable by
|
|
236
|
+
job ID. `wakeState(now)` still covers both partitions. These owner-local
|
|
237
|
+
operations require no stored schema change and do not change the D1 or
|
|
238
|
+
PostgreSQL adapters.
|
|
194
239
|
|
|
195
240
|
## Shared D1 and PostgreSQL storage
|
|
196
241
|
|
|
197
242
|
`@fungi.computer/watchdog/d1` exports `createD1Watchdog` and `watchdogD1Schema`.
|
|
198
243
|
`@fungi.computer/watchdog/postgres` exports `createPostgresWatchdog` and
|
|
199
244
|
`watchdogPostgresSchema`. Apply the package-owned schema through your migration
|
|
200
|
-
tool before you open a runtime. Opening a runtime never migrates a database.
|
|
245
|
+
tool before you open a runtime. Opening a runtime never migrates a database. The
|
|
246
|
+
named `watchdog_execution_state` constraint identifies the current lifecycle
|
|
247
|
+
grammar. An obsolete grammar is refused before execution. Native prepared
|
|
248
|
+
admission remains part of the caller's migrated transaction; it performs no
|
|
249
|
+
schema repair. Owner-local SQLite applies the same refusal when opening.
|
|
201
250
|
|
|
202
251
|
These runtimes have the same `ingest`, `read`, `readQueueHead`, `tick`,
|
|
203
|
-
`cancel`, `failLane`, `purge` and `
|
|
252
|
+
`cancel`, `failLane`, `purge` and `wakeState(now)` operations. They take
|
|
204
253
|
`execute`, `wake` and an optional `now` clock instead of an owner and liveness
|
|
205
254
|
probe. Watchdog answers liveness itself from the claim's lease.
|
|
206
255
|
|
|
207
256
|
D1 `enqueue(job)` returns a prepared statement for the producer's native batch.
|
|
208
257
|
`enqueueFromSelect({ text, values })` accepts a trusted native SQLite
|
|
209
258
|
`?NNN`-bound SELECT of `job_id`, `queue_name`, `lane`, `priority`,
|
|
210
|
-
`payload_json` and nullable `
|
|
211
|
-
conflict checks. This lets a job depend on
|
|
212
|
-
in the same batch, without a JavaScript
|
|
213
|
-
`enqueue(client, job)` uses the caller's checked-out
|
|
214
|
-
caller owns BEGIN, COMMIT and ROLLBACK. An identical
|
|
215
|
-
conflicting identity or cargo fails the whole native
|
|
259
|
+
`payload_json`, nullable `recovery_max` and nullable `recovery_window_ms`.
|
|
260
|
+
Watchdog supplies its INSERT and conflict checks. This lets a job depend on
|
|
261
|
+
facts conditionally inserted earlier in the same batch, without a JavaScript
|
|
262
|
+
read in between. PostgreSQL `enqueue(client, job)` uses the caller's checked-out
|
|
263
|
+
transaction client. The caller owns BEGIN, COMMIT and ROLLBACK. An identical
|
|
264
|
+
replay does nothing. A conflicting identity or cargo fails the whole native
|
|
265
|
+
transaction.
|
|
216
266
|
|
|
217
267
|
Both adapters use the same lifecycle engine as the SQLite entrypoint. Claims
|
|
218
268
|
stay private, with two-minute leases, ten-second renewal and bounded renewal
|
|
@@ -241,7 +291,7 @@ modules:
|
|
|
241
291
|
with their Effect schemas.
|
|
242
292
|
- `@fungi.computer/watchdog/effect`: `makeRuntime`, `WatchdogRuntimeEffect` and
|
|
243
293
|
`EffectWatchdogOptions`. The Effect runtime also has
|
|
244
|
-
`readSettled(limit, afterSequence?)` for paging through settled jobs.
|
|
294
|
+
`readSettled(limit, afterSequence?, lane?)` for paging through settled jobs.
|
|
245
295
|
- `@fungi.computer/watchdog/node-sqlite`: `fromNodeSqlite(database)` for a
|
|
246
296
|
caller-owned Node SQLite database. It returns `NodeSqliteOwner`; its
|
|
247
297
|
`NodeSqliteCursor` provides `toArray()` and `one()`, which returns exactly one
|
|
@@ -301,9 +351,12 @@ Every failure is a `WatchdogError` with a closed `cause`:
|
|
|
301
351
|
|
|
302
352
|
- `store_unavailable`: the store threw, or an ingest replay conflicted with the
|
|
303
353
|
stored job.
|
|
304
|
-
- `store_invalid`: the
|
|
305
|
-
Other store faults remain
|
|
354
|
+
- `store_invalid`: the lifecycle schema is obsolete, or lane-refusal input
|
|
355
|
+
failed its strict lane/reason decoder. Other store faults remain
|
|
356
|
+
`store_unavailable`.
|
|
306
357
|
- `claim_unavailable`: a claim could not be taken or renewed.
|
|
358
|
+
- `drain_failed`: interrupted execution did not prove that foreign cleanup
|
|
359
|
+
finished.
|
|
307
360
|
- `execution_failed`: your Promise executor threw. `tick()` does not reject. The
|
|
308
361
|
job settles as `failed` with this code as its `failureReason`.
|
|
309
362
|
- `settlement_rejected`: the claim was replaced or expired before settlement.
|
package/dist/d1.js
CHANGED
|
@@ -9,6 +9,7 @@ export function createD1Watchdog(options) {
|
|
|
9
9
|
.prepare(statement.text.replace(/\$(\d+)/gu, "?$1"))
|
|
10
10
|
.bind(...statement.values);
|
|
11
11
|
const runtime = createAsyncRuntime({
|
|
12
|
+
dialect: "sqlite",
|
|
12
13
|
clock: options.now === undefined ? "unixepoch('subsec')*1000" : undefined,
|
|
13
14
|
query: async (statement) => (await prepare(statement).all()).results,
|
|
14
15
|
}, options);
|
package/dist/effect.d.ts
CHANGED
|
@@ -1,6 +1,27 @@
|
|
|
1
1
|
import * as Effect from "effect/Effect";
|
|
2
2
|
import * as Schema from "effect/Schema";
|
|
3
3
|
import { WatchdogError } from "./internal/watchdog-error.js";
|
|
4
|
+
/** Immediate reconciliation and the earliest future durable deadline. */
|
|
5
|
+
export type WatchdogWakeState = Readonly<{
|
|
6
|
+
pending: boolean;
|
|
7
|
+
nextWakeAt: number | null;
|
|
8
|
+
}>;
|
|
9
|
+
/** Bounded receiver policy selected by the executor's existing delivery owner. */
|
|
10
|
+
export type WatchdogRetryPolicy = Readonly<{
|
|
11
|
+
baseMs: number;
|
|
12
|
+
maxMs: number;
|
|
13
|
+
limit: Readonly<{
|
|
14
|
+
maxAttempts: number;
|
|
15
|
+
}> | Readonly<{
|
|
16
|
+
deadlineAt: number;
|
|
17
|
+
}>;
|
|
18
|
+
}>;
|
|
19
|
+
/** Reservation committed before receiver I/O; exhausted work may only finish its receipt. */
|
|
20
|
+
export type WatchdogAttempt = Readonly<{
|
|
21
|
+
count: number;
|
|
22
|
+
status: "admitted" | "exhausted";
|
|
23
|
+
final: boolean;
|
|
24
|
+
}>;
|
|
4
25
|
/** The public queued-job grammar. Payload is deliberately uninterpreted. */
|
|
5
26
|
export declare const WatchdogQueuedJobSchema: Schema.Struct<{
|
|
6
27
|
readonly state: Schema.Literal<"queued">;
|
|
@@ -9,25 +30,46 @@ export declare const WatchdogQueuedJobSchema: Schema.Struct<{
|
|
|
9
30
|
readonly lane: Schema.String;
|
|
10
31
|
readonly priority: Schema.Int;
|
|
11
32
|
readonly payload: Schema.Unknown;
|
|
33
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
12
34
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
13
35
|
readonly maxRecoveries: Schema.Int;
|
|
36
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
14
37
|
}>>;
|
|
15
38
|
}>;
|
|
16
39
|
/** Generic durable work waiting for execution. */
|
|
17
40
|
export type WatchdogQueuedJob = Schema.Schema.Type<typeof WatchdogQueuedJobSchema>;
|
|
41
|
+
/** Execution has decided; its atomic owner projection has not committed yet. */
|
|
42
|
+
export declare const WatchdogCompletingJobSchema: Schema.Struct<{
|
|
43
|
+
readonly state: Schema.Literal<"completing">;
|
|
44
|
+
readonly outcome: Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>;
|
|
45
|
+
readonly failureReason: Schema.optionalKey<Schema.String>;
|
|
46
|
+
readonly jobId: Schema.String;
|
|
47
|
+
readonly queue: Schema.String;
|
|
48
|
+
readonly lane: Schema.String;
|
|
49
|
+
readonly priority: Schema.Int;
|
|
50
|
+
readonly payload: Schema.Unknown;
|
|
51
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
52
|
+
readonly recovery: Schema.optional<Schema.Struct<{
|
|
53
|
+
readonly maxRecoveries: Schema.Int;
|
|
54
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
55
|
+
}>>;
|
|
56
|
+
}>;
|
|
57
|
+
/** Durable decision awaiting its owner projection, never executor replay. */
|
|
58
|
+
export type WatchdogCompletingJob = Schema.Schema.Type<typeof WatchdogCompletingJobSchema>;
|
|
18
59
|
/** The public terminal-job grammar. */
|
|
19
60
|
export declare const WatchdogSettledJobSchema: Schema.Struct<{
|
|
20
61
|
readonly state: Schema.Literal<"settled">;
|
|
21
62
|
readonly outcome: Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>;
|
|
22
|
-
/** Why a job failed or exhausted disappearance recovery, when known. */
|
|
23
63
|
readonly failureReason: Schema.optionalKey<Schema.String>;
|
|
24
64
|
readonly jobId: Schema.String;
|
|
25
65
|
readonly queue: Schema.String;
|
|
26
66
|
readonly lane: Schema.String;
|
|
27
67
|
readonly priority: Schema.Int;
|
|
28
68
|
readonly payload: Schema.Unknown;
|
|
69
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
29
70
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
30
71
|
readonly maxRecoveries: Schema.Int;
|
|
72
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
31
73
|
}>>;
|
|
32
74
|
}>;
|
|
33
75
|
/** Generic durable work with a terminal outcome. */
|
|
@@ -45,8 +87,10 @@ export declare const WatchdogRunningJobSchema: Schema.Struct<{
|
|
|
45
87
|
readonly lane: Schema.String;
|
|
46
88
|
readonly priority: Schema.Int;
|
|
47
89
|
readonly payload: Schema.Unknown;
|
|
90
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
48
91
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
49
92
|
readonly maxRecoveries: Schema.Int;
|
|
93
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
50
94
|
}>>;
|
|
51
95
|
}>;
|
|
52
96
|
/** Generic durable work held by Watchdog's private claim. */
|
|
@@ -59,8 +103,10 @@ export declare const WatchdogStoredJobSchema: Schema.Union<readonly [Schema.Stru
|
|
|
59
103
|
readonly lane: Schema.String;
|
|
60
104
|
readonly priority: Schema.Int;
|
|
61
105
|
readonly payload: Schema.Unknown;
|
|
106
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
62
107
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
63
108
|
readonly maxRecoveries: Schema.Int;
|
|
109
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
64
110
|
}>>;
|
|
65
111
|
}>, Schema.Struct<{
|
|
66
112
|
readonly state: Schema.Literal<"running">;
|
|
@@ -69,27 +115,46 @@ export declare const WatchdogStoredJobSchema: Schema.Union<readonly [Schema.Stru
|
|
|
69
115
|
readonly lane: Schema.String;
|
|
70
116
|
readonly priority: Schema.Int;
|
|
71
117
|
readonly payload: Schema.Unknown;
|
|
118
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
119
|
+
readonly recovery: Schema.optional<Schema.Struct<{
|
|
120
|
+
readonly maxRecoveries: Schema.Int;
|
|
121
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
122
|
+
}>>;
|
|
123
|
+
}>, Schema.Struct<{
|
|
124
|
+
readonly state: Schema.Literal<"completing">;
|
|
125
|
+
readonly outcome: Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>;
|
|
126
|
+
readonly failureReason: Schema.optionalKey<Schema.String>;
|
|
127
|
+
readonly jobId: Schema.String;
|
|
128
|
+
readonly queue: Schema.String;
|
|
129
|
+
readonly lane: Schema.String;
|
|
130
|
+
readonly priority: Schema.Int;
|
|
131
|
+
readonly payload: Schema.Unknown;
|
|
132
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
72
133
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
73
134
|
readonly maxRecoveries: Schema.Int;
|
|
135
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
74
136
|
}>>;
|
|
75
137
|
}>, Schema.Struct<{
|
|
76
138
|
readonly state: Schema.Literal<"settled">;
|
|
77
139
|
readonly outcome: Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>;
|
|
78
|
-
/** Why a job failed or exhausted disappearance recovery, when known. */
|
|
79
140
|
readonly failureReason: Schema.optionalKey<Schema.String>;
|
|
80
141
|
readonly jobId: Schema.String;
|
|
81
142
|
readonly queue: Schema.String;
|
|
82
143
|
readonly lane: Schema.String;
|
|
83
144
|
readonly priority: Schema.Int;
|
|
84
145
|
readonly payload: Schema.Unknown;
|
|
146
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
85
147
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
86
148
|
readonly maxRecoveries: Schema.Int;
|
|
149
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
87
150
|
}>>;
|
|
88
151
|
}>]>;
|
|
89
152
|
/** Any publicly readable lifecycle state for a generic job. */
|
|
90
153
|
export type WatchdogStoredJob = Schema.Schema.Type<typeof WatchdogStoredJobSchema>;
|
|
91
154
|
/** Terminal decisions returned by the execution adapter. */
|
|
92
155
|
export declare const WatchdogExecutionDecisionSchema: Schema.Union<readonly [Schema.Struct<{
|
|
156
|
+
readonly kind: Schema.Literal<"retry">;
|
|
157
|
+
}>, Schema.Struct<{
|
|
93
158
|
readonly kind: Schema.Literal<"completed">;
|
|
94
159
|
}>, Schema.Struct<{
|
|
95
160
|
readonly kind: Schema.Literal<"failed">;
|
|
@@ -113,17 +178,29 @@ export type WatchdogCancellationIntent = Schema.Schema.Type<typeof WatchdogCance
|
|
|
113
178
|
export type WatchdogCancellationDecision = Readonly<{
|
|
114
179
|
status: "recorded" | "replayed";
|
|
115
180
|
intent: WatchdogCancellationIntent;
|
|
181
|
+
}> | Readonly<{
|
|
182
|
+
status: "decided";
|
|
183
|
+
outcome: WatchdogSettledJob["outcome"];
|
|
116
184
|
}> | Readonly<{
|
|
117
185
|
status: "conflict" | "missing";
|
|
118
186
|
}>;
|
|
119
187
|
/** Narrow synchronous seam for owner-atomic queue inspection, ingest, and cancellation. */
|
|
120
188
|
export type WatchdogTransactionalProjection = Readonly<{
|
|
189
|
+
/** Read the same durable readiness for synchronous native host wake composition. */
|
|
190
|
+
wakeState: (now: number) => WatchdogWakeState;
|
|
191
|
+
/** Reads a job's exact lifecycle state and durable deadline in the owner transaction. */
|
|
192
|
+
readJob: (jobId: string) => Readonly<{
|
|
193
|
+
state: WatchdogStoredJob["state"];
|
|
194
|
+
notBefore: number;
|
|
195
|
+
}> | undefined;
|
|
121
196
|
/** Reads the bounded current queue head inside an enclosing owner transaction. */
|
|
122
197
|
readQueueHead: (queue: string) => WatchdogQueueHead;
|
|
123
198
|
/** Reads the most recently settled job of one queue, if any. */
|
|
124
199
|
readLatestSettledJob: (queue: string) => WatchdogSettledJob | undefined;
|
|
125
200
|
/** Applies the same durable queued-row mutation as store.ingest. */
|
|
126
201
|
enqueueAcceptedJob: (job: WatchdogQueuedJob) => void;
|
|
202
|
+
/** Expedite existing queued work without changing its charged attempt count. */
|
|
203
|
+
expediteQueuedJob: (jobId: string, now: number) => boolean;
|
|
127
204
|
/** Records or replays cancellation without interpreting the payload. */
|
|
128
205
|
recordCancellationIntent: (intent: WatchdogCancellationIntent) => WatchdogCancellationDecision;
|
|
129
206
|
}>;
|
|
@@ -137,8 +214,10 @@ export declare const WatchdogReadResultSchema: Schema.Union<readonly [Schema.Str
|
|
|
137
214
|
readonly lane: Schema.String;
|
|
138
215
|
readonly priority: Schema.Int;
|
|
139
216
|
readonly payload: Schema.Unknown;
|
|
217
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
140
218
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
141
219
|
readonly maxRecoveries: Schema.Int;
|
|
220
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
142
221
|
}>>;
|
|
143
222
|
}>, Schema.Struct<{
|
|
144
223
|
readonly state: Schema.Literal<"running">;
|
|
@@ -147,21 +226,38 @@ export declare const WatchdogReadResultSchema: Schema.Union<readonly [Schema.Str
|
|
|
147
226
|
readonly lane: Schema.String;
|
|
148
227
|
readonly priority: Schema.Int;
|
|
149
228
|
readonly payload: Schema.Unknown;
|
|
229
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
230
|
+
readonly recovery: Schema.optional<Schema.Struct<{
|
|
231
|
+
readonly maxRecoveries: Schema.Int;
|
|
232
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
233
|
+
}>>;
|
|
234
|
+
}>, Schema.Struct<{
|
|
235
|
+
readonly state: Schema.Literal<"completing">;
|
|
236
|
+
readonly outcome: Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>;
|
|
237
|
+
readonly failureReason: Schema.optionalKey<Schema.String>;
|
|
238
|
+
readonly jobId: Schema.String;
|
|
239
|
+
readonly queue: Schema.String;
|
|
240
|
+
readonly lane: Schema.String;
|
|
241
|
+
readonly priority: Schema.Int;
|
|
242
|
+
readonly payload: Schema.Unknown;
|
|
243
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
150
244
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
151
245
|
readonly maxRecoveries: Schema.Int;
|
|
246
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
152
247
|
}>>;
|
|
153
248
|
}>, Schema.Struct<{
|
|
154
249
|
readonly state: Schema.Literal<"settled">;
|
|
155
250
|
readonly outcome: Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>;
|
|
156
|
-
/** Why a job failed or exhausted disappearance recovery, when known. */
|
|
157
251
|
readonly failureReason: Schema.optionalKey<Schema.String>;
|
|
158
252
|
readonly jobId: Schema.String;
|
|
159
253
|
readonly queue: Schema.String;
|
|
160
254
|
readonly lane: Schema.String;
|
|
161
255
|
readonly priority: Schema.Int;
|
|
162
256
|
readonly payload: Schema.Unknown;
|
|
257
|
+
readonly notBefore: Schema.optionalKey<Schema.Int>;
|
|
163
258
|
readonly recovery: Schema.optional<Schema.Struct<{
|
|
164
259
|
readonly maxRecoveries: Schema.Int;
|
|
260
|
+
readonly windowMs: Schema.optionalKey<Schema.Int>;
|
|
165
261
|
}>>;
|
|
166
262
|
}>]>;
|
|
167
263
|
readonly cancellation: Schema.Union<readonly [Schema.Struct<{
|
|
@@ -187,7 +283,7 @@ export declare const WatchdogQueueHeadSchema: Schema.Union<readonly [Schema.Stru
|
|
|
187
283
|
readonly jobId: Schema.String;
|
|
188
284
|
readonly cancellationRequested: Schema.Boolean;
|
|
189
285
|
}>]>;
|
|
190
|
-
/**
|
|
286
|
+
/** Queue occupancy: completing work remains running here until its projection commits. Read the job for its exact lifecycle state. */
|
|
191
287
|
export type WatchdogQueueHead = Schema.Schema.Type<typeof WatchdogQueueHeadSchema>;
|
|
192
288
|
/** One queue's state after a durable transition. */
|
|
193
289
|
export type WatchdogQueueTransition = Readonly<{
|
|
@@ -214,6 +310,9 @@ export declare const WatchdogIngestResultSchema: Schema.Union<readonly [Schema.S
|
|
|
214
310
|
export type WatchdogIngestResult = Schema.Schema.Type<typeof WatchdogIngestResultSchema>;
|
|
215
311
|
/** Results returned by durable cancellation recording. */
|
|
216
312
|
export declare const WatchdogCancellationResultSchema: Schema.Union<readonly [Schema.Struct<{
|
|
313
|
+
readonly status: Schema.Literal<"decided">;
|
|
314
|
+
readonly outcome: Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>;
|
|
315
|
+
}>, Schema.Struct<{
|
|
217
316
|
readonly status: Schema.Literals<readonly ["recorded", "replayed"]>;
|
|
218
317
|
}>, Schema.Struct<{
|
|
219
318
|
readonly status: Schema.Literals<readonly ["conflict", "missing"]>;
|
|
@@ -222,8 +321,14 @@ export declare const WatchdogCancellationResultSchema: Schema.Union<readonly [Sc
|
|
|
222
321
|
export type WatchdogCancellationResult = Schema.Schema.Type<typeof WatchdogCancellationResultSchema>;
|
|
223
322
|
/** Effect-native host executor for one generic running job. */
|
|
224
323
|
export type EffectWatchdogExecutor = Readonly<{
|
|
324
|
+
/** Code-owned proof that consumption and every effect commit in one owner transaction. */
|
|
325
|
+
replaySafe?: (job: WatchdogRunningJob) => boolean;
|
|
326
|
+
/** Resolve the current delivery policy from the existing owner, outside a storage transaction. */
|
|
327
|
+
retryPolicy?: (job: WatchdogRunningJob) => Effect.Effect<WatchdogRetryPolicy | undefined, WatchdogError>;
|
|
225
328
|
/** Runs one generic claimed job until a terminal decision. */
|
|
226
|
-
execute: (job: WatchdogRunningJob) => Effect.Effect<WatchdogExecutionDecision, WatchdogError>;
|
|
329
|
+
execute: (job: WatchdogRunningJob, attempt: WatchdogAttempt | undefined) => Effect.Effect<WatchdogExecutionDecision, WatchdogError>;
|
|
330
|
+
/** After interruption and scope release, prove foreign resources finished cleanup. */
|
|
331
|
+
isDrained?: (job: WatchdogRunningJob) => Effect.Effect<boolean, WatchdogError>;
|
|
227
332
|
}>;
|
|
228
333
|
/** Scalar grammar shared by native SQLite adapters and their storage owners. */
|
|
229
334
|
export declare const watchdogSqliteValueSchema: Schema.Union<readonly [Schema.instanceOf<ArrayBuffer, unknown>, Schema.String, Schema.Number, Schema.Null]>;
|
|
@@ -277,7 +382,12 @@ export type WatchdogLivenessUnknown = Readonly<{
|
|
|
277
382
|
}>;
|
|
278
383
|
/** Result of one bounded deep tick. */
|
|
279
384
|
export type WatchdogTickResult = Readonly<{
|
|
385
|
+
status: "requeued";
|
|
386
|
+
jobId: string;
|
|
387
|
+
}> | Readonly<{
|
|
280
388
|
status: "busy";
|
|
389
|
+
}> | Readonly<{
|
|
390
|
+
status: "stopped";
|
|
281
391
|
}> | Readonly<{
|
|
282
392
|
status: "no_ready";
|
|
283
393
|
}> | WatchdogLivenessUnknown | Readonly<{
|
|
@@ -300,8 +410,8 @@ export type WatchdogLaneFailureResult = Readonly<{
|
|
|
300
410
|
}>;
|
|
301
411
|
/** Effect-native Watchdog runtime surface. */
|
|
302
412
|
export type WatchdogRuntimeEffect = Readonly<{
|
|
303
|
-
/**
|
|
304
|
-
|
|
413
|
+
/** Read readiness and the next deadline from the same durable work rows. */
|
|
414
|
+
wakeState: (now: number) => Effect.Effect<WatchdogWakeState, WatchdogError>;
|
|
305
415
|
/** Persists one queued job. */
|
|
306
416
|
ingest: (job: WatchdogQueuedJob) => Effect.Effect<WatchdogIngestResult, WatchdogError>;
|
|
307
417
|
/** Reads one durable job. */
|
|
@@ -309,7 +419,7 @@ export type WatchdogRuntimeEffect = Readonly<{
|
|
|
309
419
|
/** Reads the bounded current head of one generic queue. */
|
|
310
420
|
readQueueHead: (queue: string) => Effect.Effect<WatchdogQueueHead, WatchdogError>;
|
|
311
421
|
/** Reads a bounded terminal prefix of the serial partition for owner recovery. */
|
|
312
|
-
readSettled: (limit: number, afterSequence?: number) => Effect.Effect<readonly WatchdogSettledRecord[], WatchdogError>;
|
|
422
|
+
readSettled: (limit: number, afterSequence?: number, lane?: string) => Effect.Effect<readonly WatchdogSettledRecord[], WatchdogError>;
|
|
313
423
|
/** Runs one bounded generic lifecycle tick. */
|
|
314
424
|
tick: () => Effect.Effect<WatchdogTickResult, WatchdogError>;
|
|
315
425
|
/** Records cancellation and signals live execution. */
|
|
@@ -318,6 +428,8 @@ export type WatchdogRuntimeEffect = Readonly<{
|
|
|
318
428
|
* Refused claims are invalidated; both late success and failure reports are rejected.
|
|
319
429
|
*/
|
|
320
430
|
failLane: (input: WatchdogLaneFailure) => Effect.Effect<WatchdogLaneFailureResult, WatchdogError>;
|
|
431
|
+
/** Stop dispatch, interrupt owned execution, and hand off only a proven drained claim. */
|
|
432
|
+
close: Effect.Effect<void, WatchdogError>;
|
|
321
433
|
/** Removes queued jobs in one queue. */
|
|
322
434
|
purge: (queue: string) => Effect.Effect<void, WatchdogError>;
|
|
323
435
|
/** Joins accepted enqueue and cancellation to an existing owner transaction. */
|
package/dist/effect.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import * as Effect from "effect/Effect";
|
|
2
|
+
import * as Exit from "effect/Exit";
|
|
2
3
|
import * as Schema from "effect/Schema";
|
|
3
4
|
import { bindWatchdogEngine } from "./internal/runtime.js";
|
|
4
5
|
import { openSqliteState } from "./internal/sqlite-store.js";
|
|
@@ -11,6 +12,7 @@ const Priority = Schema.Int.check(Schema.isGreaterThanOrEqualTo(Number.MIN_SAFE_
|
|
|
11
12
|
const RecoveryCount = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER));
|
|
12
13
|
const RecoveryPolicy = Schema.Struct({
|
|
13
14
|
maxRecoveries: RecoveryCount,
|
|
15
|
+
windowMs: Schema.optionalKey(Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER))),
|
|
14
16
|
});
|
|
15
17
|
const PublicJobFields = {
|
|
16
18
|
jobId: BoundedString,
|
|
@@ -18,6 +20,8 @@ const PublicJobFields = {
|
|
|
18
20
|
lane: laneSchema,
|
|
19
21
|
priority: Priority,
|
|
20
22
|
payload: Schema.Unknown,
|
|
23
|
+
/** Earliest initial dispatch; replay never changes a retained deadline. */
|
|
24
|
+
notBefore: Schema.optionalKey(RecoveryCount),
|
|
21
25
|
recovery: Schema.optional(RecoveryPolicy),
|
|
22
26
|
};
|
|
23
27
|
/** The public queued-job grammar. Payload is deliberately uninterpreted. */
|
|
@@ -25,19 +29,27 @@ export const WatchdogQueuedJobSchema = Schema.Struct({
|
|
|
25
29
|
...PublicJobFields,
|
|
26
30
|
state: Schema.Literal("queued"),
|
|
27
31
|
});
|
|
32
|
+
const Outcome = Schema.Literals([
|
|
33
|
+
"completed",
|
|
34
|
+
"failed",
|
|
35
|
+
"cancelled",
|
|
36
|
+
"outcome_unknown",
|
|
37
|
+
"interrupted",
|
|
38
|
+
]);
|
|
39
|
+
const DecidedJobFields = {
|
|
40
|
+
...PublicJobFields,
|
|
41
|
+
outcome: Outcome,
|
|
42
|
+
failureReason: Schema.optionalKey(FailureReason),
|
|
43
|
+
};
|
|
44
|
+
/** Execution has decided; its atomic owner projection has not committed yet. */
|
|
45
|
+
export const WatchdogCompletingJobSchema = Schema.Struct({
|
|
46
|
+
...DecidedJobFields,
|
|
47
|
+
state: Schema.Literal("completing"),
|
|
48
|
+
});
|
|
28
49
|
/** The public terminal-job grammar. */
|
|
29
50
|
export const WatchdogSettledJobSchema = Schema.Struct({
|
|
30
|
-
...
|
|
51
|
+
...DecidedJobFields,
|
|
31
52
|
state: Schema.Literal("settled"),
|
|
32
|
-
outcome: Schema.Literals([
|
|
33
|
-
"completed",
|
|
34
|
-
"failed",
|
|
35
|
-
"cancelled",
|
|
36
|
-
"outcome_unknown",
|
|
37
|
-
"interrupted",
|
|
38
|
-
]),
|
|
39
|
-
/** Why a job failed or exhausted disappearance recovery, when known. */
|
|
40
|
-
failureReason: Schema.optionalKey(FailureReason),
|
|
41
53
|
});
|
|
42
54
|
/** The public running-job grammar. Private claim data is deliberately absent. */
|
|
43
55
|
export const WatchdogRunningJobSchema = Schema.Struct({
|
|
@@ -48,10 +60,12 @@ export const WatchdogRunningJobSchema = Schema.Struct({
|
|
|
48
60
|
export const WatchdogStoredJobSchema = Schema.Union([
|
|
49
61
|
WatchdogQueuedJobSchema,
|
|
50
62
|
WatchdogRunningJobSchema,
|
|
63
|
+
WatchdogCompletingJobSchema,
|
|
51
64
|
WatchdogSettledJobSchema,
|
|
52
65
|
]);
|
|
53
66
|
/** Terminal decisions returned by the execution adapter. */
|
|
54
67
|
export const WatchdogExecutionDecisionSchema = Schema.Union([
|
|
68
|
+
Schema.Struct({ kind: Schema.Literal("retry") }),
|
|
55
69
|
Schema.Struct({ kind: Schema.Literal("completed") }),
|
|
56
70
|
Schema.Struct({
|
|
57
71
|
kind: Schema.Literal("failed"),
|
|
@@ -98,6 +112,7 @@ export const WatchdogIngestResultSchema = Schema.Union([
|
|
|
98
112
|
]);
|
|
99
113
|
/** Results returned by durable cancellation recording. */
|
|
100
114
|
export const WatchdogCancellationResultSchema = Schema.Union([
|
|
115
|
+
Schema.Struct({ status: Schema.Literal("decided"), outcome: Outcome }),
|
|
101
116
|
Schema.Struct({ status: Schema.Literals(["recorded", "replayed"]) }),
|
|
102
117
|
Schema.Struct({ status: Schema.Literals(["conflict", "missing"]) }),
|
|
103
118
|
]);
|
|
@@ -125,6 +140,12 @@ export function makeRuntime(options) {
|
|
|
125
140
|
return Object.freeze({
|
|
126
141
|
...base,
|
|
127
142
|
tickConcurrent: concurrent.tick,
|
|
143
|
+
close: Effect.all([Effect.exit(serial.close), Effect.exit(concurrent.close)], { concurrency: 2 }).pipe(Effect.flatMap((exits) => {
|
|
144
|
+
const failed = exits.find(Exit.isFailure);
|
|
145
|
+
return failed === undefined
|
|
146
|
+
? Effect.void
|
|
147
|
+
: Effect.failCause(failed.cause);
|
|
148
|
+
})),
|
|
128
149
|
failLane: (input) => input.lane === lane
|
|
129
150
|
? concurrent.failLane(input)
|
|
130
151
|
: serial.failLane(input),
|
|
@@ -135,6 +156,8 @@ export function makeRuntime(options) {
|
|
|
135
156
|
: serial.cancel(intent))),
|
|
136
157
|
});
|
|
137
158
|
},
|
|
138
|
-
catch: (error) =>
|
|
159
|
+
catch: (error) => error instanceof WatchdogError
|
|
160
|
+
? error
|
|
161
|
+
: new WatchdogError("store_unavailable", error),
|
|
139
162
|
});
|
|
140
163
|
}
|