@fungi.computer/watchdog 0.0.0-stage → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +334 -2
- package/dist/d1.d.ts +36 -0
- package/dist/d1.js +41 -0
- package/dist/effect.d.ts +336 -0
- package/dist/effect.js +140 -0
- package/dist/index.d.ts +65 -0
- package/dist/index.js +41 -0
- package/dist/internal/async-runtime.d.ts +17 -0
- package/dist/internal/async-runtime.js +39 -0
- package/dist/internal/async-sql.d.ts +15 -0
- package/dist/internal/async-sql.js +66 -0
- package/dist/internal/async-store.d.ts +58 -0
- package/dist/internal/async-store.js +235 -0
- package/dist/internal/identities.d.ts +3 -0
- package/dist/internal/identities.js +8 -0
- package/dist/internal/lane.d.ts +3 -0
- package/dist/internal/lane.js +6 -0
- package/dist/internal/promise-watchdog-host.d.ts +4 -0
- package/dist/internal/promise-watchdog-host.js +25 -0
- package/dist/internal/runtime.d.ts +3 -0
- package/dist/internal/runtime.js +199 -0
- package/dist/internal/sqlite-store.d.ts +107 -0
- package/dist/internal/sqlite-store.js +478 -0
- package/dist/internal/watchdog-error.d.ts +12 -0
- package/dist/internal/watchdog-error.js +17 -0
- package/dist/node-sqlite.d.ts +17 -0
- package/dist/node-sqlite.js +63 -0
- package/dist/postgres.d.ts +22 -0
- package/dist/postgres.js +31 -0
- package/docs/api/README.md +17 -0
- package/docs/api/d1.md +162 -0
- package/docs/api/effect.md +875 -0
- package/docs/api/index.md +421 -0
- package/docs/api/node-sqlite.md +57 -0
- package/docs/api/postgres.md +102 -0
- package/package.json +78 -3
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import * as Effect from "effect/Effect";
|
|
2
|
+
import * as Schema from "effect/Schema";
|
|
3
|
+
import type { WatchdogLaneFailure, WatchdogCancellationIntent, WatchdogCancellationResult, WatchdogExecutionDecision, WatchdogIngestResult, WatchdogQueuedJob, WatchdogQueueHead, WatchdogQueueProjector, WatchdogReadResult, WatchdogRunningJob, WatchdogSettledJob, WatchdogSettledRecord, WatchdogSqliteOwner, WatchdogSqliteRow, WatchdogStoredJob, WatchdogTransactionalProjection } from "../effect.js";
|
|
4
|
+
import { WatchdogError } from "./watchdog-error.js";
|
|
5
|
+
export type { WatchdogSqliteRow } from "../effect.js";
|
|
6
|
+
export declare const laneFailureBatchSize = 256;
|
|
7
|
+
/** One strict owner input grammar shared by the storage implementations. */
|
|
8
|
+
export declare function parseLaneFailure(input: WatchdogLaneFailure): WatchdogLaneFailure;
|
|
9
|
+
declare const JobRowSchema: Schema.Struct<{
|
|
10
|
+
readonly sequence: Schema.Int;
|
|
11
|
+
readonly job_id: Schema.String;
|
|
12
|
+
readonly queue_name: Schema.String;
|
|
13
|
+
readonly lane: Schema.String;
|
|
14
|
+
readonly priority: Schema.Int;
|
|
15
|
+
readonly payload_json: Schema.String;
|
|
16
|
+
readonly recovery_max: Schema.NullOr<Schema.Int>;
|
|
17
|
+
readonly state: Schema.Literals<readonly ["queued", "running", "settled"]>;
|
|
18
|
+
readonly outcome: Schema.NullOr<Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>>;
|
|
19
|
+
readonly claim_token: Schema.NullOr<Schema.String>;
|
|
20
|
+
readonly recovery_count: Schema.Int;
|
|
21
|
+
readonly cancellation_id: Schema.NullOr<Schema.String>;
|
|
22
|
+
readonly failure_reason: Schema.NullOr<Schema.String>;
|
|
23
|
+
}>;
|
|
24
|
+
/** Decoded relational row used inside the SQLite adapter family. */
|
|
25
|
+
export type WatchdogSqliteJobRow = Schema.Schema.Type<typeof JobRowSchema>;
|
|
26
|
+
/** Decoded row and domain job shared with the async SQLite adapter. */
|
|
27
|
+
export type ParsedWatchdogSqliteJob = Readonly<{
|
|
28
|
+
row: WatchdogSqliteJobRow;
|
|
29
|
+
job: WatchdogStoredJob;
|
|
30
|
+
}>;
|
|
31
|
+
/** Package-private result of selecting or claiming one durable row. */
|
|
32
|
+
export type WatchdogSelection = Readonly<{
|
|
33
|
+
status: "no_ready";
|
|
34
|
+
}> | Readonly<{
|
|
35
|
+
status: "claim_unavailable";
|
|
36
|
+
}> | Readonly<{
|
|
37
|
+
status: "cancelled";
|
|
38
|
+
job: WatchdogSettledJob;
|
|
39
|
+
}> | Readonly<{
|
|
40
|
+
status: "claimed";
|
|
41
|
+
job: WatchdogRunningJob;
|
|
42
|
+
claimToken: string;
|
|
43
|
+
}>;
|
|
44
|
+
/** One private execution partition; no payload or product meaning. */
|
|
45
|
+
export type LanePartition = Readonly<{
|
|
46
|
+
lane: string;
|
|
47
|
+
include: boolean;
|
|
48
|
+
}>;
|
|
49
|
+
/** Package-private storage operations consumed by Watchdog's Effect engine. */
|
|
50
|
+
export type WatchdogStore = Readonly<{
|
|
51
|
+
hasPendingWork: Effect.Effect<boolean, WatchdogError>;
|
|
52
|
+
ingest: (job: WatchdogQueuedJob) => Effect.Effect<WatchdogIngestResult, WatchdogError>;
|
|
53
|
+
read: (jobId: string) => Effect.Effect<(Extract<WatchdogReadResult, {
|
|
54
|
+
status: "found";
|
|
55
|
+
}> & Readonly<{
|
|
56
|
+
claimToken: string | null;
|
|
57
|
+
recoveryCount: number;
|
|
58
|
+
}>) | Extract<WatchdogReadResult, {
|
|
59
|
+
status: "missing";
|
|
60
|
+
}>, WatchdogError>;
|
|
61
|
+
readQueueHead: (queue: string) => Effect.Effect<WatchdogQueueHead, WatchdogError>;
|
|
62
|
+
readSettled: (limit: number, afterSequence?: number) => Effect.Effect<readonly WatchdogSettledRecord[], WatchdogError>;
|
|
63
|
+
selectAndClaim: (claimToken: string) => Effect.Effect<WatchdogSelection, WatchdogError>;
|
|
64
|
+
settleExactClaim: (input: Readonly<{
|
|
65
|
+
jobId: string;
|
|
66
|
+
claimToken: string;
|
|
67
|
+
decision: WatchdogExecutionDecision;
|
|
68
|
+
}>) => Effect.Effect<Readonly<{
|
|
69
|
+
status: "settled" | "replayed";
|
|
70
|
+
job: WatchdogSettledJob;
|
|
71
|
+
}> | Readonly<{
|
|
72
|
+
status: "conflict" | "missing";
|
|
73
|
+
}>, WatchdogError>;
|
|
74
|
+
recordCancellation: (input: WatchdogCancellationIntent) => Effect.Effect<WatchdogCancellationResult, WatchdogError>;
|
|
75
|
+
failLane: (input: WatchdogLaneFailure) => Effect.Effect<Readonly<{
|
|
76
|
+
jobs: readonly WatchdogSettledJob[];
|
|
77
|
+
more: boolean;
|
|
78
|
+
}>, WatchdogError>;
|
|
79
|
+
purgeQueued: (queue: string) => Effect.Effect<void, WatchdogError>;
|
|
80
|
+
nextRunningClaim: () => Effect.Effect<Readonly<{
|
|
81
|
+
job: WatchdogRunningJob;
|
|
82
|
+
claimToken: string;
|
|
83
|
+
recoveryCount: number;
|
|
84
|
+
}> | undefined, WatchdogError>;
|
|
85
|
+
reconcileDisappearedClaim: (input: Readonly<{
|
|
86
|
+
job: WatchdogRunningJob;
|
|
87
|
+
claimToken: string;
|
|
88
|
+
recoveryCount: number;
|
|
89
|
+
disposition: "requeue" | "interrupted";
|
|
90
|
+
}>) => Effect.Effect<Readonly<{
|
|
91
|
+
status: "requeued";
|
|
92
|
+
}> | Readonly<{
|
|
93
|
+
status: "settled";
|
|
94
|
+
job: WatchdogSettledJob;
|
|
95
|
+
}> | Readonly<{
|
|
96
|
+
status: "conflict" | "missing";
|
|
97
|
+
}>, WatchdogError>;
|
|
98
|
+
}>;
|
|
99
|
+
/** Package-private SQLite state plus the one owner-transaction projection. */
|
|
100
|
+
export type WatchdogSqliteState = Readonly<{
|
|
101
|
+
store: WatchdogStore;
|
|
102
|
+
partition: (partition: LanePartition) => WatchdogStore;
|
|
103
|
+
transactional: WatchdogTransactionalProjection;
|
|
104
|
+
}>;
|
|
105
|
+
export declare function parseJobRow(raw: WatchdogSqliteRow): ParsedWatchdogSqliteJob;
|
|
106
|
+
/** Construct Watchdog's private owner-local SQLite state. */
|
|
107
|
+
export declare function openSqliteState(owner: WatchdogSqliteOwner, project?: WatchdogQueueProjector, partition?: LanePartition): WatchdogSqliteState;
|
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
import * as Cause from "effect/Cause";
|
|
2
|
+
import * as Effect from "effect/Effect";
|
|
3
|
+
import * as Schema from "effect/Schema";
|
|
4
|
+
import { WatchdogError } from "./watchdog-error.js";
|
|
5
|
+
import { laneSchema } from "./lane.js";
|
|
6
|
+
const Identifier = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(256));
|
|
7
|
+
const Priority = Schema.Int.check(Schema.isGreaterThanOrEqualTo(Number.MIN_SAFE_INTEGER), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER));
|
|
8
|
+
const Count = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER));
|
|
9
|
+
const FailureReason = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(200));
|
|
10
|
+
const LaneFailureSchema = Schema.Struct({
|
|
11
|
+
lane: laneSchema,
|
|
12
|
+
reason: FailureReason,
|
|
13
|
+
});
|
|
14
|
+
// A transition batch, not a lifetime limit on lane work; matches terminal reads.
|
|
15
|
+
export const laneFailureBatchSize = 256;
|
|
16
|
+
/** One strict owner input grammar shared by the storage implementations. */
|
|
17
|
+
export function parseLaneFailure(input) {
|
|
18
|
+
try {
|
|
19
|
+
return Schema.decodeUnknownSync(LaneFailureSchema, {
|
|
20
|
+
onExcessProperty: "error",
|
|
21
|
+
})(input);
|
|
22
|
+
}
|
|
23
|
+
catch (cause) {
|
|
24
|
+
throw new WatchdogError("store_invalid", cause);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const JobRowSchema = Schema.Struct({
|
|
28
|
+
sequence: Schema.Int,
|
|
29
|
+
job_id: Identifier,
|
|
30
|
+
queue_name: Identifier,
|
|
31
|
+
lane: laneSchema,
|
|
32
|
+
priority: Priority,
|
|
33
|
+
payload_json: Schema.String,
|
|
34
|
+
recovery_max: Schema.NullOr(Count),
|
|
35
|
+
state: Schema.Literals(["queued", "running", "settled"]),
|
|
36
|
+
outcome: Schema.NullOr(Schema.Literals([
|
|
37
|
+
"completed",
|
|
38
|
+
"failed",
|
|
39
|
+
"cancelled",
|
|
40
|
+
"outcome_unknown",
|
|
41
|
+
"interrupted",
|
|
42
|
+
])),
|
|
43
|
+
claim_token: Schema.NullOr(Identifier),
|
|
44
|
+
recovery_count: Count,
|
|
45
|
+
cancellation_id: Schema.NullOr(Identifier),
|
|
46
|
+
failure_reason: Schema.NullOr(FailureReason),
|
|
47
|
+
});
|
|
48
|
+
const PayloadTextSchema = Schema.fromJsonString(Schema.Unknown);
|
|
49
|
+
const decodeJobRow = Schema.decodeUnknownExit(JobRowSchema, {
|
|
50
|
+
onExcessProperty: "error",
|
|
51
|
+
});
|
|
52
|
+
const decodePayload = Schema.decodeUnknownExit(PayloadTextSchema);
|
|
53
|
+
const encodePayload = Schema.encodeUnknownExit(PayloadTextSchema);
|
|
54
|
+
function laneQuery(partition) {
|
|
55
|
+
return partition === undefined
|
|
56
|
+
? { predicate: "1 = 1", values: [] }
|
|
57
|
+
: {
|
|
58
|
+
predicate: partition.include ? "lane = ?" : "lane <> ?",
|
|
59
|
+
values: [partition.lane],
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
const columns = `sequence, job_id, queue_name, lane, priority, payload_json,
|
|
63
|
+
recovery_max, state, outcome, claim_token, recovery_count,
|
|
64
|
+
cancellation_id, failure_reason`;
|
|
65
|
+
function unavailable(cause) {
|
|
66
|
+
return new WatchdogError("store_unavailable", cause);
|
|
67
|
+
}
|
|
68
|
+
function storeEffect(operation) {
|
|
69
|
+
return Effect.try({
|
|
70
|
+
try: operation,
|
|
71
|
+
// The SQL error stays on the failure; a Watchdog failure raised inside
|
|
72
|
+
// (an invariant or a decode) already carries its own.
|
|
73
|
+
catch: (error) => error instanceof WatchdogError ? error : unavailable(error),
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
function encodeOpaquePayload(payload) {
|
|
77
|
+
const encoded = encodePayload(payload);
|
|
78
|
+
if (encoded._tag === "Failure")
|
|
79
|
+
throw unavailable(Cause.squash(encoded.cause));
|
|
80
|
+
return encoded.value;
|
|
81
|
+
}
|
|
82
|
+
export function parseJobRow(raw) {
|
|
83
|
+
const decoded = decodeJobRow(raw);
|
|
84
|
+
if (decoded._tag === "Failure")
|
|
85
|
+
throw unavailable(Cause.squash(decoded.cause));
|
|
86
|
+
const row = decoded.value;
|
|
87
|
+
const payload = decodePayload(row.payload_json);
|
|
88
|
+
if (payload._tag === "Failure")
|
|
89
|
+
throw unavailable(Cause.squash(payload.cause));
|
|
90
|
+
const common = row.recovery_max === null
|
|
91
|
+
? {
|
|
92
|
+
jobId: row.job_id,
|
|
93
|
+
queue: row.queue_name,
|
|
94
|
+
lane: row.lane,
|
|
95
|
+
priority: row.priority,
|
|
96
|
+
payload: payload.value,
|
|
97
|
+
}
|
|
98
|
+
: {
|
|
99
|
+
jobId: row.job_id,
|
|
100
|
+
queue: row.queue_name,
|
|
101
|
+
lane: row.lane,
|
|
102
|
+
priority: row.priority,
|
|
103
|
+
payload: payload.value,
|
|
104
|
+
recovery: { maxRecoveries: row.recovery_max },
|
|
105
|
+
};
|
|
106
|
+
if (row.state === "queued")
|
|
107
|
+
return { row, job: { ...common, state: "queued" } };
|
|
108
|
+
if (row.state === "running") {
|
|
109
|
+
if (row.claim_token === null)
|
|
110
|
+
throw unavailable();
|
|
111
|
+
return { row, job: { ...common, state: "running" } };
|
|
112
|
+
}
|
|
113
|
+
if (row.outcome === null)
|
|
114
|
+
throw unavailable();
|
|
115
|
+
const settled = {
|
|
116
|
+
...common,
|
|
117
|
+
state: "settled",
|
|
118
|
+
outcome: row.outcome,
|
|
119
|
+
};
|
|
120
|
+
return {
|
|
121
|
+
row,
|
|
122
|
+
job: (row.outcome === "failed" || row.outcome === "interrupted") &&
|
|
123
|
+
row.failure_reason !== null
|
|
124
|
+
? { ...settled, failureReason: row.failure_reason }
|
|
125
|
+
: settled,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
function readJob(owner, jobId) {
|
|
129
|
+
const raw = owner.sql
|
|
130
|
+
.exec(`SELECT ${columns}
|
|
131
|
+
FROM watchdog_jobs
|
|
132
|
+
WHERE job_id = ?`, jobId)
|
|
133
|
+
.toArray()[0];
|
|
134
|
+
return raw === undefined ? undefined : parseJobRow(raw);
|
|
135
|
+
}
|
|
136
|
+
function readSettled(owner, limit, afterSequence, partition) {
|
|
137
|
+
const lane = laneQuery(partition);
|
|
138
|
+
const rows = owner.sql
|
|
139
|
+
.exec(`SELECT ${columns} FROM watchdog_jobs NOT INDEXED WHERE state = 'settled' AND ${lane.predicate} AND sequence > ?
|
|
140
|
+
AND sequence < COALESCE(
|
|
141
|
+
(SELECT MIN(sequence) FROM watchdog_jobs INDEXED BY idx_watchdog_ready
|
|
142
|
+
WHERE state IN ('queued', 'running') AND ${lane.predicate}),
|
|
143
|
+
9223372036854775807)
|
|
144
|
+
ORDER BY sequence ASC LIMIT ?`, ...lane.values, afterSequence ?? 0, ...lane.values, limit)
|
|
145
|
+
.toArray();
|
|
146
|
+
return rows.map((raw) => {
|
|
147
|
+
const parsed = parseJobRow(raw);
|
|
148
|
+
if (parsed.job.state !== "settled")
|
|
149
|
+
throw unavailable();
|
|
150
|
+
return { sequence: parsed.row.sequence, job: parsed.job };
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
function readQueueHead(owner, queue) {
|
|
154
|
+
const raw = owner.sql
|
|
155
|
+
.exec(`SELECT ${columns}
|
|
156
|
+
FROM watchdog_jobs
|
|
157
|
+
WHERE queue_name = ? AND state IN ('running', 'queued')
|
|
158
|
+
ORDER BY CASE state WHEN 'running' THEN 0 ELSE 1 END,
|
|
159
|
+
priority DESC, sequence ASC
|
|
160
|
+
LIMIT 1`, queue)
|
|
161
|
+
.toArray()[0];
|
|
162
|
+
if (raw === undefined)
|
|
163
|
+
return { state: "idle" };
|
|
164
|
+
const parsed = parseJobRow(raw);
|
|
165
|
+
if (parsed.job.state === "settled")
|
|
166
|
+
throw unavailable();
|
|
167
|
+
return {
|
|
168
|
+
state: parsed.job.state,
|
|
169
|
+
jobId: parsed.job.jobId,
|
|
170
|
+
cancellationRequested: parsed.row.cancellation_id !== null,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
function readLatestSettledJob(owner, queue) {
|
|
174
|
+
const raw = owner.sql
|
|
175
|
+
.exec(`SELECT ${columns}
|
|
176
|
+
FROM watchdog_jobs
|
|
177
|
+
WHERE queue_name = ? AND state = 'settled'
|
|
178
|
+
ORDER BY sequence DESC
|
|
179
|
+
LIMIT 1`, queue)
|
|
180
|
+
.toArray()[0];
|
|
181
|
+
if (raw === undefined)
|
|
182
|
+
return undefined;
|
|
183
|
+
const parsed = parseJobRow(raw);
|
|
184
|
+
if (parsed.job.state !== "settled")
|
|
185
|
+
throw unavailable();
|
|
186
|
+
return parsed.job;
|
|
187
|
+
}
|
|
188
|
+
/** Tell the owner projection one queue changed, inside the same transaction. */
|
|
189
|
+
function projectQueue(owner, project, queue, settled) {
|
|
190
|
+
if (project === undefined)
|
|
191
|
+
return;
|
|
192
|
+
const head = readQueueHead(owner, queue);
|
|
193
|
+
project(settled === undefined ? { queue, head } : { queue, head, settled });
|
|
194
|
+
}
|
|
195
|
+
function prepare(owner) {
|
|
196
|
+
owner.transactionSync(() => {
|
|
197
|
+
owner.sql.exec(`
|
|
198
|
+
CREATE TABLE IF NOT EXISTS watchdog_jobs (
|
|
199
|
+
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
200
|
+
job_id TEXT UNIQUE NOT NULL,
|
|
201
|
+
queue_name TEXT NOT NULL,
|
|
202
|
+
lane TEXT NOT NULL,
|
|
203
|
+
priority INTEGER NOT NULL,
|
|
204
|
+
payload_json TEXT NOT NULL,
|
|
205
|
+
recovery_max INTEGER,
|
|
206
|
+
state TEXT NOT NULL CHECK (state IN ('queued', 'running', 'settled')),
|
|
207
|
+
outcome TEXT CHECK (
|
|
208
|
+
outcome IS NULL OR outcome IN (
|
|
209
|
+
'completed', 'failed', 'cancelled', 'outcome_unknown', 'interrupted'
|
|
210
|
+
)
|
|
211
|
+
),
|
|
212
|
+
claim_token TEXT,
|
|
213
|
+
recovery_count INTEGER NOT NULL DEFAULT 0,
|
|
214
|
+
cancellation_id TEXT UNIQUE,
|
|
215
|
+
failure_reason TEXT,
|
|
216
|
+
CHECK (recovery_max IS NULL OR recovery_max >= 0),
|
|
217
|
+
CHECK (
|
|
218
|
+
(state = 'queued' AND outcome IS NULL AND claim_token IS NULL) OR
|
|
219
|
+
(state = 'running' AND outcome IS NULL AND claim_token IS NOT NULL) OR
|
|
220
|
+
(state = 'settled' AND outcome IS NOT NULL)
|
|
221
|
+
)
|
|
222
|
+
)
|
|
223
|
+
`);
|
|
224
|
+
owner.sql.exec(`CREATE INDEX IF NOT EXISTS idx_watchdog_lane_failure
|
|
225
|
+
ON watchdog_jobs(lane, state DESC, sequence ASC)`);
|
|
226
|
+
owner.sql.exec(`CREATE INDEX IF NOT EXISTS idx_watchdog_ready
|
|
227
|
+
ON watchdog_jobs(state, priority DESC, sequence ASC)`);
|
|
228
|
+
owner.sql.exec(`CREATE INDEX IF NOT EXISTS idx_watchdog_queue_head
|
|
229
|
+
ON watchdog_jobs(queue_name, state, priority DESC, sequence ASC)`);
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
function enqueue(owner, project, job) {
|
|
233
|
+
const payloadJson = encodeOpaquePayload(job.payload);
|
|
234
|
+
const prior = readJob(owner, job.jobId);
|
|
235
|
+
if (prior !== undefined) {
|
|
236
|
+
const priorRecovery = prior.job.recovery?.maxRecoveries;
|
|
237
|
+
const nextRecovery = job.recovery?.maxRecoveries;
|
|
238
|
+
if (prior.job.queue !== job.queue ||
|
|
239
|
+
prior.job.lane !== job.lane ||
|
|
240
|
+
prior.job.priority !== job.priority ||
|
|
241
|
+
prior.row.payload_json !== payloadJson ||
|
|
242
|
+
priorRecovery !== nextRecovery)
|
|
243
|
+
throw unavailable();
|
|
244
|
+
return false;
|
|
245
|
+
}
|
|
246
|
+
owner.sql.exec(`INSERT INTO watchdog_jobs
|
|
247
|
+
(job_id, queue_name, lane, priority, payload_json, recovery_max,
|
|
248
|
+
state, outcome, claim_token, recovery_count, cancellation_id)
|
|
249
|
+
VALUES (?, ?, ?, ?, ?, ?, 'queued', NULL, NULL, 0, NULL)`, job.jobId, job.queue, job.lane, job.priority, payloadJson, job.recovery?.maxRecoveries ?? null);
|
|
250
|
+
projectQueue(owner, project, job.queue);
|
|
251
|
+
return true;
|
|
252
|
+
}
|
|
253
|
+
function recordCancellationIntent(owner, project, intent) {
|
|
254
|
+
const prior = readJob(owner, intent.jobId);
|
|
255
|
+
if (prior === undefined)
|
|
256
|
+
return { status: "missing" };
|
|
257
|
+
if (prior.row.cancellation_id !== null)
|
|
258
|
+
return prior.row.cancellation_id === intent.cancellationId
|
|
259
|
+
? { status: "replayed", intent }
|
|
260
|
+
: { status: "conflict" };
|
|
261
|
+
if (intent.onlyIfQueued === true && prior.job.state !== "queued")
|
|
262
|
+
return { status: "conflict" };
|
|
263
|
+
owner.sql.exec(`UPDATE watchdog_jobs
|
|
264
|
+
SET cancellation_id = ?
|
|
265
|
+
WHERE job_id = ? AND cancellation_id IS NULL`, intent.cancellationId, intent.jobId);
|
|
266
|
+
projectQueue(owner, project, prior.job.queue);
|
|
267
|
+
return { status: "recorded", intent };
|
|
268
|
+
}
|
|
269
|
+
function selectAndClaim(owner, project, claimToken, partition) {
|
|
270
|
+
const lane = laneQuery(partition);
|
|
271
|
+
return owner.transactionSync(() => {
|
|
272
|
+
const raw = owner.sql
|
|
273
|
+
.exec(`SELECT ${columns}
|
|
274
|
+
FROM watchdog_jobs
|
|
275
|
+
WHERE state = 'queued' AND ${lane.predicate}
|
|
276
|
+
AND (cancellation_id IS NOT NULL OR NOT EXISTS (
|
|
277
|
+
SELECT 1 FROM watchdog_jobs predecessor INDEXED BY idx_watchdog_queue_head
|
|
278
|
+
WHERE predecessor.queue_name = watchdog_jobs.queue_name
|
|
279
|
+
AND predecessor.state IN ('queued', 'running')
|
|
280
|
+
AND (predecessor.state = 'running' OR
|
|
281
|
+
(predecessor.state = 'queued' AND
|
|
282
|
+
(predecessor.priority > watchdog_jobs.priority OR
|
|
283
|
+
(predecessor.priority = watchdog_jobs.priority AND
|
|
284
|
+
predecessor.sequence < watchdog_jobs.sequence))))))
|
|
285
|
+
ORDER BY priority DESC, sequence ASC
|
|
286
|
+
LIMIT 1`, ...lane.values)
|
|
287
|
+
.toArray()[0];
|
|
288
|
+
if (raw === undefined)
|
|
289
|
+
return { status: "no_ready" };
|
|
290
|
+
const selected = parseJobRow(raw);
|
|
291
|
+
if (selected.job.state !== "queued")
|
|
292
|
+
return { status: "claim_unavailable" };
|
|
293
|
+
if (selected.row.cancellation_id !== null) {
|
|
294
|
+
owner.sql.exec(`UPDATE watchdog_jobs
|
|
295
|
+
SET state = 'settled', outcome = 'cancelled'
|
|
296
|
+
WHERE job_id = ? AND state = 'queued' AND cancellation_id = ?`, selected.job.jobId, selected.row.cancellation_id);
|
|
297
|
+
const settled = readJob(owner, selected.job.jobId)?.job;
|
|
298
|
+
if (settled?.state !== "settled" || settled.outcome !== "cancelled")
|
|
299
|
+
return { status: "claim_unavailable" };
|
|
300
|
+
projectQueue(owner, project, settled.queue, settled);
|
|
301
|
+
return { status: "cancelled", job: settled };
|
|
302
|
+
}
|
|
303
|
+
owner.sql.exec(`UPDATE watchdog_jobs
|
|
304
|
+
SET state = 'running', claim_token = ?
|
|
305
|
+
WHERE job_id = ? AND state = 'queued'`, claimToken, selected.job.jobId);
|
|
306
|
+
const claimed = readJob(owner, selected.job.jobId)?.job;
|
|
307
|
+
if (claimed?.state !== "running")
|
|
308
|
+
return { status: "claim_unavailable" };
|
|
309
|
+
projectQueue(owner, project, claimed.queue);
|
|
310
|
+
return { status: "claimed", job: claimed, claimToken };
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
function settleExactClaim(owner, project, input) {
|
|
314
|
+
return owner.transactionSync(() => {
|
|
315
|
+
const prior = readJob(owner, input.jobId);
|
|
316
|
+
if (prior === undefined)
|
|
317
|
+
return { status: "missing" };
|
|
318
|
+
const outcome = prior.row.cancellation_id === null ? input.decision.kind : "cancelled";
|
|
319
|
+
if (prior.job.state === "settled")
|
|
320
|
+
return prior.row.claim_token === input.claimToken &&
|
|
321
|
+
prior.job.outcome === outcome
|
|
322
|
+
? { status: "replayed", job: prior.job }
|
|
323
|
+
: { status: "conflict" };
|
|
324
|
+
if (prior.job.state !== "running" ||
|
|
325
|
+
prior.row.claim_token !== input.claimToken)
|
|
326
|
+
return { status: "conflict" };
|
|
327
|
+
owner.sql.exec(`UPDATE watchdog_jobs
|
|
328
|
+
SET state = 'settled', outcome = ?, failure_reason = ?
|
|
329
|
+
WHERE job_id = ? AND state = 'running' AND claim_token = ?`, outcome, outcome === "failed" && input.decision.kind === "failed"
|
|
330
|
+
? (input.decision.reason ?? null)
|
|
331
|
+
: null, input.jobId, input.claimToken);
|
|
332
|
+
const settled = readJob(owner, input.jobId)?.job;
|
|
333
|
+
if (settled?.state !== "settled")
|
|
334
|
+
return { status: "conflict" };
|
|
335
|
+
projectQueue(owner, project, settled.queue, settled);
|
|
336
|
+
return { status: "settled", job: settled };
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
/** Construct Watchdog's private owner-local SQLite state. */
|
|
340
|
+
export function openSqliteState(owner, project, partition) {
|
|
341
|
+
prepare(owner);
|
|
342
|
+
const transactional = Object.freeze({
|
|
343
|
+
readQueueHead: (queue) => readQueueHead(owner, queue),
|
|
344
|
+
readLatestSettledJob: (queue) => readLatestSettledJob(owner, queue),
|
|
345
|
+
enqueueAcceptedJob: (job) => {
|
|
346
|
+
enqueue(owner, project, job);
|
|
347
|
+
},
|
|
348
|
+
recordCancellationIntent: (intent) => recordCancellationIntent(owner, project, intent),
|
|
349
|
+
});
|
|
350
|
+
const createStore = (partition) => {
|
|
351
|
+
const lane = laneQuery(partition);
|
|
352
|
+
return Object.freeze({
|
|
353
|
+
hasPendingWork: storeEffect(() => owner.sql
|
|
354
|
+
.exec("SELECT 1 FROM watchdog_jobs WHERE state IN ('queued', 'running') LIMIT 1")
|
|
355
|
+
.toArray().length !== 0),
|
|
356
|
+
ingest: (job) => storeEffect(() => owner.transactionSync(() => ({
|
|
357
|
+
status: enqueue(owner, project, job) ? "queued" : "duplicate",
|
|
358
|
+
}))),
|
|
359
|
+
read: (jobId) => storeEffect(() => {
|
|
360
|
+
const parsed = readJob(owner, jobId);
|
|
361
|
+
if (parsed === undefined)
|
|
362
|
+
return { status: "missing" };
|
|
363
|
+
return {
|
|
364
|
+
status: "found",
|
|
365
|
+
job: parsed.job,
|
|
366
|
+
claimToken: parsed.row.claim_token,
|
|
367
|
+
recoveryCount: parsed.row.recovery_count,
|
|
368
|
+
cancellation: parsed.row.cancellation_id === null
|
|
369
|
+
? { status: "absent" }
|
|
370
|
+
: {
|
|
371
|
+
status: "present",
|
|
372
|
+
cancellation: {
|
|
373
|
+
jobId,
|
|
374
|
+
cancellationId: parsed.row.cancellation_id,
|
|
375
|
+
},
|
|
376
|
+
},
|
|
377
|
+
};
|
|
378
|
+
}),
|
|
379
|
+
readQueueHead: (queue) => storeEffect(() => readQueueHead(owner, queue)),
|
|
380
|
+
readSettled: (limit, afterSequence) => storeEffect(() => readSettled(owner, limit, afterSequence, partition)),
|
|
381
|
+
selectAndClaim: (claimToken) => storeEffect(() => selectAndClaim(owner, project, claimToken, partition)),
|
|
382
|
+
settleExactClaim: (input) => storeEffect(() => settleExactClaim(owner, project, input)),
|
|
383
|
+
recordCancellation: (intent) => storeEffect(() => owner.transactionSync(() => {
|
|
384
|
+
const result = recordCancellationIntent(owner, project, intent);
|
|
385
|
+
return { status: result.status };
|
|
386
|
+
})),
|
|
387
|
+
failLane: (raw) => storeEffect(() => {
|
|
388
|
+
const input = parseLaneFailure(raw);
|
|
389
|
+
return owner.transactionSync(() => {
|
|
390
|
+
// Under the declared state grammar, DESC puts running before queued.
|
|
391
|
+
const rows = owner.sql
|
|
392
|
+
.exec(`SELECT ${columns} FROM watchdog_jobs
|
|
393
|
+
WHERE lane = ? AND state IN ('queued', 'running')
|
|
394
|
+
ORDER BY state DESC, sequence ASC LIMIT ?`, input.lane, laneFailureBatchSize)
|
|
395
|
+
.toArray();
|
|
396
|
+
const jobs = [];
|
|
397
|
+
for (const row of rows) {
|
|
398
|
+
const prior = parseJobRow(row);
|
|
399
|
+
owner.sql.exec(`UPDATE watchdog_jobs SET state = 'settled', outcome = 'failed', failure_reason = ?, claim_token = NULL
|
|
400
|
+
WHERE job_id = ? AND state IN ('queued', 'running')`, input.reason, prior.job.jobId);
|
|
401
|
+
const settled = readJob(owner, prior.job.jobId)?.job;
|
|
402
|
+
if (settled?.state !== "settled" || settled.outcome !== "failed")
|
|
403
|
+
throw unavailable();
|
|
404
|
+
projectQueue(owner, project, settled.queue, settled);
|
|
405
|
+
jobs.push(settled);
|
|
406
|
+
}
|
|
407
|
+
const more = owner.sql
|
|
408
|
+
.exec("SELECT 1 FROM watchdog_jobs WHERE lane = ? AND state IN ('queued', 'running') LIMIT 1", input.lane)
|
|
409
|
+
.toArray().length !== 0;
|
|
410
|
+
return { jobs, more };
|
|
411
|
+
});
|
|
412
|
+
}),
|
|
413
|
+
purgeQueued: (queue) => storeEffect(() => {
|
|
414
|
+
owner.transactionSync(() => {
|
|
415
|
+
owner.sql.exec(`DELETE FROM watchdog_jobs
|
|
416
|
+
WHERE queue_name = ? AND state = 'queued'`, queue);
|
|
417
|
+
projectQueue(owner, project, queue);
|
|
418
|
+
});
|
|
419
|
+
}),
|
|
420
|
+
nextRunningClaim: () => storeEffect(() => {
|
|
421
|
+
const raw = owner.sql
|
|
422
|
+
.exec(`SELECT ${columns}
|
|
423
|
+
FROM watchdog_jobs
|
|
424
|
+
WHERE state = 'running' AND ${lane.predicate}
|
|
425
|
+
ORDER BY sequence ASC
|
|
426
|
+
LIMIT 1`, ...lane.values)
|
|
427
|
+
.toArray()[0];
|
|
428
|
+
if (raw === undefined)
|
|
429
|
+
return undefined;
|
|
430
|
+
const parsed = parseJobRow(raw);
|
|
431
|
+
if (parsed.job.state !== "running" || parsed.row.claim_token === null)
|
|
432
|
+
throw unavailable();
|
|
433
|
+
return {
|
|
434
|
+
job: parsed.job,
|
|
435
|
+
claimToken: parsed.row.claim_token,
|
|
436
|
+
recoveryCount: parsed.row.recovery_count,
|
|
437
|
+
};
|
|
438
|
+
}),
|
|
439
|
+
reconcileDisappearedClaim: (input) => storeEffect(() => owner.transactionSync(() => {
|
|
440
|
+
const prior = readJob(owner, input.job.jobId);
|
|
441
|
+
if (prior === undefined)
|
|
442
|
+
return { status: "missing" };
|
|
443
|
+
if (prior.job.state !== "running" ||
|
|
444
|
+
prior.row.claim_token !== input.claimToken ||
|
|
445
|
+
prior.row.recovery_count !== input.recoveryCount)
|
|
446
|
+
return { status: "conflict" };
|
|
447
|
+
if (input.disposition === "requeue" &&
|
|
448
|
+
prior.row.cancellation_id === null) {
|
|
449
|
+
owner.sql.exec(`UPDATE watchdog_jobs
|
|
450
|
+
SET state = 'queued', claim_token = NULL,
|
|
451
|
+
recovery_count = recovery_count + 1
|
|
452
|
+
WHERE job_id = ? AND state = 'running' AND claim_token = ?`, input.job.jobId, input.claimToken);
|
|
453
|
+
projectQueue(owner, project, input.job.queue);
|
|
454
|
+
return { status: "requeued" };
|
|
455
|
+
}
|
|
456
|
+
const cancelled = prior.row.cancellation_id !== null;
|
|
457
|
+
const reason = cancelled
|
|
458
|
+
? null
|
|
459
|
+
: prior.job.recovery === undefined
|
|
460
|
+
? "Execution disappeared without a recovery policy"
|
|
461
|
+
: `Restart recovery budget exhausted (${String(prior.job.recovery.maxRecoveries)} recoveries)`;
|
|
462
|
+
owner.sql.exec(`UPDATE watchdog_jobs
|
|
463
|
+
SET state = 'settled', outcome = ?, failure_reason = ?
|
|
464
|
+
WHERE job_id = ? AND state = 'running' AND claim_token = ?`, cancelled ? "cancelled" : "interrupted", reason, input.job.jobId, input.claimToken);
|
|
465
|
+
const settled = readJob(owner, input.job.jobId)?.job;
|
|
466
|
+
if (settled?.state !== "settled")
|
|
467
|
+
return { status: "conflict" };
|
|
468
|
+
projectQueue(owner, project, settled.queue, settled);
|
|
469
|
+
return { status: "settled", job: settled };
|
|
470
|
+
})),
|
|
471
|
+
});
|
|
472
|
+
};
|
|
473
|
+
return Object.freeze({
|
|
474
|
+
store: createStore(partition),
|
|
475
|
+
partition: createStore,
|
|
476
|
+
transactional,
|
|
477
|
+
});
|
|
478
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Typed failures owned by the generic Watchdog engine. */
|
|
2
|
+
export declare class WatchdogError extends Error {
|
|
3
|
+
readonly _tag: "WatchdogError";
|
|
4
|
+
readonly cause: "store_unavailable" | "store_invalid" | "claim_unavailable" | "execution_failed" | "settlement_rejected" | "wake_unavailable" | "liveness_unavailable";
|
|
5
|
+
readonly underlying?: unknown;
|
|
6
|
+
constructor(failureKind: "store_unavailable" | "store_invalid" | "claim_unavailable" | "execution_failed" | "settlement_rejected" | "wake_unavailable" | "liveness_unavailable",
|
|
7
|
+
/**
|
|
8
|
+
* The original thrown value, kept for server-side diagnosis only. `cause`
|
|
9
|
+
* is already the closed category above, so this is a separate field.
|
|
10
|
+
*/
|
|
11
|
+
cause?: unknown);
|
|
12
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** Typed failures owned by the generic Watchdog engine. */
|
|
2
|
+
export class WatchdogError extends Error {
|
|
3
|
+
_tag = "WatchdogError";
|
|
4
|
+
cause;
|
|
5
|
+
underlying;
|
|
6
|
+
constructor(failureKind,
|
|
7
|
+
/**
|
|
8
|
+
* The original thrown value, kept for server-side diagnosis only. `cause`
|
|
9
|
+
* is already the closed category above, so this is a separate field.
|
|
10
|
+
*/
|
|
11
|
+
cause) {
|
|
12
|
+
super(`Watchdog ${failureKind.replaceAll("_", " ")}`);
|
|
13
|
+
this.name = this._tag;
|
|
14
|
+
this.cause = failureKind;
|
|
15
|
+
this.underlying = cause;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import type { WatchdogSqliteOwner, WatchdogSqliteCursor, WatchdogSqliteRow, WatchdogSqliteValue } from "./effect.js";
|
|
3
|
+
/** Materialized Node SQLite rows, including the exact-one native operation. */
|
|
4
|
+
export type NodeSqliteCursor = WatchdogSqliteCursor & Readonly<{
|
|
5
|
+
/** Return the sole decoded row; throw for zero or multiple rows. */
|
|
6
|
+
one: () => WatchdogSqliteRow;
|
|
7
|
+
}>;
|
|
8
|
+
/** Concrete Node adapter; generic Watchdog owners need only toArray(). */
|
|
9
|
+
export type NodeSqliteOwner = Omit<WatchdogSqliteOwner, "sql"> & Readonly<{
|
|
10
|
+
/** Statements execute synchronously against the caller-owned database. */
|
|
11
|
+
sql: Readonly<{
|
|
12
|
+
/** Decode materialized scalar rows without choosing the query owner’s schema. */
|
|
13
|
+
exec: (statement: string, ...bindings: WatchdogSqliteValue[]) => NodeSqliteCursor;
|
|
14
|
+
}>;
|
|
15
|
+
}>;
|
|
16
|
+
/** Adapt a caller-owned Node SQLite database; the caller retains its lifetime. */
|
|
17
|
+
export declare function fromNodeSqlite(database: DatabaseSync): NodeSqliteOwner;
|