cairnq 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/_protocol/migrations/postgres/0002_purge_index.sql +6 -0
- package/dist/_protocol/migrations/postgres/0003_notify.sql +38 -0
- package/dist/_protocol/migrations/sqlite/0002_purge_index.sql +6 -0
- package/dist/_protocol/sql/postgres/claim.sql +18 -5
- package/dist/_protocol/sql/postgres/fail.sql +28 -8
- package/dist/_protocol/sql/postgres/insert_task.sql +6 -3
- package/dist/_protocol/sql/postgres/list.sql +3 -1
- package/dist/_protocol/sql/postgres/lock_key.sql +9 -0
- package/dist/_protocol/sql/postgres/progress.sql +4 -3
- package/dist/_protocol/sql/postgres/protocol_version.sql +4 -0
- package/dist/_protocol/sql/postgres/purge.sql +25 -0
- package/dist/_protocol/sql/postgres/recover_leases.sql +38 -14
- package/dist/_protocol/sql/postgres/retry.sql +3 -0
- package/dist/_protocol/sql/postgres/stats.sql +8 -0
- package/dist/_protocol/sql/sqlite/claim.sql +13 -2
- package/dist/_protocol/sql/sqlite/claimable_probe.sql +6 -2
- package/dist/_protocol/sql/sqlite/fail.sql +30 -8
- package/dist/_protocol/sql/sqlite/list.sql +3 -1
- package/dist/_protocol/sql/sqlite/lock_key.sql +5 -0
- package/dist/_protocol/sql/sqlite/progress.sql +6 -2
- package/dist/_protocol/sql/sqlite/protocol_version.sql +4 -0
- package/dist/_protocol/sql/sqlite/purge.sql +18 -0
- package/dist/_protocol/sql/sqlite/recover_leases.sql +25 -7
- package/dist/_protocol/sql/sqlite/retry.sql +3 -0
- package/dist/_protocol/sql/sqlite/stats.sql +8 -0
- package/dist/client.d.ts +10 -2
- package/dist/client.js +12 -0
- package/dist/context.d.ts +17 -1
- package/dist/context.js +60 -6
- package/dist/errors.d.ts +18 -2
- package/dist/errors.js +49 -3
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/sql.js +16 -9
- package/dist/store/base.d.ts +107 -9
- package/dist/store/base.js +370 -1
- package/dist/store/postgres.d.ts +62 -63
- package/dist/store/postgres.js +245 -222
- package/dist/store/sqlite.d.ts +34 -59
- package/dist/store/sqlite.js +200 -232
- package/dist/wait.d.ts +15 -2
- package/dist/wait.js +23 -5
- package/dist/worker.d.ts +53 -1
- package/dist/worker.js +202 -42
- package/package.json +9 -2
- package/src/client.ts +16 -2
- package/src/context.ts +70 -13
- package/src/errors.ts +59 -4
- package/src/index.ts +3 -1
- package/src/sql.ts +15 -8
- package/src/store/base.ts +430 -27
- package/src/store/postgres.ts +243 -267
- package/src/store/sqlite.ts +211 -265
- package/src/wait.ts +28 -5
- package/src/worker.ts +242 -42
package/dist/store/postgres.js
CHANGED
|
@@ -1,14 +1,5 @@
|
|
|
1
|
-
import { newId } from "../ids.js";
|
|
2
|
-
import { AlreadyExists, errorEnvelope, LostLease, ProtocolVersionMismatch } from "../errors.js";
|
|
3
|
-
import { rowToTask } from "../models.js";
|
|
4
1
|
import { loadMigrations, loadStatements } from "../sql.js";
|
|
5
|
-
|
|
6
|
-
const LEASE_EXPIRED_ERROR_JSON = JSON.stringify(errorEnvelope({
|
|
7
|
-
type: "LeaseExpired",
|
|
8
|
-
code: "lease_expired",
|
|
9
|
-
message: "task lease expired and max attempts reached",
|
|
10
|
-
retryable: false,
|
|
11
|
-
}));
|
|
2
|
+
import { checkProtocolVersion, COMMENT, NAMED, statementParams, TaskStore, } from "./base.js";
|
|
12
3
|
// `pg` is an optional dependency: the SDK is SQLite-first, so it's loaded lazily
|
|
13
4
|
// the first time a PostgresStore connects. Absent -> a clear install hint.
|
|
14
5
|
let pgModule = null;
|
|
@@ -32,41 +23,100 @@ async function loadPg() {
|
|
|
32
23
|
return pg;
|
|
33
24
|
}
|
|
34
25
|
/**
|
|
35
|
-
*
|
|
26
|
+
* Roll back on the way out of a failed transaction, without letting the rollback
|
|
27
|
+
* become the error the caller sees. A dropped connection fails both the statement
|
|
28
|
+
* and the rollback, and it is the first one that says what went wrong.
|
|
29
|
+
*/
|
|
30
|
+
async function rollbackQuietly(client) {
|
|
31
|
+
try {
|
|
32
|
+
await client.query("rollback");
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
// Already rolled back, or the connection is gone. Either way the original
|
|
36
|
+
// error is the one worth propagating.
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
// Notification channels, emitted by the 0003_notify trigger.
|
|
40
|
+
const QUEUED_CHANNEL = "cairnq_queued";
|
|
41
|
+
const DONE_CHANNEL = "cairnq_done";
|
|
42
|
+
// Backoff between attempts to (re)connect the LISTEN connection after a
|
|
43
|
+
// transient failure. Doubles per failure up to the cap; polling covers the gap.
|
|
44
|
+
const LISTENER_RETRY_MS = 1_000;
|
|
45
|
+
const LISTENER_RETRY_MAX_MS = 30_000;
|
|
46
|
+
const translated = new Map();
|
|
47
|
+
/**
|
|
48
|
+
* The rewritten SQL and the order its `$n` slots must be filled in.
|
|
49
|
+
*
|
|
50
|
+
* Translates the protocol's named-parameter SQL (`:name`) into Postgres positional
|
|
36
51
|
* placeholders (`$1`), collapsing each DISTINCT name to ONE slot — statements reuse
|
|
37
|
-
* a name across CASE branches / IS NULL guards (e.g. list.sql).
|
|
38
|
-
*
|
|
39
|
-
*
|
|
52
|
+
* a name across CASE branches / IS NULL guards (e.g. list.sql). Which names count
|
|
53
|
+
* as parameters is `statementParams`' decision, shared with the SQLite binding path
|
|
54
|
+
* so the two can't disagree about, say, a `::type` cast.
|
|
55
|
+
*
|
|
56
|
+
* Memoized on the statement text, which is loaded once and never varies: this runs
|
|
57
|
+
* on every query, including the worker's poll loop, for a result that cannot have
|
|
58
|
+
* changed. Exported for unit testing.
|
|
59
|
+
*/
|
|
60
|
+
export function positionalStatement(sql) {
|
|
61
|
+
let entry = translated.get(sql);
|
|
62
|
+
if (!entry) {
|
|
63
|
+
const order = statementParams(sql);
|
|
64
|
+
const slot = new Map(order.map((name, i) => [name, i + 1])); // 1-based $n
|
|
65
|
+
const text = sql
|
|
66
|
+
.replace(COMMENT, "")
|
|
67
|
+
.replace(NAMED, (_m, name) => `$${slot.get(name)}`);
|
|
68
|
+
entry = { text, order };
|
|
69
|
+
translated.set(sql, entry);
|
|
70
|
+
}
|
|
71
|
+
return entry;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* The statement's rewritten text plus this call's values, in slot order. Names the
|
|
75
|
+
* statement does not use are simply not bound, so callers may pass a superset.
|
|
40
76
|
*/
|
|
41
77
|
export function toPositional(sql, params) {
|
|
42
|
-
const
|
|
43
|
-
const order = [];
|
|
44
|
-
const slot = new Map();
|
|
45
|
-
const text = body.replace(/(?<!:):(\w+)/g, (_m, name) => {
|
|
46
|
-
let i = slot.get(name);
|
|
47
|
-
if (i === undefined) {
|
|
48
|
-
order.push(name);
|
|
49
|
-
i = order.length; // 1-based $n
|
|
50
|
-
slot.set(name, i);
|
|
51
|
-
}
|
|
52
|
-
return `$${i}`;
|
|
53
|
-
});
|
|
78
|
+
const { text, order } = positionalStatement(sql);
|
|
54
79
|
return { text, values: order.map((n) => params[n]) };
|
|
55
80
|
}
|
|
56
81
|
/**
|
|
57
|
-
* PostgresStore —
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
82
|
+
* PostgresStore — the Postgres dialect of the shared cairnq-protocol SQL.
|
|
83
|
+
*
|
|
84
|
+
* Everything protocol-shaped lives in TaskStore; this file is only what Postgres
|
|
85
|
+
* does differently: a `pg` Pool, `:name` -> `$n` translation, and time taken from
|
|
86
|
+
* the DB clock (`now()`) instead of from the SDK, which is what makes this backend
|
|
87
|
+
* multi-host — unlike SQLite it coordinates API and worker processes across
|
|
88
|
+
* machines, with no shared clock to agree on. claim uses FOR UPDATE SKIP LOCKED
|
|
89
|
+
* and needs no claimable_probe, because PG readers don't block writers. JSON
|
|
90
|
+
* columns are jsonb (bound as JSON text, read back as objects by rowToTask).
|
|
62
91
|
*/
|
|
63
|
-
export class PostgresStore {
|
|
92
|
+
export class PostgresStore extends TaskStore {
|
|
64
93
|
dsn;
|
|
65
94
|
opts;
|
|
66
95
|
pool = null;
|
|
67
96
|
connecting = null;
|
|
68
97
|
statements;
|
|
98
|
+
// ------------------------------------------------------- LISTEN/NOTIFY state
|
|
99
|
+
// One dedicated connection LISTENs on both channels (see 0003_notify.sql and
|
|
100
|
+
// the claimWake/taskDoneWake contract on TaskStore). Failure to establish or
|
|
101
|
+
// keep it silently degrades the store to the base class's plain polling.
|
|
102
|
+
listener = null;
|
|
103
|
+
listenerConnecting = null;
|
|
104
|
+
/** LISTEN is off for good: the store was closed, or the server accepted a
|
|
105
|
+
* connection but refused LISTEN (e.g. a transaction-mode pooler) —
|
|
106
|
+
* deterministic, so retrying would fail the same way every time. */
|
|
107
|
+
listenerUnavailable = false;
|
|
108
|
+
/** A failure to even connect is transient (network blip, server restarting):
|
|
109
|
+
* retry, but not before this time, backing off so a down server is not
|
|
110
|
+
* hammered from the poll loop. */
|
|
111
|
+
listenerRetryAt = 0;
|
|
112
|
+
listenerBackoffMs = LISTENER_RETRY_MS;
|
|
113
|
+
/** Queues notified while nobody was waiting; consumed by the next claimWake
|
|
114
|
+
* so a wake that lands between polls is not lost. */
|
|
115
|
+
pendingQueues = new Set();
|
|
116
|
+
/** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
|
|
117
|
+
waiters = new Map();
|
|
69
118
|
constructor(dsn, opts = {}) {
|
|
119
|
+
super();
|
|
70
120
|
this.dsn = dsn;
|
|
71
121
|
this.opts = opts;
|
|
72
122
|
this.statements = loadStatements("postgres");
|
|
@@ -75,6 +125,8 @@ export class PostgresStore {
|
|
|
75
125
|
await this.ensure();
|
|
76
126
|
}
|
|
77
127
|
async close() {
|
|
128
|
+
this.listenerUnavailable = true; // no revival after close
|
|
129
|
+
this.dropListener();
|
|
78
130
|
if (this.pool) {
|
|
79
131
|
const p = this.pool;
|
|
80
132
|
this.pool = null;
|
|
@@ -102,10 +154,7 @@ export class PostgresStore {
|
|
|
102
154
|
const client = await pool.connect();
|
|
103
155
|
try {
|
|
104
156
|
await this.applyMigrations(client);
|
|
105
|
-
|
|
106
|
-
if (version !== SUPPORTED_PROTOCOL_MAJOR) {
|
|
107
|
-
throw new ProtocolVersionMismatch(`storage protocol_version=${version}, SDK supports ${SUPPORTED_PROTOCOL_MAJOR}`);
|
|
108
|
-
}
|
|
157
|
+
checkProtocolVersion(await this.readProtocolVersion(client));
|
|
109
158
|
}
|
|
110
159
|
finally {
|
|
111
160
|
client.release();
|
|
@@ -116,29 +165,40 @@ export class PostgresStore {
|
|
|
116
165
|
throw e;
|
|
117
166
|
}
|
|
118
167
|
this.pool = pool; // publish only a fully-migrated, version-checked pool
|
|
168
|
+
// Warm the LISTEN connection in the background so the first idle sleep is
|
|
169
|
+
// already wakeable. Fire-and-forget: failure just means polling.
|
|
170
|
+
this.listenerReady();
|
|
119
171
|
}
|
|
120
172
|
async applyMigrations(client) {
|
|
121
173
|
await client.query("create table if not exists cairnq_migrations " +
|
|
122
174
|
"(name text primary key, applied_at_ms bigint not null)");
|
|
123
|
-
const applied = new Set((await client.query("select name from cairnq_migrations")).rows.map((r) => r.name));
|
|
124
175
|
for (const { name, sql } of loadMigrations("postgres")) {
|
|
125
|
-
|
|
126
|
-
|
|
176
|
+
// Check and apply inside one transaction, with the table lock taken up
|
|
177
|
+
// front: two processes cold-starting together would otherwise both see a
|
|
178
|
+
// migration as unapplied and both run it.
|
|
127
179
|
try {
|
|
128
180
|
await client.query("begin");
|
|
129
|
-
await client.query(
|
|
130
|
-
await client.query("
|
|
131
|
-
|
|
181
|
+
await client.query("lock table cairnq_migrations in exclusive mode");
|
|
182
|
+
const applied = await client.query("select 1 from cairnq_migrations where name = $1", [
|
|
183
|
+
name,
|
|
184
|
+
]);
|
|
185
|
+
if (applied.rowCount === 0) {
|
|
186
|
+
await client.query(sql); // multi-statement DDL (simple-query, no params)
|
|
187
|
+
await client.query("insert into cairnq_migrations (name, applied_at_ms) values " +
|
|
188
|
+
"($1, (extract(epoch from now()) * 1000)::bigint)", [name]);
|
|
189
|
+
}
|
|
132
190
|
await client.query("commit");
|
|
133
191
|
}
|
|
134
192
|
catch (e) {
|
|
135
|
-
await client
|
|
193
|
+
await rollbackQuietly(client);
|
|
136
194
|
throw e;
|
|
137
195
|
}
|
|
138
196
|
}
|
|
139
197
|
}
|
|
198
|
+
// Takes an explicit client: during doConnect the pool is not published yet, so
|
|
199
|
+
// this cannot go through fetch(). The statement binds nothing, so it runs as-is.
|
|
140
200
|
async readProtocolVersion(client) {
|
|
141
|
-
const res = await client.query(
|
|
201
|
+
const res = await client.query(this.statements.protocol_version);
|
|
142
202
|
return res.rows.length ? Number(res.rows[0].value) : 0;
|
|
143
203
|
}
|
|
144
204
|
async protocolVersion() {
|
|
@@ -151,199 +211,162 @@ export class PostgresStore {
|
|
|
151
211
|
client.release();
|
|
152
212
|
}
|
|
153
213
|
}
|
|
154
|
-
//
|
|
155
|
-
async
|
|
156
|
-
|
|
157
|
-
|
|
214
|
+
// ------------------------------------------------------------ wake channel
|
|
215
|
+
async claimWake(queues, timeoutMs) {
|
|
216
|
+
if (!this.listenerReady())
|
|
217
|
+
return super.claimWake(queues, timeoutMs);
|
|
218
|
+
const deadline = Date.now() + timeoutMs;
|
|
219
|
+
for (;;) {
|
|
220
|
+
// Consume every pending notification for our queues; any hit wakes us.
|
|
221
|
+
let hit = false;
|
|
222
|
+
for (const q of queues)
|
|
223
|
+
if (this.pendingQueues.delete(q))
|
|
224
|
+
hit = true;
|
|
225
|
+
if (hit)
|
|
226
|
+
return;
|
|
227
|
+
const remaining = deadline - Date.now();
|
|
228
|
+
if (remaining <= 0)
|
|
229
|
+
return;
|
|
230
|
+
// Broadcast wake: another queue's notification loops us back to waiting
|
|
231
|
+
// with the remaining budget instead of waking the worker for nothing.
|
|
232
|
+
await this.wakeOn("queued", remaining);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
taskDoneWake(taskId, timeoutMs) {
|
|
236
|
+
if (!this.listenerReady())
|
|
237
|
+
return super.taskDoneWake(taskId, timeoutMs);
|
|
238
|
+
return this.wakeOn(`done:${taskId}`, timeoutMs);
|
|
239
|
+
}
|
|
240
|
+
/** A promise resolving on notification-or-timeout, deregistering either way.
|
|
241
|
+
* The timer is unref'd: while a listener exists its socket keeps the process
|
|
242
|
+
* alive, and dropListener wakes every waiter the moment it goes away. */
|
|
243
|
+
wakeOn(key, timeoutMs) {
|
|
244
|
+
return new Promise((resolve) => {
|
|
245
|
+
let set = this.waiters.get(key);
|
|
246
|
+
if (!set)
|
|
247
|
+
this.waiters.set(key, (set = new Set()));
|
|
248
|
+
const peers = set;
|
|
249
|
+
const waiter = () => {
|
|
250
|
+
clearTimeout(timer);
|
|
251
|
+
peers.delete(waiter);
|
|
252
|
+
if (peers.size === 0)
|
|
253
|
+
this.waiters.delete(key);
|
|
254
|
+
resolve();
|
|
255
|
+
};
|
|
256
|
+
const timer = setTimeout(waiter, timeoutMs);
|
|
257
|
+
timer.unref?.();
|
|
258
|
+
peers.add(waiter);
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
/** True once the LISTEN connection is up; starts connecting it otherwise
|
|
262
|
+
* (respecting the transient-failure backoff). Callers fall back to plain
|
|
263
|
+
* polling until it is ready (or forever, if it can't be established) —
|
|
264
|
+
* correctness never depends on it. */
|
|
265
|
+
listenerReady() {
|
|
266
|
+
if (this.listener)
|
|
267
|
+
return true;
|
|
268
|
+
if (!this.listenerUnavailable && !this.listenerConnecting && Date.now() >= this.listenerRetryAt) {
|
|
269
|
+
this.listenerConnecting = this.startListener();
|
|
270
|
+
}
|
|
271
|
+
return false;
|
|
158
272
|
}
|
|
159
|
-
async
|
|
273
|
+
async startListener() {
|
|
274
|
+
try {
|
|
275
|
+
const pg = await loadPg();
|
|
276
|
+
// Built from the raw DSN: if `opts` ever grows connection-level settings
|
|
277
|
+
// (ssl, application_name), the listener must receive them too.
|
|
278
|
+
const client = new pg.Client({ connectionString: this.dsn });
|
|
279
|
+
try {
|
|
280
|
+
await client.connect();
|
|
281
|
+
}
|
|
282
|
+
catch {
|
|
283
|
+
// Could not even connect — transient. Schedule a backed-off retry;
|
|
284
|
+
// polling covers the gap.
|
|
285
|
+
this.listenerRetryAt = Date.now() + this.listenerBackoffMs;
|
|
286
|
+
this.listenerBackoffMs = Math.min(LISTENER_RETRY_MAX_MS, this.listenerBackoffMs * 2);
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
client.on("notification", (msg) => this.onNotification(msg.channel, msg.payload));
|
|
290
|
+
// A dropped listener degrades to polling; the next wake call reconnects.
|
|
291
|
+
client.on("error", () => this.dropListener());
|
|
292
|
+
try {
|
|
293
|
+
await client.query(`listen ${QUEUED_CHANNEL}; listen ${DONE_CHANNEL}`);
|
|
294
|
+
}
|
|
295
|
+
catch {
|
|
296
|
+
// Connected, but LISTEN was refused (e.g. a transaction-mode pooler) —
|
|
297
|
+
// deterministic, so off for good. Polling covers it.
|
|
298
|
+
this.listenerUnavailable = true;
|
|
299
|
+
void client.end().catch(() => { });
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
if (this.listenerUnavailable) {
|
|
303
|
+
void client.end().catch(() => { }); // closed while we were connecting
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
this.listener = client;
|
|
307
|
+
this.listenerBackoffMs = LISTENER_RETRY_MS;
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
// Anything unexpected (e.g. `pg` failed to load): off for good rather
|
|
311
|
+
// than a retry loop that cannot succeed.
|
|
312
|
+
this.listenerUnavailable = true;
|
|
313
|
+
}
|
|
314
|
+
finally {
|
|
315
|
+
this.listenerConnecting = null;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
onNotification(channel, payload) {
|
|
319
|
+
let key;
|
|
320
|
+
if (channel === QUEUED_CHANNEL && payload) {
|
|
321
|
+
this.pendingQueues.add(payload);
|
|
322
|
+
key = "queued";
|
|
323
|
+
}
|
|
324
|
+
else if (channel === DONE_CHANNEL && payload) {
|
|
325
|
+
key = `done:${payload}`;
|
|
326
|
+
}
|
|
327
|
+
else {
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
const set = this.waiters.get(key);
|
|
331
|
+
// A waiter only deletes itself, which Set iteration tolerates — no copy.
|
|
332
|
+
if (set)
|
|
333
|
+
for (const w of set)
|
|
334
|
+
w();
|
|
335
|
+
}
|
|
336
|
+
dropListener() {
|
|
337
|
+
const client = this.listener;
|
|
338
|
+
this.listener = null;
|
|
339
|
+
if (client)
|
|
340
|
+
void client.end().catch(() => { });
|
|
341
|
+
// Release everyone promptly; their fallback poll takes over.
|
|
342
|
+
for (const set of [...this.waiters.values()])
|
|
343
|
+
for (const w of [...set])
|
|
344
|
+
w();
|
|
345
|
+
}
|
|
346
|
+
// ------------------------------------------------------------ dialect seam
|
|
347
|
+
async fetch(name, params) {
|
|
348
|
+
await this.ensure();
|
|
160
349
|
const { text, values } = toPositional(this.statements[name], params);
|
|
161
|
-
return (await
|
|
350
|
+
return (await this.pool.query(text, values)).rows;
|
|
162
351
|
}
|
|
163
|
-
// BEGIN … COMMIT on a dedicated pool client, rolling back on any error.
|
|
164
352
|
async tx(fn) {
|
|
353
|
+
await this.ensure();
|
|
165
354
|
const client = await this.pool.connect();
|
|
166
355
|
try {
|
|
167
356
|
await client.query("begin");
|
|
168
|
-
const out = await fn(
|
|
357
|
+
const out = await fn(async (name, params) => {
|
|
358
|
+
const { text, values } = toPositional(this.statements[name], params);
|
|
359
|
+
return (await client.query(text, values)).rows;
|
|
360
|
+
});
|
|
169
361
|
await client.query("commit");
|
|
170
362
|
return out;
|
|
171
363
|
}
|
|
172
364
|
catch (e) {
|
|
173
|
-
await client
|
|
365
|
+
await rollbackQuietly(client);
|
|
174
366
|
throw e;
|
|
175
367
|
}
|
|
176
368
|
finally {
|
|
177
369
|
client.release();
|
|
178
370
|
}
|
|
179
371
|
}
|
|
180
|
-
// An ownership-checked worker write: each statement's WHERE pins worker_id + a
|
|
181
|
-
// live lease, so 0 rows back means the lease was lost.
|
|
182
|
-
async ownedWrite(name, taskId, params) {
|
|
183
|
-
const rows = await this.run(name, params);
|
|
184
|
-
if (!rows.length)
|
|
185
|
-
throw new LostLease(taskId);
|
|
186
|
-
return rowToTask(rows[0]);
|
|
187
|
-
}
|
|
188
|
-
// ------------------------------------------------------------- client side
|
|
189
|
-
async submit(input) {
|
|
190
|
-
await this.ensure();
|
|
191
|
-
const id = newId("task");
|
|
192
|
-
// No now_ms / run_at_ms: the DB clock supplies time, so submit passes a
|
|
193
|
-
// relative :delay_ms and the SQL computes run_at = now + delay.
|
|
194
|
-
const ins = {
|
|
195
|
-
id,
|
|
196
|
-
name: input.name,
|
|
197
|
-
queue: input.queue ?? "default",
|
|
198
|
-
payload: JSON.stringify(input.payload ?? {}),
|
|
199
|
-
metadata: JSON.stringify(input.metadata ?? {}),
|
|
200
|
-
max_attempts: input.maxAttempts ?? 3,
|
|
201
|
-
priority: input.priority ?? 0,
|
|
202
|
-
delay_ms: input.runAtDelayMs ?? 0,
|
|
203
|
-
parent_id: input.parentId ?? null,
|
|
204
|
-
root_id: input.rootId ?? id,
|
|
205
|
-
correlation_id: input.correlationId ?? null,
|
|
206
|
-
};
|
|
207
|
-
const key = input.key ?? null;
|
|
208
|
-
const conflict = input.conflict ?? "reuse";
|
|
209
|
-
if (key === null)
|
|
210
|
-
return rowToTask((await this.run("insert_task", ins))[0]);
|
|
211
|
-
return this.tx(async (client) => {
|
|
212
|
-
const existing = (await this.runOn(client, "get_key", { key }));
|
|
213
|
-
if (existing.length) {
|
|
214
|
-
const exId = existing[0].task_id;
|
|
215
|
-
if (conflict === "reuse")
|
|
216
|
-
return rowToTask((await this.runOn(client, "get", { id: exId }))[0]);
|
|
217
|
-
if (conflict === "reject")
|
|
218
|
-
throw new AlreadyExists(key);
|
|
219
|
-
if (conflict === "replace") {
|
|
220
|
-
await this.runOn(client, "cancel", { id: exId });
|
|
221
|
-
const row = (await this.runOn(client, "insert_task", ins))[0];
|
|
222
|
-
await this.runOn(client, "upsert_key", { key, task_id: id });
|
|
223
|
-
return rowToTask(row);
|
|
224
|
-
}
|
|
225
|
-
throw new Error(`unknown conflict strategy: ${conflict}`);
|
|
226
|
-
}
|
|
227
|
-
const row = (await this.runOn(client, "insert_task", ins))[0];
|
|
228
|
-
await this.runOn(client, "upsert_key", { key, task_id: id });
|
|
229
|
-
return rowToTask(row);
|
|
230
|
-
});
|
|
231
|
-
}
|
|
232
|
-
async get(taskId) {
|
|
233
|
-
await this.ensure();
|
|
234
|
-
const rows = await this.run("get", { id: taskId });
|
|
235
|
-
return rows.length ? rowToTask(rows[0]) : null;
|
|
236
|
-
}
|
|
237
|
-
async getByKey(key) {
|
|
238
|
-
await this.ensure();
|
|
239
|
-
const rows = await this.run("get_by_key", { key });
|
|
240
|
-
return rows.length ? rowToTask(rows[0]) : null;
|
|
241
|
-
}
|
|
242
|
-
async list(input = {}) {
|
|
243
|
-
await this.ensure();
|
|
244
|
-
const rows = await this.run("list", {
|
|
245
|
-
status: input.status ?? null,
|
|
246
|
-
queue: input.queue ?? null,
|
|
247
|
-
name: input.name ?? null,
|
|
248
|
-
root_id: input.rootId ?? null,
|
|
249
|
-
correlation_id: input.correlationId ?? null,
|
|
250
|
-
limit: input.limit ?? 100,
|
|
251
|
-
offset: input.offset ?? 0,
|
|
252
|
-
});
|
|
253
|
-
return rows.map(rowToTask);
|
|
254
|
-
}
|
|
255
|
-
async cancel(taskId) {
|
|
256
|
-
await this.ensure();
|
|
257
|
-
const rows = await this.run("cancel", { id: taskId });
|
|
258
|
-
return rows.length ? rowToTask(rows[0]) : null;
|
|
259
|
-
}
|
|
260
|
-
async cancelByKey(key) {
|
|
261
|
-
await this.ensure();
|
|
262
|
-
return this.tx(async (client) => {
|
|
263
|
-
const existing = (await this.runOn(client, "get_key", { key }));
|
|
264
|
-
if (!existing.length)
|
|
265
|
-
return null;
|
|
266
|
-
const rows = await this.runOn(client, "cancel", { id: existing[0].task_id });
|
|
267
|
-
return rows.length ? rowToTask(rows[0]) : null;
|
|
268
|
-
});
|
|
269
|
-
}
|
|
270
|
-
async retry(taskId, opts = {}) {
|
|
271
|
-
await this.ensure();
|
|
272
|
-
const rows = await this.run("retry", { id: taskId, reset_attempt: opts.resetAttempt ?? false });
|
|
273
|
-
return rows.length ? rowToTask(rows[0]) : null;
|
|
274
|
-
}
|
|
275
|
-
async retryByKey(key, opts = {}) {
|
|
276
|
-
await this.ensure();
|
|
277
|
-
return this.tx(async (client) => {
|
|
278
|
-
const existing = (await this.runOn(client, "get_key", { key }));
|
|
279
|
-
if (!existing.length)
|
|
280
|
-
return null;
|
|
281
|
-
const rows = await this.runOn(client, "retry", {
|
|
282
|
-
id: existing[0].task_id,
|
|
283
|
-
reset_attempt: opts.resetAttempt ?? false,
|
|
284
|
-
});
|
|
285
|
-
return rows.length ? rowToTask(rows[0]) : null;
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
// ------------------------------------------------------------- worker side
|
|
289
|
-
async claim(input) {
|
|
290
|
-
await this.ensure();
|
|
291
|
-
// No claimable_probe: PG readers don't block writers, so a plain transaction
|
|
292
|
-
// (recover then claim) is cheap even when idle. FOR UPDATE SKIP LOCKED in
|
|
293
|
-
// claim.sql gives true concurrent, non-contending dispatch.
|
|
294
|
-
return this.tx(async (client) => {
|
|
295
|
-
await this.runOn(client, "recover_leases", { lease_expired_error: LEASE_EXPIRED_ERROR_JSON });
|
|
296
|
-
const rows = await this.runOn(client, "claim", {
|
|
297
|
-
queues: input.queues,
|
|
298
|
-
worker_id: input.workerId,
|
|
299
|
-
lease_ms: input.leaseMs ?? 30_000,
|
|
300
|
-
limit: input.limit ?? 1,
|
|
301
|
-
});
|
|
302
|
-
return rows.map(rowToTask);
|
|
303
|
-
});
|
|
304
|
-
}
|
|
305
|
-
async heartbeat(input) {
|
|
306
|
-
await this.ensure();
|
|
307
|
-
return this.ownedWrite("heartbeat", input.taskId, {
|
|
308
|
-
id: input.taskId,
|
|
309
|
-
worker_id: input.workerId,
|
|
310
|
-
lease_ms: input.leaseMs ?? 30_000,
|
|
311
|
-
});
|
|
312
|
-
}
|
|
313
|
-
async progress(input) {
|
|
314
|
-
await this.ensure();
|
|
315
|
-
return this.ownedWrite("progress", input.taskId, {
|
|
316
|
-
id: input.taskId,
|
|
317
|
-
worker_id: input.workerId,
|
|
318
|
-
progress: input.progress,
|
|
319
|
-
message: input.message,
|
|
320
|
-
});
|
|
321
|
-
}
|
|
322
|
-
async succeed(input) {
|
|
323
|
-
await this.ensure();
|
|
324
|
-
return this.ownedWrite("succeed", input.taskId, {
|
|
325
|
-
id: input.taskId,
|
|
326
|
-
worker_id: input.workerId,
|
|
327
|
-
result: input.result == null ? null : JSON.stringify(input.result),
|
|
328
|
-
message: null,
|
|
329
|
-
});
|
|
330
|
-
}
|
|
331
|
-
async complete(input) {
|
|
332
|
-
await this.ensure();
|
|
333
|
-
return this.ownedWrite("complete", input.taskId, {
|
|
334
|
-
id: input.taskId,
|
|
335
|
-
worker_id: input.workerId,
|
|
336
|
-
result: input.result == null ? null : JSON.stringify(input.result),
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
async fail(input) {
|
|
340
|
-
await this.ensure();
|
|
341
|
-
return this.ownedWrite("fail", input.taskId, {
|
|
342
|
-
id: input.taskId,
|
|
343
|
-
worker_id: input.workerId,
|
|
344
|
-
error: JSON.stringify(input.error ?? {}),
|
|
345
|
-
retryable: input.retryable !== false,
|
|
346
|
-
delay_ms: input.delayMs ?? 0,
|
|
347
|
-
});
|
|
348
|
-
}
|
|
349
372
|
}
|
package/dist/store/sqlite.d.ts
CHANGED
|
@@ -1,20 +1,25 @@
|
|
|
1
|
-
import { type
|
|
2
|
-
import type { ListInput, SubmitInput, TaskStore } from "./base.js";
|
|
1
|
+
import { type Fetch, type Params, TaskStore } from "./base.js";
|
|
3
2
|
/**
|
|
4
|
-
* SQLiteStore —
|
|
3
|
+
* SQLiteStore — the SQLite dialect of the shared cairnq-protocol SQL.
|
|
5
4
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
5
|
+
* Everything protocol-shaped lives in TaskStore; this file is only what SQLite
|
|
6
|
+
* does differently: better-sqlite3's synchronous driver, BEGIN IMMEDIATE
|
|
7
|
+
* transactions, a read-only probe in front of the write lock, and time supplied
|
|
8
|
+
* by the SDK (`:now_ms`) rather than by the database.
|
|
9
|
+
*
|
|
10
|
+
* The driver being synchronous suits SQLite's single writer: claim is one short
|
|
11
|
+
* transaction, the handler runs outside any transaction, and
|
|
12
|
+
* progress/heartbeat/succeed/fail are each their own short write. Cross-process
|
|
13
|
+
* contention is absorbed by busy_timeout.
|
|
11
14
|
*/
|
|
12
|
-
export declare class SQLiteStore
|
|
15
|
+
export declare class SQLiteStore extends TaskStore {
|
|
13
16
|
private readonly path;
|
|
14
17
|
private readonly opts;
|
|
15
18
|
private db;
|
|
16
19
|
private stmts;
|
|
17
20
|
private readonly statements;
|
|
21
|
+
/** This store's entry in `fileLocks` — see there for why it is per-database. */
|
|
22
|
+
private readonly lockKey;
|
|
18
23
|
constructor(path: string, opts?: {
|
|
19
24
|
busyTimeoutMs?: number;
|
|
20
25
|
});
|
|
@@ -22,56 +27,26 @@ export declare class SQLiteStore implements TaskStore {
|
|
|
22
27
|
close(): Promise<void>;
|
|
23
28
|
private ensure;
|
|
24
29
|
private applyMigrations;
|
|
25
|
-
private checkVersion;
|
|
26
30
|
private readProtocolVersion;
|
|
27
31
|
protocolVersion(): Promise<number>;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}): Promise<Task[]>;
|
|
49
|
-
heartbeat(input: {
|
|
50
|
-
taskId: string;
|
|
51
|
-
workerId: string;
|
|
52
|
-
leaseMs?: number;
|
|
53
|
-
}): Promise<Task>;
|
|
54
|
-
progress(input: {
|
|
55
|
-
taskId: string;
|
|
56
|
-
workerId: string;
|
|
57
|
-
progress: number | null;
|
|
58
|
-
message: string | null;
|
|
59
|
-
}): Promise<Task>;
|
|
60
|
-
succeed(input: {
|
|
61
|
-
taskId: string;
|
|
62
|
-
workerId: string;
|
|
63
|
-
result: unknown;
|
|
64
|
-
}): Promise<Task>;
|
|
65
|
-
complete(input: {
|
|
66
|
-
taskId: string;
|
|
67
|
-
workerId: string;
|
|
68
|
-
result: unknown;
|
|
69
|
-
}): Promise<Task>;
|
|
70
|
-
fail(input: {
|
|
71
|
-
taskId: string;
|
|
72
|
-
workerId: string;
|
|
73
|
-
error: unknown;
|
|
74
|
-
retryable?: boolean;
|
|
75
|
-
delayMs?: number;
|
|
76
|
-
}): Promise<Task>;
|
|
32
|
+
/**
|
|
33
|
+
* Adapt the dialect-neutral parameters to what this statement binds.
|
|
34
|
+
*
|
|
35
|
+
* SQLite statements carry no DB clock, so every absolute `*_ms` is derived here
|
|
36
|
+
* from one `now`, and booleans cross as 0/1. The result is narrowed to the
|
|
37
|
+
* names the SQL actually uses, which is what makes it safe for a caller to pass
|
|
38
|
+
* one superset of parameters for both dialects.
|
|
39
|
+
*
|
|
40
|
+
* Each derivation writes a name Postgres does not use (`lease_until_ms` from
|
|
41
|
+
* `lease_ms`, and so on), so a statement binds one or the other, never both —
|
|
42
|
+
* which is why the derived values can be computed unconditionally and left for
|
|
43
|
+
* the narrowing step to discard.
|
|
44
|
+
*/
|
|
45
|
+
private bind;
|
|
46
|
+
private runNow;
|
|
47
|
+
/** Serialize an operation against every other operation on this database. */
|
|
48
|
+
private withLock;
|
|
49
|
+
protected fetch(name: string, params: Params): Promise<any[]>;
|
|
50
|
+
protected tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
|
|
51
|
+
protected hasClaimableWork(params: Params): Promise<boolean>;
|
|
77
52
|
}
|