cairnq 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -2
- package/dist/_protocol/sql/postgres/installations.sql +33 -0
- package/dist/client.d.ts +19 -4
- package/dist/client.js +20 -5
- package/dist/context.d.ts +29 -2
- package/dist/context.js +44 -7
- package/dist/errors.d.ts +17 -0
- package/dist/errors.js +20 -0
- package/dist/index.d.ts +6 -2
- package/dist/index.js +4 -1
- package/dist/models.js +27 -0
- package/dist/store/base.d.ts +99 -0
- package/dist/store/base.js +87 -11
- package/dist/store/pg-executor.d.ts +78 -0
- package/dist/store/pg-executor.js +27 -0
- package/dist/store/pg-pool.d.ts +16 -0
- package/dist/store/pg-pool.js +148 -0
- package/dist/store/postgres.d.ts +64 -13
- package/dist/store/postgres.js +191 -147
- package/dist/store/sqlite.js +20 -2
- package/dist/worker.d.ts +10 -5
- package/dist/worker.js +16 -13
- package/package.json +6 -4
- package/src/client.ts +33 -6
- package/src/context.ts +46 -5
- package/src/errors.ts +21 -0
- package/src/index.ts +13 -1
- package/src/models.ts +27 -0
- package/src/store/base.ts +150 -0
- package/src/store/pg-executor.ts +91 -0
- package/src/store/pg-pool.ts +157 -0
- package/src/store/postgres.ts +205 -141
- package/src/store/sqlite.ts +23 -2
- package/src/worker.ts +18 -14
package/dist/store/postgres.js
CHANGED
|
@@ -1,45 +1,12 @@
|
|
|
1
|
+
import { SchemaMismatch } from "../errors.js";
|
|
1
2
|
import { loadMigrations, loadStatements } from "../sql.js";
|
|
2
3
|
import { checkProtocolVersion, COMMENT, NAMED, statementParams, TaskStore, } from "./base.js";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
let pgModule = null;
|
|
6
|
-
async function loadPg() {
|
|
7
|
-
if (pgModule)
|
|
8
|
-
return pgModule;
|
|
9
|
-
let mod;
|
|
10
|
-
try {
|
|
11
|
-
mod = (await import("pg"));
|
|
12
|
-
}
|
|
13
|
-
catch {
|
|
14
|
-
throw new Error("PostgresStore requires the 'pg' package — install it (e.g. `npm i pg`)");
|
|
15
|
-
}
|
|
16
|
-
const pg = (mod.default ?? mod);
|
|
17
|
-
// Postgres returns bigint (int8, OID 20) as a string to avoid precision loss.
|
|
18
|
-
// Every cairnq bigint is an epoch-ms or a counter, all within Number's safe
|
|
19
|
-
// integer range, so parse to number once (globally) to match the Task model
|
|
20
|
-
// (*_ms typed as number, same as the SQLite SDK). Set before any query runs.
|
|
21
|
-
pg.types.setTypeParser(pg.types.builtins.INT8, (v) => (v == null ? null : Number(v)));
|
|
22
|
-
pgModule = pg;
|
|
23
|
-
return pg;
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Roll back on the way out of a failed transaction, without letting the rollback
|
|
27
|
-
* become the error the caller sees. A dropped connection fails both the statement
|
|
28
|
-
* and the rollback, and it is the first one that says what went wrong.
|
|
29
|
-
*/
|
|
30
|
-
async function rollbackQuietly(client) {
|
|
31
|
-
try {
|
|
32
|
-
await client.query("rollback");
|
|
33
|
-
}
|
|
34
|
-
catch {
|
|
35
|
-
// Already rolled back, or the connection is gone. Either way the original
|
|
36
|
-
// error is the one worth propagating.
|
|
37
|
-
}
|
|
38
|
-
}
|
|
4
|
+
import { ListenUnavailable } from "./pg-executor.js";
|
|
5
|
+
import { createPoolExecutor } from "./pg-pool.js";
|
|
39
6
|
// Notification channels, emitted by the 0003_notify trigger.
|
|
40
7
|
const QUEUED_CHANNEL = "cairnq_queued";
|
|
41
8
|
const DONE_CHANNEL = "cairnq_done";
|
|
42
|
-
// Backoff between attempts to (re)
|
|
9
|
+
// Backoff between attempts to (re)establish the LISTEN subscription after a
|
|
43
10
|
// transient failure. Doubles per failure up to the cap; polling covers the gap.
|
|
44
11
|
const LISTENER_RETRY_MS = 1_000;
|
|
45
12
|
const LISTENER_RETRY_MAX_MS = 30_000;
|
|
@@ -82,27 +49,38 @@ export function toPositional(sql, params) {
|
|
|
82
49
|
* PostgresStore — the Postgres dialect of the shared cairnq-protocol SQL.
|
|
83
50
|
*
|
|
84
51
|
* Everything protocol-shaped lives in TaskStore; this file is only what Postgres
|
|
85
|
-
* does differently:
|
|
86
|
-
* the DB clock (`now()`) instead of from the SDK,
|
|
87
|
-
* multi-host — unlike SQLite it coordinates API
|
|
88
|
-
* machines, with no shared clock to agree on. claim
|
|
89
|
-
* and needs no claimable_probe, because PG readers
|
|
90
|
-
* columns are jsonb (bound as JSON text, read back as
|
|
52
|
+
* does differently: `:name` -> `$n` translation, the migration ledger, LISTEN
|
|
53
|
+
* policy, and time taken from the DB clock (`now()`) instead of from the SDK,
|
|
54
|
+
* which is what makes this backend multi-host — unlike SQLite it coordinates API
|
|
55
|
+
* and worker processes across machines, with no shared clock to agree on. claim
|
|
56
|
+
* uses FOR UPDATE SKIP LOCKED and needs no claimable_probe, because PG readers
|
|
57
|
+
* don't block writers. JSON columns are jsonb (bound as JSON text, read back as
|
|
58
|
+
* objects by rowToTask).
|
|
59
|
+
*
|
|
60
|
+
* What it deliberately does NOT own is the connection. Given a DSN it builds a
|
|
61
|
+
* `pg` pool (see pg-pool.ts); given a PgExecutor it runs inside the caller's
|
|
62
|
+
* session instead — no second driver, no second pool, and the caller's writes and
|
|
63
|
+
* cairnq's can share one transaction.
|
|
91
64
|
*/
|
|
92
65
|
export class PostgresStore extends TaskStore {
|
|
93
|
-
dsn;
|
|
94
66
|
opts;
|
|
95
|
-
|
|
67
|
+
dsn;
|
|
68
|
+
/** The caller's executor, if one was injected — never closed by this store. */
|
|
69
|
+
provided;
|
|
70
|
+
/** Set once migrations have run and the protocol version checked out. */
|
|
71
|
+
executor = null;
|
|
96
72
|
connecting = null;
|
|
97
73
|
statements;
|
|
98
74
|
// ------------------------------------------------------- LISTEN/NOTIFY state
|
|
99
|
-
//
|
|
100
|
-
// the claimWake/taskDoneWake contract on TaskStore).
|
|
101
|
-
// keep it silently degrades the store to the base
|
|
75
|
+
// The executor subscribes one dedicated connection to both channels (see
|
|
76
|
+
// 0003_notify.sql and the claimWake/taskDoneWake contract on TaskStore).
|
|
77
|
+
// Failure to establish or keep it silently degrades the store to the base
|
|
78
|
+
// class's plain polling. Held as the unsubscribe function, not the connection:
|
|
79
|
+
// whose connection it is, is the executor's business.
|
|
102
80
|
listener = null;
|
|
103
81
|
listenerConnecting = null;
|
|
104
|
-
/** LISTEN is off for good: the store was closed,
|
|
105
|
-
*
|
|
82
|
+
/** LISTEN is off for good: the store was closed, the executor does not support
|
|
83
|
+
* it, or the server refused it (e.g. a transaction-mode pooler) —
|
|
106
84
|
* deterministic, so retrying would fail the same way every time. */
|
|
107
85
|
listenerUnavailable = false;
|
|
108
86
|
/** A failure to even connect is transient (network blip, server restarting):
|
|
@@ -115,10 +93,19 @@ export class PostgresStore extends TaskStore {
|
|
|
115
93
|
pendingQueues = new Set();
|
|
116
94
|
/** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
|
|
117
95
|
waiters = new Map();
|
|
118
|
-
|
|
96
|
+
/** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
|
|
97
|
+
* consumes the notification, a subscriber is standing and only observes. */
|
|
98
|
+
subscribers = new Set();
|
|
99
|
+
/**
|
|
100
|
+
* `source` is either a libpq connection string — this store then owns a `pg`
|
|
101
|
+
* pool and requires the optional `pg` package — or a PgExecutor the caller
|
|
102
|
+
* already has, which this store uses and never closes.
|
|
103
|
+
*/
|
|
104
|
+
constructor(source, opts = {}) {
|
|
119
105
|
super();
|
|
120
|
-
this.dsn = dsn;
|
|
121
106
|
this.opts = opts;
|
|
107
|
+
this.dsn = typeof source === "string" ? source : null;
|
|
108
|
+
this.provided = typeof source === "string" ? null : source;
|
|
122
109
|
this.statements = loadStatements("postgres");
|
|
123
110
|
}
|
|
124
111
|
async connect() {
|
|
@@ -127,18 +114,20 @@ export class PostgresStore extends TaskStore {
|
|
|
127
114
|
async close() {
|
|
128
115
|
this.listenerUnavailable = true; // no revival after close
|
|
129
116
|
this.dropListener();
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
117
|
+
const executor = this.executor;
|
|
118
|
+
this.executor = null;
|
|
119
|
+
this.connecting = null;
|
|
120
|
+
// An injected executor belongs to the caller, whose other work would not
|
|
121
|
+
// survive cairnq closing it.
|
|
122
|
+
if (executor && !this.provided)
|
|
123
|
+
await executor.close();
|
|
136
124
|
}
|
|
137
125
|
async ensure() {
|
|
138
|
-
if (this.
|
|
126
|
+
if (this.executor)
|
|
139
127
|
return;
|
|
140
|
-
// Cache the in-flight connect so concurrent calls share one
|
|
141
|
-
// clear it so a later call retries instead of re-awaiting a
|
|
128
|
+
// Cache the in-flight connect so concurrent calls share one executor. On
|
|
129
|
+
// failure, clear it so a later call retries instead of re-awaiting a
|
|
130
|
+
// rejected promise.
|
|
142
131
|
if (!this.connecting) {
|
|
143
132
|
this.connecting = this.doConnect().catch((e) => {
|
|
144
133
|
this.connecting = null;
|
|
@@ -148,68 +137,105 @@ export class PostgresStore extends TaskStore {
|
|
|
148
137
|
await this.connecting;
|
|
149
138
|
}
|
|
150
139
|
async doConnect() {
|
|
151
|
-
const
|
|
152
|
-
|
|
140
|
+
const executor = this.provided ??
|
|
141
|
+
(await createPoolExecutor(this.dsn, { max: this.opts.max, schema: this.opts.schema }));
|
|
153
142
|
try {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
finally {
|
|
160
|
-
client.release();
|
|
161
|
-
}
|
|
143
|
+
// Before migrations, which would otherwise create the very installation
|
|
144
|
+
// this is trying to warn about.
|
|
145
|
+
await this.checkSchema(executor);
|
|
146
|
+
await this.applyMigrations(executor);
|
|
147
|
+
checkProtocolVersion(await this.readProtocolVersion(executor));
|
|
162
148
|
}
|
|
163
149
|
catch (e) {
|
|
164
|
-
|
|
150
|
+
// Never leak an executor we created; never close one we were handed.
|
|
151
|
+
if (!this.provided)
|
|
152
|
+
await executor.close().catch(() => { });
|
|
165
153
|
throw e;
|
|
166
154
|
}
|
|
167
|
-
this.
|
|
168
|
-
// Warm the LISTEN
|
|
155
|
+
this.executor = executor; // publish only a fully-migrated, version-checked executor
|
|
156
|
+
// Warm the LISTEN subscription in the background so the first idle sleep is
|
|
169
157
|
// already wakeable. Fire-and-forget: failure just means polling.
|
|
170
158
|
this.listenerReady();
|
|
171
159
|
}
|
|
172
|
-
|
|
173
|
-
|
|
160
|
+
/**
|
|
161
|
+
* Refuse a connection pointed somewhere other than the deployment's cairnq.
|
|
162
|
+
*
|
|
163
|
+
* Two shapes, because `schema` means "the schema cairnq's tables live in" and
|
|
164
|
+
* cairnq can either arrange that (it built the connection) or only check it
|
|
165
|
+
* (the caller's executor did):
|
|
166
|
+
*
|
|
167
|
+
* - `schema` configured -> assert the connection actually resolves there. On
|
|
168
|
+
* the DSN path this is a cheap self-check; on an injected executor it is the
|
|
169
|
+
* only way to state the expectation at all.
|
|
170
|
+
* - `schema` not configured -> the dangerous case is being about to create a
|
|
171
|
+
* SECOND installation while one already exists elsewhere in this database,
|
|
172
|
+
* which is exactly what a mismatched pair of SDKs does. Joining an existing
|
|
173
|
+
* installation is fine no matter what else is around, so the check is
|
|
174
|
+
* deliberately narrow: it fires only when this schema has no cairnq and some
|
|
175
|
+
* other schema does.
|
|
176
|
+
*
|
|
177
|
+
* That narrowness is what keeps it from crying wolf. Two applications each
|
|
178
|
+
* running their own cairnq in their own schema are legitimate; the second one
|
|
179
|
+
* to be set up trips this once, and saying `schema` explicitly — which such a
|
|
180
|
+
* deployment should be doing anyway — is both the fix and the confirmation.
|
|
181
|
+
*/
|
|
182
|
+
async checkSchema(executor) {
|
|
183
|
+
// One row per installation; the statement's LEFT JOIN guarantees at least
|
|
184
|
+
// one, so current_schema is readable even where cairnq lives nowhere yet.
|
|
185
|
+
const rows = await executor.query(this.statements.installations, []);
|
|
186
|
+
const current = rows[0]?.current_schema ?? null;
|
|
187
|
+
const installations = rows
|
|
188
|
+
.map((r) => r.schema)
|
|
189
|
+
.filter((s) => s != null);
|
|
190
|
+
const wanted = this.opts.schema;
|
|
191
|
+
if (wanted != null) {
|
|
192
|
+
if (current !== wanted) {
|
|
193
|
+
throw new SchemaMismatch(`cairnq is configured for schema ${JSON.stringify(wanted)} but this ` +
|
|
194
|
+
`connection resolves to ${JSON.stringify(current)} — check the ` +
|
|
195
|
+
`connection's search_path`);
|
|
196
|
+
}
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
// A search_path naming nothing that exists: there is no "here" to compare
|
|
200
|
+
// against, and the migrations are about to fail with a clearer message.
|
|
201
|
+
if (current === null)
|
|
202
|
+
return;
|
|
203
|
+
if (installations.length === 0 || installations.includes(current))
|
|
204
|
+
return;
|
|
205
|
+
throw new SchemaMismatch(`cairnq tables already exist in schema ${installations.map((s) => JSON.stringify(s)).join(", ")} ` +
|
|
206
|
+
`of this database, but this connection resolves to ${JSON.stringify(current)}, where there are ` +
|
|
207
|
+
`none. Connecting would create a second, parallel installation that the other one can never see ` +
|
|
208
|
+
`— an API and a worker split this way agree about everything except where, and no task crosses. ` +
|
|
209
|
+
`Point this process at the same schema (\`schema\` option, or \`options=-c search_path=...\` in the ` +
|
|
210
|
+
`DSN), or pass \`schema\` explicitly to confirm a separate installation is what you meant.`);
|
|
211
|
+
}
|
|
212
|
+
async applyMigrations(executor) {
|
|
213
|
+
await executor.exec("create table if not exists cairnq_migrations " +
|
|
174
214
|
"(name text primary key, applied_at_ms bigint not null)");
|
|
175
215
|
for (const { name, sql } of loadMigrations("postgres")) {
|
|
176
216
|
// Check and apply inside one transaction, with the table lock taken up
|
|
177
217
|
// front: two processes cold-starting together would otherwise both see a
|
|
178
218
|
// migration as unapplied and both run it.
|
|
179
|
-
|
|
180
|
-
await
|
|
181
|
-
await
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
if (applied.rowCount === 0) {
|
|
186
|
-
await client.query(sql); // multi-statement DDL (simple-query, no params)
|
|
187
|
-
await client.query("insert into cairnq_migrations (name, applied_at_ms) values " +
|
|
219
|
+
await executor.tx(async (s) => {
|
|
220
|
+
await s.exec("lock table cairnq_migrations in exclusive mode");
|
|
221
|
+
const applied = await s.query("select 1 from cairnq_migrations where name = $1", [name]);
|
|
222
|
+
if (applied.length === 0) {
|
|
223
|
+
await s.exec(sql); // multi-statement DDL (simple-query, no params)
|
|
224
|
+
await s.query("insert into cairnq_migrations (name, applied_at_ms) values " +
|
|
188
225
|
"($1, (extract(epoch from now()) * 1000)::bigint)", [name]);
|
|
189
226
|
}
|
|
190
|
-
|
|
191
|
-
}
|
|
192
|
-
catch (e) {
|
|
193
|
-
await rollbackQuietly(client);
|
|
194
|
-
throw e;
|
|
195
|
-
}
|
|
227
|
+
});
|
|
196
228
|
}
|
|
197
229
|
}
|
|
198
|
-
// Takes an explicit
|
|
199
|
-
// this cannot go through fetch(). The statement binds nothing
|
|
200
|
-
async readProtocolVersion(
|
|
201
|
-
const
|
|
202
|
-
return
|
|
230
|
+
// Takes an explicit session: during doConnect the executor is not published
|
|
231
|
+
// yet, so this cannot go through fetch(). The statement binds nothing.
|
|
232
|
+
async readProtocolVersion(s) {
|
|
233
|
+
const rows = await s.query(this.statements.protocol_version, []);
|
|
234
|
+
return rows.length ? Number(rows[0].value) : 0;
|
|
203
235
|
}
|
|
204
236
|
async protocolVersion() {
|
|
205
237
|
await this.ensure();
|
|
206
|
-
|
|
207
|
-
try {
|
|
208
|
-
return await this.readProtocolVersion(client);
|
|
209
|
-
}
|
|
210
|
-
finally {
|
|
211
|
-
client.release();
|
|
212
|
-
}
|
|
238
|
+
return this.readProtocolVersion(this.executor);
|
|
213
239
|
}
|
|
214
240
|
// ------------------------------------------------------------ wake channel
|
|
215
241
|
async claimWake(queues, timeoutMs) {
|
|
@@ -232,6 +258,15 @@ export class PostgresStore extends TaskStore {
|
|
|
232
258
|
await this.wakeOn("queued", remaining);
|
|
233
259
|
}
|
|
234
260
|
}
|
|
261
|
+
subscribePush(onSignal) {
|
|
262
|
+
this.subscribers.add(onSignal);
|
|
263
|
+
this.listenerReady(); // an API-side watcher is often the only thing asking
|
|
264
|
+
return () => void this.subscribers.delete(onSignal);
|
|
265
|
+
}
|
|
266
|
+
warmPush() {
|
|
267
|
+
if (this.subscribers.size > 0)
|
|
268
|
+
this.listenerReady();
|
|
269
|
+
}
|
|
235
270
|
taskDoneWake(taskId, timeoutMs) {
|
|
236
271
|
if (!this.listenerReady())
|
|
237
272
|
return super.taskDoneWake(taskId, timeoutMs);
|
|
@@ -258,7 +293,7 @@ export class PostgresStore extends TaskStore {
|
|
|
258
293
|
peers.add(waiter);
|
|
259
294
|
});
|
|
260
295
|
}
|
|
261
|
-
/** True once the LISTEN
|
|
296
|
+
/** True once the LISTEN subscription is up; starts establishing it otherwise
|
|
262
297
|
* (respecting the transient-failure backoff). Callers fall back to plain
|
|
263
298
|
* polling until it is ready (or forever, if it can't be established) —
|
|
264
299
|
* correctness never depends on it. */
|
|
@@ -272,38 +307,38 @@ export class PostgresStore extends TaskStore {
|
|
|
272
307
|
}
|
|
273
308
|
async startListener() {
|
|
274
309
|
try {
|
|
275
|
-
const
|
|
276
|
-
//
|
|
277
|
-
|
|
278
|
-
|
|
310
|
+
const executor = this.executor ?? this.provided;
|
|
311
|
+
// Not connected yet: transient by definition — doConnect calls back in.
|
|
312
|
+
if (!executor)
|
|
313
|
+
return;
|
|
314
|
+
if (!executor.listen) {
|
|
315
|
+
this.listenerUnavailable = true; // this executor will never push
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
let stop;
|
|
279
319
|
try {
|
|
280
|
-
await
|
|
320
|
+
stop = await executor.listen([QUEUED_CHANNEL, DONE_CHANNEL], (channel, payload) => this.onNotification(channel, payload),
|
|
321
|
+
// A dropped listener degrades to polling; the next wake reconnects.
|
|
322
|
+
() => this.dropListener());
|
|
281
323
|
}
|
|
282
|
-
catch {
|
|
283
|
-
|
|
324
|
+
catch (e) {
|
|
325
|
+
if (e instanceof ListenUnavailable) {
|
|
326
|
+
// Deterministic (e.g. a transaction-mode pooler): off for good rather
|
|
327
|
+
// than a reconnect loop that cannot succeed. Polling covers it.
|
|
328
|
+
this.listenerUnavailable = true;
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
// Could not establish it — transient. Schedule a backed-off retry;
|
|
284
332
|
// polling covers the gap.
|
|
285
333
|
this.listenerRetryAt = Date.now() + this.listenerBackoffMs;
|
|
286
334
|
this.listenerBackoffMs = Math.min(LISTENER_RETRY_MAX_MS, this.listenerBackoffMs * 2);
|
|
287
335
|
return;
|
|
288
336
|
}
|
|
289
|
-
client.on("notification", (msg) => this.onNotification(msg.channel, msg.payload));
|
|
290
|
-
// A dropped listener degrades to polling; the next wake call reconnects.
|
|
291
|
-
client.on("error", () => this.dropListener());
|
|
292
|
-
try {
|
|
293
|
-
await client.query(`listen ${QUEUED_CHANNEL}; listen ${DONE_CHANNEL}`);
|
|
294
|
-
}
|
|
295
|
-
catch {
|
|
296
|
-
// Connected, but LISTEN was refused (e.g. a transaction-mode pooler) —
|
|
297
|
-
// deterministic, so off for good. Polling covers it.
|
|
298
|
-
this.listenerUnavailable = true;
|
|
299
|
-
void client.end().catch(() => { });
|
|
300
|
-
return;
|
|
301
|
-
}
|
|
302
337
|
if (this.listenerUnavailable) {
|
|
303
|
-
|
|
338
|
+
stop(); // closed while we were subscribing
|
|
304
339
|
return;
|
|
305
340
|
}
|
|
306
|
-
this.listener =
|
|
341
|
+
this.listener = stop;
|
|
307
342
|
this.listenerBackoffMs = LISTENER_RETRY_MS;
|
|
308
343
|
}
|
|
309
344
|
catch {
|
|
@@ -320,9 +355,11 @@ export class PostgresStore extends TaskStore {
|
|
|
320
355
|
if (channel === QUEUED_CHANNEL && payload) {
|
|
321
356
|
this.pendingQueues.add(payload);
|
|
322
357
|
key = "queued";
|
|
358
|
+
this.publish({ reason: "queued", queue: payload });
|
|
323
359
|
}
|
|
324
360
|
else if (channel === DONE_CHANNEL && payload) {
|
|
325
361
|
key = `done:${payload}`;
|
|
362
|
+
this.publish({ reason: "done", taskId: payload });
|
|
326
363
|
}
|
|
327
364
|
else {
|
|
328
365
|
return;
|
|
@@ -333,11 +370,23 @@ export class PostgresStore extends TaskStore {
|
|
|
333
370
|
for (const w of set)
|
|
334
371
|
w();
|
|
335
372
|
}
|
|
373
|
+
/** Hand a notification to every watch() subscriber. A throwing subscriber is
|
|
374
|
+
* its own problem: it must not cost the others their signal, nor take down the
|
|
375
|
+
* listener connection that delivered it. */
|
|
376
|
+
publish(signal) {
|
|
377
|
+
for (const s of [...this.subscribers]) {
|
|
378
|
+
try {
|
|
379
|
+
s(signal);
|
|
380
|
+
}
|
|
381
|
+
catch {
|
|
382
|
+
// Deliberately swallowed — see above.
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
336
386
|
dropListener() {
|
|
337
|
-
const
|
|
387
|
+
const stop = this.listener;
|
|
338
388
|
this.listener = null;
|
|
339
|
-
|
|
340
|
-
void client.end().catch(() => { });
|
|
389
|
+
stop?.();
|
|
341
390
|
// Release everyone promptly; their fallback poll takes over.
|
|
342
391
|
for (const set of [...this.waiters.values()])
|
|
343
392
|
for (const w of [...set])
|
|
@@ -347,26 +396,21 @@ export class PostgresStore extends TaskStore {
|
|
|
347
396
|
async fetch(name, params) {
|
|
348
397
|
await this.ensure();
|
|
349
398
|
const { text, values } = toPositional(this.statements[name], params);
|
|
350
|
-
return
|
|
399
|
+
return this.executor.query(text, values);
|
|
351
400
|
}
|
|
352
401
|
async tx(fn) {
|
|
353
402
|
await this.ensure();
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
throw e;
|
|
367
|
-
}
|
|
368
|
-
finally {
|
|
369
|
-
client.release();
|
|
370
|
-
}
|
|
403
|
+
return this.executor.tx((s) => fn(this.boundFetch(s)));
|
|
404
|
+
}
|
|
405
|
+
async txWithSession(fn) {
|
|
406
|
+
await this.ensure();
|
|
407
|
+
return this.executor.tx((s) => fn(this.boundFetch(s), s));
|
|
408
|
+
}
|
|
409
|
+
/** A Fetch that runs the protocol's statements on one particular session. */
|
|
410
|
+
boundFetch(s) {
|
|
411
|
+
return async (name, params) => {
|
|
412
|
+
const { text, values } = toPositional(this.statements[name], params);
|
|
413
|
+
return s.query(text, values);
|
|
414
|
+
};
|
|
371
415
|
}
|
|
372
416
|
}
|
package/dist/store/sqlite.js
CHANGED
|
@@ -1,9 +1,27 @@
|
|
|
1
1
|
import { mkdirSync } from "node:fs";
|
|
2
2
|
import { dirname, resolve } from "node:path";
|
|
3
|
-
import
|
|
3
|
+
import { createRequire } from "node:module";
|
|
4
4
|
import { nowMs } from "../ids.js";
|
|
5
5
|
import { loadMigrations, loadStatements } from "../sql.js";
|
|
6
6
|
import { checkProtocolVersion, COMMENT, statementParams, TaskStore, } from "./base.js";
|
|
7
|
+
// `better-sqlite3` is an optional dependency, matching `pg` on the Postgres side:
|
|
8
|
+
// a Postgres-only deployment should not have to build a native module it never
|
|
9
|
+
// loads, and importing this file must not pull one in. Required (not imported)
|
|
10
|
+
// because ensure() is synchronous — the open path applies migrations and cannot
|
|
11
|
+
// await — and createRequire gives a synchronous load from ESM.
|
|
12
|
+
const require = createRequire(import.meta.url);
|
|
13
|
+
let sqliteModule = null;
|
|
14
|
+
function loadSqlite() {
|
|
15
|
+
if (sqliteModule)
|
|
16
|
+
return sqliteModule;
|
|
17
|
+
try {
|
|
18
|
+
sqliteModule = require("better-sqlite3");
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
throw new Error("SQLiteStore requires the 'better-sqlite3' package — install it (e.g. `npm i better-sqlite3`)");
|
|
22
|
+
}
|
|
23
|
+
return sqliteModule;
|
|
24
|
+
}
|
|
7
25
|
const WAL_RETRY_DELAY_MS = 50;
|
|
8
26
|
const WAL_RETRY_BUDGET_MS = 5_000;
|
|
9
27
|
const BUSY_RETRY_BASE_MS = 1;
|
|
@@ -221,7 +239,7 @@ export class SQLiteStore extends TaskStore {
|
|
|
221
239
|
const memory = isMemory(this.path);
|
|
222
240
|
if (!memory)
|
|
223
241
|
mkdirSync(dirname(this.path), { recursive: true });
|
|
224
|
-
const db = new
|
|
242
|
+
const db = new (loadSqlite())(this.path);
|
|
225
243
|
// Only the synchronous part of the open path gets a real busy_timeout: the WAL
|
|
226
244
|
// switch and the migrations cannot await a retry. See the class comment.
|
|
227
245
|
db.pragma(`busy_timeout = ${this.busyBudgetMs}`);
|
package/dist/worker.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { BackpressureOptions } from "./backpressure.js";
|
|
2
2
|
import { TaskContext } from "./context.js";
|
|
3
|
+
import type { PgExecutor } from "./store/pg-executor.js";
|
|
3
4
|
import type { TaskStore } from "./store/base.js";
|
|
4
5
|
import { type TaskDef } from "./task.js";
|
|
5
6
|
export { retryDelayMs, DEFAULT_RETRY_BACKOFF_MS, DEFAULT_RETRY_BACKOFF_MAX_MS } from "./backoff.js";
|
|
@@ -97,7 +98,7 @@ export declare class Worker {
|
|
|
97
98
|
private readonly queues;
|
|
98
99
|
private readonly opts;
|
|
99
100
|
private readonly handlers;
|
|
100
|
-
private readonly
|
|
101
|
+
private readonly ownId;
|
|
101
102
|
/** Payload bytes charged to running handlers — see maxInFlightBytes. */
|
|
102
103
|
private inFlightBytes;
|
|
103
104
|
/** Calls in flight, for the names that cap their own concurrency. */
|
|
@@ -130,13 +131,17 @@ export declare class Worker {
|
|
|
130
131
|
queues?: string[];
|
|
131
132
|
busyTimeoutMs?: number;
|
|
132
133
|
}): Worker;
|
|
133
|
-
/** Multi-host backend. `
|
|
134
|
-
* optional `pg` package
|
|
135
|
-
|
|
134
|
+
/** Multi-host backend. `source` is a libpq connection string — which requires
|
|
135
|
+
* the optional `pg` package — or a PgExecutor over a driver the application
|
|
136
|
+
* already runs, which cairnq then shares instead of opening a second pool. */
|
|
137
|
+
static postgres(source: string | PgExecutor, opts?: WorkerOptions & {
|
|
136
138
|
queues?: string[];
|
|
137
139
|
max?: number;
|
|
140
|
+
schema?: string;
|
|
138
141
|
}): Worker;
|
|
139
|
-
|
|
142
|
+
/** This worker's id — what `worker_id` on a running task points at, and what
|
|
143
|
+
* the Python SDK calls `worker_id` too. */
|
|
144
|
+
get workerId(): string;
|
|
140
145
|
task(handler: Handler): this;
|
|
141
146
|
task(name: string, handler: Handler): this;
|
|
142
147
|
task<P, R>(def: TaskDef<P, R>, handler: TypedHandler<P, R>): this;
|
package/dist/worker.js
CHANGED
|
@@ -72,7 +72,7 @@ export class Worker {
|
|
|
72
72
|
queues;
|
|
73
73
|
opts;
|
|
74
74
|
handlers = new Map();
|
|
75
|
-
|
|
75
|
+
ownId = newId("worker");
|
|
76
76
|
/** Payload bytes charged to running handlers — see maxInFlightBytes. */
|
|
77
77
|
inFlightBytes = 0;
|
|
78
78
|
/** Calls in flight, for the names that cap their own concurrency. */
|
|
@@ -133,16 +133,19 @@ export class Worker {
|
|
|
133
133
|
worker.ownsStore = true;
|
|
134
134
|
return worker;
|
|
135
135
|
}
|
|
136
|
-
/** Multi-host backend. `
|
|
137
|
-
* optional `pg` package
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
const
|
|
136
|
+
/** Multi-host backend. `source` is a libpq connection string — which requires
|
|
137
|
+
* the optional `pg` package — or a PgExecutor over a driver the application
|
|
138
|
+
* already runs, which cairnq then shares instead of opening a second pool. */
|
|
139
|
+
static postgres(source, opts = {}) {
|
|
140
|
+
const { queues = ["default"], max, schema, ...rest } = opts;
|
|
141
|
+
const worker = new Worker(new PostgresStore(source, { max, schema }), queues, rest);
|
|
141
142
|
worker.ownsStore = true;
|
|
142
143
|
return worker;
|
|
143
144
|
}
|
|
144
|
-
|
|
145
|
-
|
|
145
|
+
/** This worker's id — what `worker_id` on a running task points at, and what
|
|
146
|
+
* the Python SDK calls `worker_id` too. */
|
|
147
|
+
get workerId() {
|
|
148
|
+
return this.ownId;
|
|
146
149
|
}
|
|
147
150
|
task(arg, second, third) {
|
|
148
151
|
// Option form: (name | def, { batch?, concurrency?, resource? }, handler).
|
|
@@ -273,7 +276,7 @@ export class Worker {
|
|
|
273
276
|
return out;
|
|
274
277
|
}
|
|
275
278
|
context(task, leaseMs) {
|
|
276
|
-
return new TaskContext(this.store, task, this.
|
|
279
|
+
return new TaskContext(this.store, task, this.ownId, leaseMs, {
|
|
277
280
|
retryBackoffMs: this.backoffMs,
|
|
278
281
|
retryBackoffMaxMs: this.backoffMaxMs,
|
|
279
282
|
});
|
|
@@ -379,7 +382,7 @@ export class Worker {
|
|
|
379
382
|
// Only what this worker can run. Queues do not partition work by task
|
|
380
383
|
// name, so another worker's tasks would otherwise be claimed here and
|
|
381
384
|
// failed for want of a handler.
|
|
382
|
-
{ queues: this.queues, workerId: this.
|
|
385
|
+
{ queues: this.queues, workerId: this.ownId, leaseMs, names }, async (claim) => {
|
|
383
386
|
const drawn = [];
|
|
384
387
|
let left = free;
|
|
385
388
|
// Resource units this poll has already drawn. resourceCalls only
|
|
@@ -611,7 +614,7 @@ export class Worker {
|
|
|
611
614
|
try {
|
|
612
615
|
// complete (not succeed): finalizes as canceled if a cancel was requested
|
|
613
616
|
// while the handler ran, else succeeded.
|
|
614
|
-
await this.store.complete({ taskId: ctx.taskId, workerId: this.
|
|
617
|
+
await this.store.complete({ taskId: ctx.taskId, workerId: this.ownId, result });
|
|
615
618
|
ctx.markSettled();
|
|
616
619
|
}
|
|
617
620
|
catch (err) {
|
|
@@ -744,7 +747,7 @@ export class Worker {
|
|
|
744
747
|
try {
|
|
745
748
|
const renewed = await this.store.heartbeatBatch({
|
|
746
749
|
taskIds: live.map((c) => c.taskId),
|
|
747
|
-
workerId: this.
|
|
750
|
+
workerId: this.ownId,
|
|
748
751
|
leaseMs,
|
|
749
752
|
});
|
|
750
753
|
for (const ctx of live) {
|
|
@@ -779,7 +782,7 @@ export class Worker {
|
|
|
779
782
|
try {
|
|
780
783
|
await this.store.fail({
|
|
781
784
|
taskId: ctx.taskId,
|
|
782
|
-
workerId: this.
|
|
785
|
+
workerId: this.ownId,
|
|
783
786
|
error: envelope,
|
|
784
787
|
retryable,
|
|
785
788
|
delayMs: failDelayMs(ctx.attempt, retryable, this.backoffMs, this.backoffMaxMs),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cairnq",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "SQLite-first, cross-language, storage-centered durable task runtime",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Jannchie <jannchie@gmail.com>",
|
|
@@ -39,19 +39,21 @@
|
|
|
39
39
|
"test": "vitest run",
|
|
40
40
|
"typecheck": "tsc -p tsconfig.json --noEmit"
|
|
41
41
|
},
|
|
42
|
-
"dependencies": {
|
|
43
|
-
"better-sqlite3": "^13.0.2"
|
|
44
|
-
},
|
|
45
42
|
"peerDependencies": {
|
|
43
|
+
"better-sqlite3": "^13.0.2",
|
|
46
44
|
"pg": "^8.13.0"
|
|
47
45
|
},
|
|
48
46
|
"peerDependenciesMeta": {
|
|
47
|
+
"better-sqlite3": {
|
|
48
|
+
"optional": true
|
|
49
|
+
},
|
|
49
50
|
"pg": {
|
|
50
51
|
"optional": true
|
|
51
52
|
}
|
|
52
53
|
},
|
|
53
54
|
"devDependencies": {
|
|
54
55
|
"@types/better-sqlite3": "^7.6.11",
|
|
56
|
+
"better-sqlite3": "^13.0.2",
|
|
55
57
|
"@types/node": "^22.7.0",
|
|
56
58
|
"@types/pg": "^8.11.10",
|
|
57
59
|
"pg": "^8.13.0",
|