cairnq 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/dist/_protocol/migrations/postgres/0001_init.sql +2 -0
- package/dist/_protocol/migrations/postgres/0006_claim_name_index.sql +2 -0
- package/dist/_protocol/migrations/postgres/0008_claim_due_index.sql +68 -0
- package/dist/_protocol/migrations/sqlite/0001_init.sql +2 -0
- package/dist/_protocol/migrations/sqlite/0006_claim_name_index.sql +2 -0
- package/dist/_protocol/migrations/sqlite/0008_claim_due_index.sql +84 -0
- package/dist/_protocol/sql/postgres/claim.sql +6 -3
- package/dist/_protocol/sql/postgres/claim_one_name.sql +6 -3
- package/dist/_protocol/sql/postgres/claim_one_queue.sql +9 -5
- package/dist/_protocol/sql/postgres/claim_one_queue_one_name.sql +6 -3
- package/dist/_protocol/sql/postgres/installations.sql +33 -0
- package/dist/_protocol/sql/sqlite/claim.sql +6 -3
- package/dist/_protocol/sql/sqlite/claim_one_name.sql +6 -3
- package/dist/_protocol/sql/sqlite/claim_one_queue.sql +9 -6
- package/dist/_protocol/sql/sqlite/claim_one_queue_one_name.sql +9 -4
- package/dist/context.d.ts +4 -2
- package/dist/context.js +12 -8
- package/dist/errors.d.ts +28 -0
- package/dist/errors.js +34 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models.d.ts +2 -0
- package/dist/models.js +28 -1
- package/dist/retention.d.ts +13 -1
- package/dist/retention.js +26 -11
- package/dist/store/base.d.ts +12 -3
- package/dist/store/base.js +21 -6
- package/dist/store/pg-executor.d.ts +6 -5
- package/dist/store/pg-executor.js +6 -5
- package/dist/store/pg-pool.js +37 -11
- package/dist/store/postgres.d.ts +64 -0
- package/dist/store/postgres.js +121 -1
- package/dist/worker.d.ts +4 -2
- package/dist/worker.js +10 -8
- package/package.json +1 -1
- package/src/context.ts +10 -6
- package/src/errors.ts +36 -0
- package/src/index.ts +2 -0
- package/src/models.ts +28 -1
- package/src/retention.ts +27 -11
- package/src/store/base.ts +27 -5
- package/src/store/pg-executor.ts +6 -5
- package/src/store/pg-pool.ts +38 -11
- package/src/store/postgres.ts +125 -1
- package/src/worker.ts +10 -8
package/dist/worker.js
CHANGED
|
@@ -72,7 +72,7 @@ export class Worker {
|
|
|
72
72
|
queues;
|
|
73
73
|
opts;
|
|
74
74
|
handlers = new Map();
|
|
75
|
-
|
|
75
|
+
ownId = newId("worker");
|
|
76
76
|
/** Payload bytes charged to running handlers — see maxInFlightBytes. */
|
|
77
77
|
inFlightBytes = 0;
|
|
78
78
|
/** Calls in flight, for the names that cap their own concurrency. */
|
|
@@ -142,8 +142,10 @@ export class Worker {
|
|
|
142
142
|
worker.ownsStore = true;
|
|
143
143
|
return worker;
|
|
144
144
|
}
|
|
145
|
-
|
|
146
|
-
|
|
145
|
+
/** This worker's id — what `worker_id` on a running task points at, and what
|
|
146
|
+
* the Python SDK calls `worker_id` too. */
|
|
147
|
+
get workerId() {
|
|
148
|
+
return this.ownId;
|
|
147
149
|
}
|
|
148
150
|
task(arg, second, third) {
|
|
149
151
|
// Option form: (name | def, { batch?, concurrency?, resource? }, handler).
|
|
@@ -274,7 +276,7 @@ export class Worker {
|
|
|
274
276
|
return out;
|
|
275
277
|
}
|
|
276
278
|
context(task, leaseMs) {
|
|
277
|
-
return new TaskContext(this.store, task, this.
|
|
279
|
+
return new TaskContext(this.store, task, this.ownId, leaseMs, {
|
|
278
280
|
retryBackoffMs: this.backoffMs,
|
|
279
281
|
retryBackoffMaxMs: this.backoffMaxMs,
|
|
280
282
|
});
|
|
@@ -380,7 +382,7 @@ export class Worker {
|
|
|
380
382
|
// Only what this worker can run. Queues do not partition work by task
|
|
381
383
|
// name, so another worker's tasks would otherwise be claimed here and
|
|
382
384
|
// failed for want of a handler.
|
|
383
|
-
{ queues: this.queues, workerId: this.
|
|
385
|
+
{ queues: this.queues, workerId: this.ownId, leaseMs, names }, async (claim) => {
|
|
384
386
|
const drawn = [];
|
|
385
387
|
let left = free;
|
|
386
388
|
// Resource units this poll has already drawn. resourceCalls only
|
|
@@ -612,7 +614,7 @@ export class Worker {
|
|
|
612
614
|
try {
|
|
613
615
|
// complete (not succeed): finalizes as canceled if a cancel was requested
|
|
614
616
|
// while the handler ran, else succeeded.
|
|
615
|
-
await this.store.complete({ taskId: ctx.taskId, workerId: this.
|
|
617
|
+
await this.store.complete({ taskId: ctx.taskId, workerId: this.ownId, result });
|
|
616
618
|
ctx.markSettled();
|
|
617
619
|
}
|
|
618
620
|
catch (err) {
|
|
@@ -745,7 +747,7 @@ export class Worker {
|
|
|
745
747
|
try {
|
|
746
748
|
const renewed = await this.store.heartbeatBatch({
|
|
747
749
|
taskIds: live.map((c) => c.taskId),
|
|
748
|
-
workerId: this.
|
|
750
|
+
workerId: this.ownId,
|
|
749
751
|
leaseMs,
|
|
750
752
|
});
|
|
751
753
|
for (const ctx of live) {
|
|
@@ -780,7 +782,7 @@ export class Worker {
|
|
|
780
782
|
try {
|
|
781
783
|
await this.store.fail({
|
|
782
784
|
taskId: ctx.taskId,
|
|
783
|
-
workerId: this.
|
|
785
|
+
workerId: this.ownId,
|
|
784
786
|
error: envelope,
|
|
785
787
|
retryable,
|
|
786
788
|
delayMs: failDelayMs(ctx.attempt, retryable, this.backoffMs, this.backoffMaxMs),
|
package/package.json
CHANGED
package/src/context.ts
CHANGED
|
@@ -37,7 +37,7 @@ export class TaskContext {
|
|
|
37
37
|
constructor(
|
|
38
38
|
private readonly store: TaskStore,
|
|
39
39
|
private readonly task: Task,
|
|
40
|
-
|
|
40
|
+
private readonly ownerId: string,
|
|
41
41
|
private readonly leaseMs: number,
|
|
42
42
|
opts: TaskContextOptions = {},
|
|
43
43
|
) {
|
|
@@ -48,6 +48,10 @@ export class TaskContext {
|
|
|
48
48
|
get taskId(): string {
|
|
49
49
|
return this.task.id;
|
|
50
50
|
}
|
|
51
|
+
/** The worker running this task — what `worker_id` on the row points at. */
|
|
52
|
+
get workerId(): string {
|
|
53
|
+
return this.ownerId;
|
|
54
|
+
}
|
|
51
55
|
get name(): string {
|
|
52
56
|
return this.task.name;
|
|
53
57
|
}
|
|
@@ -150,7 +154,7 @@ export class TaskContext {
|
|
|
150
154
|
return this.owned(() =>
|
|
151
155
|
this.store.progress({
|
|
152
156
|
taskId: this.task.id,
|
|
153
|
-
workerId: this.
|
|
157
|
+
workerId: this.ownerId,
|
|
154
158
|
progress: value,
|
|
155
159
|
message,
|
|
156
160
|
}),
|
|
@@ -161,7 +165,7 @@ export class TaskContext {
|
|
|
161
165
|
return this.owned(() =>
|
|
162
166
|
this.store.heartbeat({
|
|
163
167
|
taskId: this.task.id,
|
|
164
|
-
workerId: this.
|
|
168
|
+
workerId: this.ownerId,
|
|
165
169
|
leaseMs: this.leaseMs,
|
|
166
170
|
}),
|
|
167
171
|
);
|
|
@@ -196,7 +200,7 @@ export class TaskContext {
|
|
|
196
200
|
async succeed(result: unknown = null): Promise<Task | null> {
|
|
197
201
|
if (this.isSettled) return null;
|
|
198
202
|
const task = await this.owned(() =>
|
|
199
|
-
this.store.complete({ taskId: this.task.id, workerId: this.
|
|
203
|
+
this.store.complete({ taskId: this.task.id, workerId: this.ownerId, result }),
|
|
200
204
|
);
|
|
201
205
|
this.markSettled();
|
|
202
206
|
return task;
|
|
@@ -229,7 +233,7 @@ export class TaskContext {
|
|
|
229
233
|
if (this.isSettled) return null;
|
|
230
234
|
const task = await this.owned(async () => {
|
|
231
235
|
const { task } = await this.store.completeIn<PgSession, T>(
|
|
232
|
-
{ taskId: this.task.id, workerId: this.
|
|
236
|
+
{ taskId: this.task.id, workerId: this.ownerId },
|
|
233
237
|
write,
|
|
234
238
|
);
|
|
235
239
|
return task;
|
|
@@ -253,7 +257,7 @@ export class TaskContext {
|
|
|
253
257
|
const task = await this.owned(() =>
|
|
254
258
|
this.store.fail({
|
|
255
259
|
taskId: this.task.id,
|
|
256
|
-
workerId: this.
|
|
260
|
+
workerId: this.ownerId,
|
|
257
261
|
error: envelope,
|
|
258
262
|
retryable,
|
|
259
263
|
delayMs: failDelayMs(this.task.attempt, retryable, this.backoffMs, this.backoffMaxMs),
|
package/src/errors.ts
CHANGED
|
@@ -208,6 +208,42 @@ export class ProtocolVersionMismatch extends CairnQError {
|
|
|
208
208
|
}
|
|
209
209
|
}
|
|
210
210
|
|
|
211
|
+
/**
|
|
212
|
+
* This store cannot do what was asked, and no argument would change that — the
|
|
213
|
+
* capability belongs to the backend, not to the call.
|
|
214
|
+
*
|
|
215
|
+
* Thrown by `completeIn` on a store with no driver session to share (SQLite has
|
|
216
|
+
* none). A CairnQError rather than a bare Error so the same catch works across
|
|
217
|
+
* both SDKs; the Python SDK raises the same named error.
|
|
218
|
+
*/
|
|
219
|
+
export class UnsupportedBackend extends CairnQError {
|
|
220
|
+
constructor(message: string) {
|
|
221
|
+
super(message);
|
|
222
|
+
this.name = "UnsupportedBackend";
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* This connection is not pointed at the cairnq installation the rest of the
|
|
228
|
+
* deployment is using — raised at connect, before any task is written.
|
|
229
|
+
*
|
|
230
|
+
* The schema a Postgres connection resolves to is out-of-band configuration
|
|
231
|
+
* (`search_path`, a `schema` option, an ORM's pool settings), so two processes
|
|
232
|
+
* given the same DSN can still land in different schemas. Every migration is
|
|
233
|
+
* `create table if not exists`, so the odd one out does not fail: it builds a
|
|
234
|
+
* second, empty installation and its protocol version check passes against the
|
|
235
|
+
* cairnq_meta it just created. Left undetected, an API and a worker then agree
|
|
236
|
+
* about everything except WHERE, and no task ever crosses.
|
|
237
|
+
*
|
|
238
|
+
* The Python SDK raises the same named error.
|
|
239
|
+
*/
|
|
240
|
+
export class SchemaMismatch extends CairnQError {
|
|
241
|
+
constructor(message: string) {
|
|
242
|
+
super(message);
|
|
243
|
+
this.name = "SchemaMismatch";
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
211
247
|
/** A value could not be encoded for a protocol JSON column (non-finite number,
|
|
212
248
|
* BigInt, circular structure, …). Raised at the boundary — submit rejects with
|
|
213
249
|
* it, and a worker records a handler result that triggers it as a permanent
|
package/src/index.ts
CHANGED
package/src/models.ts
CHANGED
|
@@ -38,7 +38,20 @@ export interface TaskRef {
|
|
|
38
38
|
status: TaskStatus;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
const JSON_COLUMNS = ["payload", "result", "error", "metadata"] as const;
|
|
41
|
+
export const JSON_COLUMNS = ["payload", "result", "error", "metadata"] as const;
|
|
42
|
+
// The bigint columns. `attempt` / `max_attempts` / `priority` are int4 and
|
|
43
|
+
// `progress` is double precision, so every driver already gives those as numbers;
|
|
44
|
+
// only int8 has a wire form worth normalizing. Nullability differs per column
|
|
45
|
+
// (completed_at_ms may be null, created_at_ms may not), so the coercion has to
|
|
46
|
+
// preserve null rather than turn it into 0.
|
|
47
|
+
export const MS_COLUMNS = [
|
|
48
|
+
"lease_until_ms",
|
|
49
|
+
"run_at_ms",
|
|
50
|
+
"cancel_requested_at_ms",
|
|
51
|
+
"created_at_ms",
|
|
52
|
+
"updated_at_ms",
|
|
53
|
+
"completed_at_ms",
|
|
54
|
+
] as const;
|
|
42
55
|
// As a const tuple so TerminalStatus derives from it — the same declare-once
|
|
43
56
|
// pattern as STATUSES/TaskStatus above.
|
|
44
57
|
export const TERMINAL = ["succeeded", "failed", "canceled"] as const;
|
|
@@ -63,6 +76,20 @@ export function rowToTask(row: Record<string, unknown>): Task {
|
|
|
63
76
|
// already-decoded object. Parse only a string — never assume one backend.
|
|
64
77
|
t[col] = typeof v === "string" ? JSON.parse(v) : (v ?? null);
|
|
65
78
|
}
|
|
79
|
+
for (const col of MS_COLUMNS) {
|
|
80
|
+
const v = row[col];
|
|
81
|
+
// Same argument, for the other column type the drivers disagree about.
|
|
82
|
+
// Postgres sends int8 down the wire as text to protect precision it cannot
|
|
83
|
+
// know is unneeded; `pg` surfaces that as a string, postgres.js does too,
|
|
84
|
+
// asyncpg decodes to int. Normalizing here rather than demanding it of every
|
|
85
|
+
// driver is what keeps an INJECTED executor honest: the alternative is
|
|
86
|
+
// asking an application to change its driver's global int8 handling to suit
|
|
87
|
+
// cairnq, which breaks that application's own bigint columns.
|
|
88
|
+
//
|
|
89
|
+
// Lossless: every one of these is an epoch-ms, and a millisecond timestamp
|
|
90
|
+
// does not reach Number.MAX_SAFE_INTEGER until the year 287396.
|
|
91
|
+
t[col] = v == null ? null : Number(v);
|
|
92
|
+
}
|
|
66
93
|
return t as unknown as Task;
|
|
67
94
|
}
|
|
68
95
|
|
package/src/retention.ts
CHANGED
|
@@ -55,8 +55,18 @@ export class RetentionSweeper {
|
|
|
55
55
|
private active = false;
|
|
56
56
|
/** Set by stop(), so a drain in progress can cut itself short too. */
|
|
57
57
|
private stopping = false;
|
|
58
|
-
/**
|
|
59
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Resolved by stop(); every sleep races it. One signal rather than a handle to
|
|
60
|
+
* the current sleep, because there can be more than one: `sweep()` is public
|
|
61
|
+
* and meant to be called directly for an on-demand drain, and its
|
|
62
|
+
* between-batches yield is a sleep of its own. A single handle let that sleep
|
|
63
|
+
* overwrite the scheduled loop's — and then clear it — leaving stop() nothing
|
|
64
|
+
* to wake and close() blocked until the whole interval (an hour, by default)
|
|
65
|
+
* ran out. Same shape as Worker's `stopped$`, and as the Python twin's
|
|
66
|
+
* asyncio.Event.
|
|
67
|
+
*/
|
|
68
|
+
private stopSignal!: Promise<void>;
|
|
69
|
+
private wake!: () => void;
|
|
60
70
|
/** The loop itself, awaited by stop() so no purge outlives the store. */
|
|
61
71
|
private loop: Promise<void> | null = null;
|
|
62
72
|
private readonly intervalMs: number;
|
|
@@ -83,6 +93,7 @@ export class RetentionSweeper {
|
|
|
83
93
|
if (!cutoffs.length) {
|
|
84
94
|
throw new Error("retention.olderThanMs must name at least one status");
|
|
85
95
|
}
|
|
96
|
+
this.arm();
|
|
86
97
|
this.purgeInputs = cutoffs.map(([status, ms]) => ({
|
|
87
98
|
olderThanMs: ms,
|
|
88
99
|
status,
|
|
@@ -97,14 +108,21 @@ export class RetentionSweeper {
|
|
|
97
108
|
if (this.active) return;
|
|
98
109
|
this.active = true;
|
|
99
110
|
this.stopping = false;
|
|
111
|
+
// A stopped sweeper can be started again, and the old signal is spent.
|
|
112
|
+
this.arm();
|
|
100
113
|
this.loop = this.run();
|
|
101
114
|
}
|
|
102
115
|
|
|
116
|
+
/** Mint a fresh stop signal. */
|
|
117
|
+
private arm(): void {
|
|
118
|
+
this.stopSignal = new Promise<void>((resolve) => (this.wake = resolve));
|
|
119
|
+
}
|
|
120
|
+
|
|
103
121
|
/** Stop sweeping and wait for the sweep in flight, if any. */
|
|
104
122
|
async stop(): Promise<void> {
|
|
105
123
|
this.stopping = true;
|
|
106
124
|
this.active = false;
|
|
107
|
-
this.wake
|
|
125
|
+
this.wake();
|
|
108
126
|
await this.loop;
|
|
109
127
|
this.loop = null;
|
|
110
128
|
}
|
|
@@ -152,15 +170,13 @@ export class RetentionSweeper {
|
|
|
152
170
|
/** Sleep, interruptible by stop(). Unref'd: retention is housekeeping, and a
|
|
153
171
|
* pending sweep must never be the reason a process refuses to exit. */
|
|
154
172
|
private sleep(ms: number): Promise<void> {
|
|
155
|
-
|
|
156
|
-
|
|
173
|
+
let timer: NodeJS.Timeout;
|
|
174
|
+
const nap = new Promise<void>((resolve) => {
|
|
175
|
+
timer = setTimeout(resolve, ms);
|
|
157
176
|
timer.unref?.();
|
|
158
|
-
this.wake = () => {
|
|
159
|
-
clearTimeout(timer);
|
|
160
|
-
resolve();
|
|
161
|
-
};
|
|
162
|
-
}).finally(() => {
|
|
163
|
-
this.wake = null;
|
|
164
177
|
});
|
|
178
|
+
// Clear the timer whichever side wins, so a stop is never followed by a
|
|
179
|
+
// leftover sweep timer.
|
|
180
|
+
return Promise.race([nap, this.stopSignal]).finally(() => clearTimeout(timer));
|
|
165
181
|
}
|
|
166
182
|
}
|
package/src/store/base.ts
CHANGED
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
LostLease,
|
|
6
6
|
ProtocolVersionMismatch,
|
|
7
7
|
SerializationError,
|
|
8
|
+
UnsupportedBackend,
|
|
8
9
|
} from "../errors.js";
|
|
9
10
|
import {
|
|
10
11
|
isTerminalStatus,
|
|
@@ -274,6 +275,13 @@ export abstract class TaskStore {
|
|
|
274
275
|
*/
|
|
275
276
|
protected subscribePush?(onSignal: (signal: WatchSignal) => void): () => void;
|
|
276
277
|
|
|
278
|
+
/**
|
|
279
|
+
* Tell a store with a push channel which queues this process will wait on, so
|
|
280
|
+
* it can buffer their notifications and ignore everyone else's. Optional: a
|
|
281
|
+
* store without a push channel has nothing to buffer.
|
|
282
|
+
*/
|
|
283
|
+
protected registerWakeable?(queues: string[]): void;
|
|
284
|
+
|
|
277
285
|
/**
|
|
278
286
|
* Nudge the push channel back up if it has dropped. Called from `watch`'s
|
|
279
287
|
* timer, which is the only thing keeping a client-side subscriber alive: a
|
|
@@ -561,7 +569,15 @@ export abstract class TaskStore {
|
|
|
561
569
|
});
|
|
562
570
|
const timer = setInterval(() => {
|
|
563
571
|
this.warmPush?.();
|
|
564
|
-
|
|
572
|
+
// Guarded for the same reason PostgresStore guards its push fan-out: a
|
|
573
|
+
// consumer that throws must not take the timer down with it. Losing the
|
|
574
|
+
// timer would silently retire the fallback that makes watch correct
|
|
575
|
+
// where there is no push channel at all.
|
|
576
|
+
try {
|
|
577
|
+
emit({ reason: "poll" });
|
|
578
|
+
} catch {
|
|
579
|
+
// The consumer's problem, not the watch's.
|
|
580
|
+
}
|
|
565
581
|
}, pollMs);
|
|
566
582
|
timer.unref?.();
|
|
567
583
|
return () => {
|
|
@@ -631,9 +647,12 @@ export abstract class TaskStore {
|
|
|
631
647
|
* `plan` runs with the write lock held, so it must await nothing but that
|
|
632
648
|
* callback.
|
|
633
649
|
*
|
|
634
|
-
* `names` is the union `plan` might ask for
|
|
635
|
-
*
|
|
636
|
-
*
|
|
650
|
+
* `names` is the union `plan` might ask for, and it filters the probe's
|
|
651
|
+
* queued-work arm. Lease recovery is deliberately NOT filtered by it — nor is
|
|
652
|
+
* the probe's expired-lease arm — because reclaiming a dead worker's task is
|
|
653
|
+
* every worker's job, whatever names it happens to handle. Returns undefined
|
|
654
|
+
* when the probe finds nothing claimable, in which case `plan` never runs and
|
|
655
|
+
* no transaction is opened.
|
|
637
656
|
*/
|
|
638
657
|
async claimSession<T>(
|
|
639
658
|
input: { queues: string[]; workerId: string; leaseMs?: number; names: string[] | null },
|
|
@@ -655,6 +674,9 @@ export abstract class TaskStore {
|
|
|
655
674
|
limit: 1,
|
|
656
675
|
lease_expired_error: LEASE_EXPIRED_ERROR_JSON,
|
|
657
676
|
};
|
|
677
|
+
// Before the probe: a push-channel store has to know what this caller waits
|
|
678
|
+
// on from its FIRST claim, not from its first empty poll.
|
|
679
|
+
this.registerWakeable?.(input.queues);
|
|
658
680
|
if (!(await this.hasClaimableWork(base))) return undefined;
|
|
659
681
|
return this.tx(async (fetch) => {
|
|
660
682
|
await fetch("recover_leases", base);
|
|
@@ -768,7 +790,7 @@ export abstract class TaskStore {
|
|
|
768
790
|
fn: (session: S) => Promise<T>,
|
|
769
791
|
): Promise<{ task: Task; value: T }> {
|
|
770
792
|
if (!this.txWithSession) {
|
|
771
|
-
throw new
|
|
793
|
+
throw new UnsupportedBackend(
|
|
772
794
|
"this store cannot share a transaction with the caller — " +
|
|
773
795
|
"completeIn requires a Postgres store (see PgExecutor)",
|
|
774
796
|
);
|
package/src/store/pg-executor.ts
CHANGED
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
* instead of opening a second one, which is what makes a task's settlement
|
|
9
9
|
* commit in the same transaction as the rows the task produced.
|
|
10
10
|
*
|
|
11
|
-
* Implementing one is small — see `createPoolExecutor`
|
|
12
|
-
* implementation over `pg
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* Implementing one is small — see `createPoolExecutor` in pg-pool.ts for the
|
|
12
|
+
* reference implementation over `pg`. An adapter passes rows through as its
|
|
13
|
+
* driver produced them: cairnq normalizes both column types the drivers disagree
|
|
14
|
+
* about (jsonb decoded or not, int8 as text or number) in `rowToTask`, so no
|
|
15
|
+
* adapter has to reconfigure its driver — and none has to change how the
|
|
16
|
+
* application's OWN columns come back in order to satisfy cairnq.
|
|
16
17
|
*/
|
|
17
18
|
|
|
18
19
|
/** A row as the driver hands it back: column name -> value. */
|
package/src/store/pg-pool.ts
CHANGED
|
@@ -15,11 +15,12 @@ async function loadPg(): Promise<typeof import("pg")> {
|
|
|
15
15
|
throw new Error("PostgresStore requires the 'pg' package — install it (e.g. `npm i pg`)");
|
|
16
16
|
}
|
|
17
17
|
const pg = (mod.default ?? mod) as typeof import("pg");
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
|
|
18
|
+
// Deliberately NOT setting a global int8 type parser here. It would be the
|
|
19
|
+
// shortest fix for Postgres sending bigint as text, and it is what this file
|
|
20
|
+
// used to do — but pg's type parsers are process-global, so a library that
|
|
21
|
+
// installs one silently changes how the APPLICATION's own bigint columns come
|
|
22
|
+
// back. rowToTask normalizes instead, which costs nothing and leaves the
|
|
23
|
+
// caller's driver as they configured it.
|
|
23
24
|
pgModule = pg;
|
|
24
25
|
return pg;
|
|
25
26
|
}
|
|
@@ -86,14 +87,40 @@ export async function createPoolExecutor(
|
|
|
86
87
|
const pool = new pg.Pool({ connectionString: dsn, max: opts.max });
|
|
87
88
|
if (schema) {
|
|
88
89
|
// Queued on the connection before it is handed out, so every statement this
|
|
89
|
-
// pool ever runs — migrations included — resolves in the right schema.
|
|
90
|
-
// connections come and go, which is why this is per-connection rather
|
|
91
|
-
// one-off at startup.
|
|
90
|
+
// pool ever runs — migrations included — resolves in the right schema.
|
|
91
|
+
// Pooled connections come and go, which is why this is per-connection rather
|
|
92
|
+
// than a one-off at startup.
|
|
93
|
+
//
|
|
94
|
+
// Deliberately NOT the `options: '-c search_path=...'` connection parameter,
|
|
95
|
+
// which reads like the tidier answer: `pg` builds its config as
|
|
96
|
+
// `Object.assign({}, config, parse(connectionString))`, so a DSN carrying its
|
|
97
|
+
// own `?options=` (a statement_timeout, say — and cairnq's own SchemaMismatch
|
|
98
|
+
// message recommends putting `search_path` there) silently REPLACES it, and
|
|
99
|
+
// the pool then resolves somewhere else entirely. A startup parameter is also
|
|
100
|
+
// a narrower compatibility surface than a plain statement behind a pooler.
|
|
101
|
+
//
|
|
102
|
+
// The Python twin does get to be a startup parameter (asyncpg's
|
|
103
|
+
// `server_settings`), so there a bad schema fails the connection atomically
|
|
104
|
+
// and needs no handler below. That asymmetry is the driver's, not a drift.
|
|
92
105
|
pool.on("connect", (client) => {
|
|
93
|
-
void
|
|
106
|
+
// The rejection must be handled: `void` on a query that rejects is an
|
|
107
|
+
// unhandled rejection, which by default takes the process down. And it
|
|
108
|
+
// must not be swallowed either — a connection whose search_path never
|
|
109
|
+
// landed would go on serving statements against the wrong schema for its
|
|
110
|
+
// whole life, which no error anywhere would explain. Ending it is what
|
|
111
|
+
// makes that loud: this client is about to be handed to a caller, whose
|
|
112
|
+
// first statement then fails instead of quietly writing to `public`.
|
|
113
|
+
client.query(`set search_path to "${schema}"`).catch(() => {
|
|
114
|
+
// `end` is the underlying Client's; the pool hands the connect event a
|
|
115
|
+
// PoolClient whose `release` does not exist yet at this point. A client
|
|
116
|
+
// ended here is dropped from the pool when its holder releases it
|
|
117
|
+
// (pg-pool checks `_queryable`), so this costs a connection, not a slot.
|
|
118
|
+
void (client as unknown as PG.Client).end().catch(() => {
|
|
119
|
+
// Already gone. The pool discards it either way.
|
|
120
|
+
});
|
|
121
|
+
});
|
|
94
122
|
});
|
|
95
|
-
|
|
96
|
-
if (schema) {
|
|
123
|
+
|
|
97
124
|
// Created once, here, rather than from the connect handler (which would ask
|
|
98
125
|
// for CREATE privilege on every new connection) or from the migrations
|
|
99
126
|
// (which name no schema, by design). Without it the first `create table`
|
package/src/store/postgres.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { SchemaMismatch } from "../errors.js";
|
|
1
2
|
import { loadMigrations, loadStatements } from "../sql.js";
|
|
2
3
|
import {
|
|
3
4
|
checkProtocolVersion,
|
|
@@ -86,6 +87,14 @@ export class PostgresStore extends TaskStore {
|
|
|
86
87
|
/** Set once migrations have run and the protocol version checked out. */
|
|
87
88
|
private executor: PgExecutor | null = null;
|
|
88
89
|
private connecting: Promise<void> | null = null;
|
|
90
|
+
/**
|
|
91
|
+
* Bumped by close(). A doConnect that started before the bump must not publish
|
|
92
|
+
* what it built: close() nulls `connecting` but does not await it, so an
|
|
93
|
+
* in-flight connect outlives the close and would otherwise install an executor
|
|
94
|
+
* — and, worse, a LISTEN connection — on a store nobody will close again. That
|
|
95
|
+
* connection's socket keeps the process alive on its own.
|
|
96
|
+
*/
|
|
97
|
+
private generation = 0;
|
|
89
98
|
private readonly statements: Record<string, string>;
|
|
90
99
|
|
|
91
100
|
// ------------------------------------------------------- LISTEN/NOTIFY state
|
|
@@ -105,9 +114,31 @@ export class PostgresStore extends TaskStore {
|
|
|
105
114
|
* hammered from the poll loop. */
|
|
106
115
|
private listenerRetryAt = 0;
|
|
107
116
|
private listenerBackoffMs = LISTENER_RETRY_MS;
|
|
117
|
+
// NB: these three are re-armed together by resetListenerState().
|
|
108
118
|
/** Queues notified while nobody was waiting; consumed by the next claimWake
|
|
109
119
|
* so a wake that lands between polls is not lost. */
|
|
110
120
|
private readonly pendingQueues = new Set<string>();
|
|
121
|
+
/**
|
|
122
|
+
* The queues some caller has actually waited on — the only ones worth
|
|
123
|
+
* buffering into `pendingQueues`.
|
|
124
|
+
*
|
|
125
|
+
* Without it that buffer is a leak: every `cairnq_queued` notification lands in
|
|
126
|
+
* it, and only claimWake consumes, and only the queues it names. A client-side
|
|
127
|
+
* handle never calls claimWake at all, and a worker only ever names its own
|
|
128
|
+
* queues — so a deployment whose queue names carry an identity (`user:1234`)
|
|
129
|
+
* grows the set forever. This one is bounded by what this process claims from,
|
|
130
|
+
* which is its own configuration.
|
|
131
|
+
*
|
|
132
|
+
* Registered from `claimSession` as well as from `claimWake`, so a worker's
|
|
133
|
+
* queues are known from its first claim rather than from its first empty poll:
|
|
134
|
+
* a notification landing between those two would otherwise be dropped, costing
|
|
135
|
+
* a poll interval exactly when work has just arrived.
|
|
136
|
+
*
|
|
137
|
+
* The bound holds because a worker's queue list is its configuration. A caller
|
|
138
|
+
* driving `claimWake`/`claim` with per-tenant queue names directly still
|
|
139
|
+
* accumulates one entry per distinct name it passes.
|
|
140
|
+
*/
|
|
141
|
+
private readonly wakeableQueues = new Set<string>();
|
|
111
142
|
/** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
|
|
112
143
|
private readonly waiters = new Map<string, Set<() => void>>();
|
|
113
144
|
/** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
|
|
@@ -134,6 +165,7 @@ export class PostgresStore extends TaskStore {
|
|
|
134
165
|
}
|
|
135
166
|
|
|
136
167
|
async close(): Promise<void> {
|
|
168
|
+
this.generation++; // disown any connect still in flight
|
|
137
169
|
this.listenerUnavailable = true; // no revival after close
|
|
138
170
|
this.dropListener();
|
|
139
171
|
const executor = this.executor;
|
|
@@ -158,11 +190,31 @@ export class PostgresStore extends TaskStore {
|
|
|
158
190
|
await this.connecting;
|
|
159
191
|
}
|
|
160
192
|
|
|
193
|
+
/**
|
|
194
|
+
* A fresh connection gets a fresh verdict on LISTEN.
|
|
195
|
+
*
|
|
196
|
+
* Every path here connects lazily, so a store used again after close simply
|
|
197
|
+
* reconnects — and without this it would reconnect permanently degraded to
|
|
198
|
+
* polling, having decided "no push channel" about a connection that no longer
|
|
199
|
+
* exists. Called from the constructor for the initial state and from doConnect
|
|
200
|
+
* for every later one, so the three fields cannot be re-armed by halves. Only
|
|
201
|
+
* ever safe under doConnect's generation check.
|
|
202
|
+
*/
|
|
203
|
+
private resetListenerState(): void {
|
|
204
|
+
this.listenerUnavailable = false;
|
|
205
|
+
this.listenerRetryAt = 0;
|
|
206
|
+
this.listenerBackoffMs = LISTENER_RETRY_MS;
|
|
207
|
+
}
|
|
208
|
+
|
|
161
209
|
private async doConnect(): Promise<void> {
|
|
210
|
+
const generation = this.generation;
|
|
162
211
|
const executor =
|
|
163
212
|
this.provided ??
|
|
164
213
|
(await createPoolExecutor(this.dsn!, { max: this.opts.max, schema: this.opts.schema }));
|
|
165
214
|
try {
|
|
215
|
+
// Before migrations, which would otherwise create the very installation
|
|
216
|
+
// this is trying to warn about.
|
|
217
|
+
await this.checkSchema(executor);
|
|
166
218
|
await this.applyMigrations(executor);
|
|
167
219
|
checkProtocolVersion(await this.readProtocolVersion(executor));
|
|
168
220
|
} catch (e) {
|
|
@@ -170,12 +222,78 @@ export class PostgresStore extends TaskStore {
|
|
|
170
222
|
if (!this.provided) await executor.close().catch(() => {});
|
|
171
223
|
throw e;
|
|
172
224
|
}
|
|
225
|
+
// Closed while we were connecting: this executor has no owner, so drop it
|
|
226
|
+
// rather than publishing it. Nothing below may run — clearing
|
|
227
|
+
// `listenerUnavailable` here is exactly how a closed store would end up with
|
|
228
|
+
// a listener holding a socket forever.
|
|
229
|
+
if (generation !== this.generation) {
|
|
230
|
+
if (!this.provided) await executor.close().catch(() => {});
|
|
231
|
+
throw new Error("store was closed while connecting");
|
|
232
|
+
}
|
|
173
233
|
this.executor = executor; // publish only a fully-migrated, version-checked executor
|
|
234
|
+
this.resetListenerState();
|
|
174
235
|
// Warm the LISTEN subscription in the background so the first idle sleep is
|
|
175
236
|
// already wakeable. Fire-and-forget: failure just means polling.
|
|
176
237
|
this.listenerReady();
|
|
177
238
|
}
|
|
178
239
|
|
|
240
|
+
/**
|
|
241
|
+
* Refuse a connection pointed somewhere other than the deployment's cairnq.
|
|
242
|
+
*
|
|
243
|
+
* Two shapes, because `schema` means "the schema cairnq's tables live in" and
|
|
244
|
+
* cairnq can either arrange that (it built the connection) or only check it
|
|
245
|
+
* (the caller's executor did):
|
|
246
|
+
*
|
|
247
|
+
* - `schema` configured -> assert the connection actually resolves there. On
|
|
248
|
+
* the DSN path this is a cheap self-check; on an injected executor it is the
|
|
249
|
+
* only way to state the expectation at all.
|
|
250
|
+
* - `schema` not configured -> the dangerous case is being about to create a
|
|
251
|
+
* SECOND installation while one already exists elsewhere in this database,
|
|
252
|
+
* which is exactly what a mismatched pair of SDKs does. Joining an existing
|
|
253
|
+
* installation is fine no matter what else is around, so the check is
|
|
254
|
+
* deliberately narrow: it fires only when this schema has no cairnq and some
|
|
255
|
+
* other schema does.
|
|
256
|
+
*
|
|
257
|
+
* That narrowness is what keeps it from crying wolf. Two applications each
|
|
258
|
+
* running their own cairnq in their own schema are legitimate; the second one
|
|
259
|
+
* to be set up trips this once, and saying `schema` explicitly — which such a
|
|
260
|
+
* deployment should be doing anyway — is both the fix and the confirmation.
|
|
261
|
+
*/
|
|
262
|
+
private async checkSchema(executor: PgExecutor): Promise<void> {
|
|
263
|
+
// One row per installation; the statement's LEFT JOIN guarantees at least
|
|
264
|
+
// one, so current_schema is readable even where cairnq lives nowhere yet.
|
|
265
|
+
const rows = await executor.query(this.statements.installations, []);
|
|
266
|
+
const current = (rows[0]?.current_schema as string | null) ?? null;
|
|
267
|
+
const installations = rows
|
|
268
|
+
.map((r) => r.schema as string | null)
|
|
269
|
+
.filter((s): s is string => s != null);
|
|
270
|
+
const wanted = this.opts.schema;
|
|
271
|
+
|
|
272
|
+
if (wanted != null) {
|
|
273
|
+
if (current !== wanted) {
|
|
274
|
+
throw new SchemaMismatch(
|
|
275
|
+
`cairnq is configured for schema ${JSON.stringify(wanted)} but this ` +
|
|
276
|
+
`connection resolves to ${JSON.stringify(current)} — check the ` +
|
|
277
|
+
`connection's search_path`,
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
// A search_path naming nothing that exists: there is no "here" to compare
|
|
283
|
+
// against, and the migrations are about to fail with a clearer message.
|
|
284
|
+
if (current === null) return;
|
|
285
|
+
if (installations.length === 0 || installations.includes(current)) return;
|
|
286
|
+
|
|
287
|
+
throw new SchemaMismatch(
|
|
288
|
+
`cairnq tables already exist in schema ${installations.map((s) => JSON.stringify(s)).join(", ")} ` +
|
|
289
|
+
`of this database, but this connection resolves to ${JSON.stringify(current)}, where there are ` +
|
|
290
|
+
`none. Connecting would create a second, parallel installation that the other one can never see ` +
|
|
291
|
+
`— an API and a worker split this way agree about everything except where, and no task crosses. ` +
|
|
292
|
+
`Point this process at the same schema (\`schema\` option, or \`options=-c search_path=...\` in the ` +
|
|
293
|
+
`DSN), or pass \`schema\` explicitly to confirm a separate installation is what you meant.`,
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
|
|
179
297
|
private async applyMigrations(executor: PgExecutor): Promise<void> {
|
|
180
298
|
await executor.exec(
|
|
181
299
|
"create table if not exists cairnq_migrations " +
|
|
@@ -213,7 +331,12 @@ export class PostgresStore extends TaskStore {
|
|
|
213
331
|
}
|
|
214
332
|
|
|
215
333
|
// ------------------------------------------------------------ wake channel
|
|
334
|
+
protected registerWakeable(queues: string[]): void {
|
|
335
|
+
for (const q of queues) this.wakeableQueues.add(q);
|
|
336
|
+
}
|
|
337
|
+
|
|
216
338
|
async claimWake(queues: string[], timeoutMs: number): Promise<void> {
|
|
339
|
+
this.registerWakeable(queues);
|
|
217
340
|
if (!this.listenerReady()) return super.claimWake(queues, timeoutMs);
|
|
218
341
|
const deadline = Date.now() + timeoutMs;
|
|
219
342
|
for (;;) {
|
|
@@ -324,7 +447,8 @@ export class PostgresStore extends TaskStore {
|
|
|
324
447
|
private onNotification(channel: string, payload: string | undefined): void {
|
|
325
448
|
let key: string;
|
|
326
449
|
if (channel === QUEUED_CHANNEL && payload) {
|
|
327
|
-
|
|
450
|
+
// Buffered only for a queue someone waits on — see wakeableQueues.
|
|
451
|
+
if (this.wakeableQueues.has(payload)) this.pendingQueues.add(payload);
|
|
328
452
|
key = "queued";
|
|
329
453
|
this.publish({ reason: "queued", queue: payload });
|
|
330
454
|
} else if (channel === DONE_CHANNEL && payload) {
|