cairnq 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +10 -2
  2. package/dist/_protocol/migrations/postgres/0001_init.sql +2 -0
  3. package/dist/_protocol/migrations/postgres/0006_claim_name_index.sql +2 -0
  4. package/dist/_protocol/migrations/postgres/0008_claim_due_index.sql +68 -0
  5. package/dist/_protocol/migrations/sqlite/0001_init.sql +2 -0
  6. package/dist/_protocol/migrations/sqlite/0006_claim_name_index.sql +2 -0
  7. package/dist/_protocol/migrations/sqlite/0008_claim_due_index.sql +84 -0
  8. package/dist/_protocol/sql/postgres/claim.sql +6 -3
  9. package/dist/_protocol/sql/postgres/claim_one_name.sql +6 -3
  10. package/dist/_protocol/sql/postgres/claim_one_queue.sql +9 -5
  11. package/dist/_protocol/sql/postgres/claim_one_queue_one_name.sql +6 -3
  12. package/dist/_protocol/sql/postgres/installations.sql +33 -0
  13. package/dist/_protocol/sql/sqlite/claim.sql +6 -3
  14. package/dist/_protocol/sql/sqlite/claim_one_name.sql +6 -3
  15. package/dist/_protocol/sql/sqlite/claim_one_queue.sql +9 -6
  16. package/dist/_protocol/sql/sqlite/claim_one_queue_one_name.sql +9 -4
  17. package/dist/context.d.ts +4 -2
  18. package/dist/context.js +12 -8
  19. package/dist/errors.d.ts +28 -0
  20. package/dist/errors.js +34 -0
  21. package/dist/index.d.ts +1 -1
  22. package/dist/index.js +1 -1
  23. package/dist/models.d.ts +2 -0
  24. package/dist/models.js +28 -1
  25. package/dist/retention.d.ts +13 -1
  26. package/dist/retention.js +26 -11
  27. package/dist/store/base.d.ts +12 -3
  28. package/dist/store/base.js +21 -6
  29. package/dist/store/pg-executor.d.ts +6 -5
  30. package/dist/store/pg-executor.js +6 -5
  31. package/dist/store/pg-pool.js +37 -11
  32. package/dist/store/postgres.d.ts +64 -0
  33. package/dist/store/postgres.js +121 -1
  34. package/dist/worker.d.ts +4 -2
  35. package/dist/worker.js +10 -8
  36. package/package.json +1 -1
  37. package/src/context.ts +10 -6
  38. package/src/errors.ts +36 -0
  39. package/src/index.ts +2 -0
  40. package/src/models.ts +28 -1
  41. package/src/retention.ts +27 -11
  42. package/src/store/base.ts +27 -5
  43. package/src/store/pg-executor.ts +6 -5
  44. package/src/store/pg-pool.ts +38 -11
  45. package/src/store/postgres.ts +125 -1
  46. package/src/worker.ts +10 -8
package/dist/models.js CHANGED
@@ -3,7 +3,20 @@
3
3
  // truth is the status CHECK constraint in cairnq-protocol's migration, which the
4
4
  // conformance suite pins this set against.
5
5
  export const STATUSES = ["queued", "running", "succeeded", "failed", "canceled"];
6
- const JSON_COLUMNS = ["payload", "result", "error", "metadata"];
6
+ export const JSON_COLUMNS = ["payload", "result", "error", "metadata"];
7
+ // The bigint columns. `attempt` / `max_attempts` / `priority` are int4 and
8
+ // `progress` is double precision, so every driver already gives those as numbers;
9
+ // only int8 has a wire form worth normalizing. Nullability differs per column
10
+ // (completed_at_ms may be null, created_at_ms may not), so the coercion has to
11
+ // preserve null rather than turn it into 0.
12
+ export const MS_COLUMNS = [
13
+ "lease_until_ms",
14
+ "run_at_ms",
15
+ "cancel_requested_at_ms",
16
+ "created_at_ms",
17
+ "updated_at_ms",
18
+ "completed_at_ms",
19
+ ];
7
20
  // As a const tuple so TerminalStatus derives from it — the same declare-once
8
21
  // pattern as STATUSES/TaskStatus above.
9
22
  export const TERMINAL = ["succeeded", "failed", "canceled"];
@@ -24,6 +37,20 @@ export function rowToTask(row) {
24
37
  // already-decoded object. Parse only a string — never assume one backend.
25
38
  t[col] = typeof v === "string" ? JSON.parse(v) : (v ?? null);
26
39
  }
40
+ for (const col of MS_COLUMNS) {
41
+ const v = row[col];
42
+ // Same argument, for the other column type the drivers disagree about.
43
+ // Postgres sends int8 down the wire as text to protect precision it cannot
44
+ // know is unneeded; `pg` surfaces that as a string, postgres.js does too,
45
+ // asyncpg decodes to int. Normalizing here rather than demanding it of every
46
+ // driver is what keeps an INJECTED executor honest: the alternative is
47
+ // asking an application to change its driver's global int8 handling to suit
48
+ // cairnq, which breaks that application's own bigint columns.
49
+ //
50
+ // Lossless: every one of these is an epoch-ms, and a millisecond timestamp
51
+ // does not reach Number.MAX_SAFE_INTEGER until the year 287396.
52
+ t[col] = v == null ? null : Number(v);
53
+ }
27
54
  return t;
28
55
  }
29
56
  /** Accepts anything carrying a status — a Task or a TaskRef probe. */
@@ -46,7 +46,17 @@ export declare class RetentionSweeper {
46
46
  private active;
47
47
  /** Set by stop(), so a drain in progress can cut itself short too. */
48
48
  private stopping;
49
- /** Resolves the current sleep early, so stop() need not wait out an interval. */
49
+ /**
50
+ * Resolved by stop(); every sleep races it. One signal rather than a handle to
51
+ * the current sleep, because there can be more than one: `sweep()` is public
52
+ * and meant to be called directly for an on-demand drain, and its
53
+ * between-batches yield is a sleep of its own. A single handle let that sleep
54
+ * overwrite the scheduled loop's — and then clear it — leaving stop() nothing
55
+ * to wake and close() blocked until the whole interval (an hour, by default)
56
+ * ran out. Same shape as Worker's `stopped$`, and as the Python twin's
57
+ * asyncio.Event.
58
+ */
59
+ private stopSignal;
50
60
  private wake;
51
61
  /** The loop itself, awaited by stop() so no purge outlives the store. */
52
62
  private loop;
@@ -57,6 +67,8 @@ export declare class RetentionSweeper {
57
67
  private readonly purgeInputs;
58
68
  constructor(store: TaskStore, opts: RetentionOptions);
59
69
  start(): void;
70
+ /** Mint a fresh stop signal. */
71
+ private arm;
60
72
  /** Stop sweeping and wait for the sweep in flight, if any. */
61
73
  stop(): Promise<void>;
62
74
  private run;
package/dist/retention.js CHANGED
@@ -27,8 +27,18 @@ export class RetentionSweeper {
27
27
  active = false;
28
28
  /** Set by stop(), so a drain in progress can cut itself short too. */
29
29
  stopping = false;
30
- /** Resolves the current sleep early, so stop() need not wait out an interval. */
31
- wake = null;
30
+ /**
31
+ * Resolved by stop(); every sleep races it. One signal rather than a handle to
32
+ * the current sleep, because there can be more than one: `sweep()` is public
33
+ * and meant to be called directly for an on-demand drain, and its
34
+ * between-batches yield is a sleep of its own. A single handle let that sleep
35
+ * overwrite the scheduled loop's — and then clear it — leaving stop() nothing
36
+ * to wake and close() blocked until the whole interval (an hour, by default)
37
+ * ran out. Same shape as Worker's `stopped$`, and as the Python twin's
38
+ * asyncio.Event.
39
+ */
40
+ stopSignal;
41
+ wake;
32
42
  /** The loop itself, awaited by stop() so no purge outlives the store. */
33
43
  loop = null;
34
44
  intervalMs;
@@ -52,6 +62,7 @@ export class RetentionSweeper {
52
62
  if (!cutoffs.length) {
53
63
  throw new Error("retention.olderThanMs must name at least one status");
54
64
  }
65
+ this.arm();
55
66
  this.purgeInputs = cutoffs.map(([status, ms]) => ({
56
67
  olderThanMs: ms,
57
68
  status,
@@ -67,13 +78,19 @@ export class RetentionSweeper {
67
78
  return;
68
79
  this.active = true;
69
80
  this.stopping = false;
81
+ // A stopped sweeper can be started again, and the old signal is spent.
82
+ this.arm();
70
83
  this.loop = this.run();
71
84
  }
85
+ /** Mint a fresh stop signal. */
86
+ arm() {
87
+ this.stopSignal = new Promise((resolve) => (this.wake = resolve));
88
+ }
72
89
  /** Stop sweeping and wait for the sweep in flight, if any. */
73
90
  async stop() {
74
91
  this.stopping = true;
75
92
  this.active = false;
76
- this.wake?.();
93
+ this.wake();
77
94
  await this.loop;
78
95
  this.loop = null;
79
96
  }
@@ -123,15 +140,13 @@ export class RetentionSweeper {
123
140
  /** Sleep, interruptible by stop(). Unref'd: retention is housekeeping, and a
124
141
  * pending sweep must never be the reason a process refuses to exit. */
125
142
  sleep(ms) {
126
- return new Promise((resolve) => {
127
- const timer = setTimeout(resolve, ms);
143
+ let timer;
144
+ const nap = new Promise((resolve) => {
145
+ timer = setTimeout(resolve, ms);
128
146
  timer.unref?.();
129
- this.wake = () => {
130
- clearTimeout(timer);
131
- resolve();
132
- };
133
- }).finally(() => {
134
- this.wake = null;
135
147
  });
148
+ // Clear the timer whichever side wins, so a stop is never followed by a
149
+ // leftover sweep timer.
150
+ return Promise.race([nap, this.stopSignal]).finally(() => clearTimeout(timer));
136
151
  }
137
152
  }
@@ -147,6 +147,12 @@ export declare abstract class TaskStore {
147
147
  * `watch` degrades to its timer alone.
148
148
  */
149
149
  protected subscribePush?(onSignal: (signal: WatchSignal) => void): () => void;
150
+ /**
151
+ * Tell a store with a push channel which queues this process will wait on, so
152
+ * it can buffer their notifications and ignore everyone else's. Optional: a
153
+ * store without a push channel has nothing to buffer.
154
+ */
155
+ protected registerWakeable?(queues: string[]): void;
150
156
  /**
151
157
  * Nudge the push channel back up if it has dropped. Called from `watch`'s
152
158
  * timer, which is the only thing keeping a client-side subscriber alive: a
@@ -289,9 +295,12 @@ export declare abstract class TaskStore {
289
295
  * `plan` runs with the write lock held, so it must await nothing but that
290
296
  * callback.
291
297
  *
292
- * `names` is the union `plan` might ask for — the probe and the recovery are
293
- * filtered by it. Returns undefined when the probe finds nothing claimable, in
294
- * which case `plan` never runs and no transaction is opened.
298
+ * `names` is the union `plan` might ask for, and it filters the probe's
299
+ * queued-work arm. Lease recovery is deliberately NOT filtered by it — nor is
300
+ * the probe's expired-lease arm — because reclaiming a dead worker's task is
301
+ * every worker's job, whatever names it happens to handle. Returns undefined
302
+ * when the probe finds nothing claimable, in which case `plan` never runs and
303
+ * no transaction is opened.
295
304
  */
296
305
  claimSession<T>(input: {
297
306
  queues: string[];
@@ -1,5 +1,5 @@
1
1
  import { newId } from "../ids.js";
2
- import { AlreadyExists, errorEnvelope, LostLease, ProtocolVersionMismatch, SerializationError, } from "../errors.js";
2
+ import { AlreadyExists, errorEnvelope, LostLease, ProtocolVersionMismatch, SerializationError, UnsupportedBackend, } from "../errors.js";
3
3
  import { isTerminalStatus, rowToRef, rowToTask, STATUSES, } from "../models.js";
4
4
  import { QueueDepthGate } from "../backpressure.js";
5
5
  const rejectMangled = function (_key, v) {
@@ -387,7 +387,16 @@ export class TaskStore {
387
387
  });
388
388
  const timer = setInterval(() => {
389
389
  this.warmPush?.();
390
- emit({ reason: "poll" });
390
+ // Guarded for the same reason PostgresStore guards its push fan-out: a
391
+ // consumer that throws must not take the timer down with it. Losing the
392
+ // timer would silently retire the fallback that makes watch correct
393
+ // where there is no push channel at all.
394
+ try {
395
+ emit({ reason: "poll" });
396
+ }
397
+ catch {
398
+ // The consumer's problem, not the watch's.
399
+ }
391
400
  }, pollMs);
392
401
  timer.unref?.();
393
402
  return () => {
@@ -445,9 +454,12 @@ export class TaskStore {
445
454
  * `plan` runs with the write lock held, so it must await nothing but that
446
455
  * callback.
447
456
  *
448
- * `names` is the union `plan` might ask for — the probe and the recovery are
449
- * filtered by it. Returns undefined when the probe finds nothing claimable, in
450
- * which case `plan` never runs and no transaction is opened.
457
+ * `names` is the union `plan` might ask for, and it filters the probe's
458
+ * queued-work arm. Lease recovery is deliberately NOT filtered by it — nor is
459
+ * the probe's expired-lease arm — because reclaiming a dead worker's task is
460
+ * every worker's job, whatever names it happens to handle. Returns undefined
461
+ * when the probe finds nothing claimable, in which case `plan` never runs and
462
+ * no transaction is opened.
451
463
  */
452
464
  async claimSession(input, plan) {
453
465
  // A list-valued filter cannot be read in claim order, so the planner sorts
@@ -466,6 +478,9 @@ export class TaskStore {
466
478
  limit: 1,
467
479
  lease_expired_error: LEASE_EXPIRED_ERROR_JSON,
468
480
  };
481
+ // Before the probe: a push-channel store has to know what this caller waits
482
+ // on from its FIRST claim, not from its first empty poll.
483
+ this.registerWakeable?.(input.queues);
469
484
  if (!(await this.hasClaimableWork(base)))
470
485
  return undefined;
471
486
  return this.tx(async (fetch) => {
@@ -564,7 +579,7 @@ export class TaskStore {
564
579
  */
565
580
  async completeIn(input, fn) {
566
581
  if (!this.txWithSession) {
567
- throw new Error("this store cannot share a transaction with the caller — " +
582
+ throw new UnsupportedBackend("this store cannot share a transaction with the caller — " +
568
583
  "completeIn requires a Postgres store (see PgExecutor)");
569
584
  }
570
585
  return this.txWithSession(async (fetch, session) => {
@@ -8,11 +8,12 @@
8
8
  * instead of opening a second one, which is what makes a task's settlement
9
9
  * commit in the same transaction as the rows the task produced.
10
10
  *
11
- * Implementing one is small — see `createPoolExecutor` below for the reference
12
- * implementation over `pg`, and PROTOCOL.md for the two things an adapter must
13
- * get right that are easy to miss: int8 must come back as a JS number (every
14
- * cairnq bigint is an epoch-ms or a counter, all inside the safe range), and
15
- * jsonb must come back as an object, not a string.
11
+ * Implementing one is small — see `createPoolExecutor` in pg-pool.ts for the
12
+ * reference implementation over `pg`. An adapter passes rows through as its
13
+ * driver produced them: cairnq normalizes both column types the drivers disagree
14
+ * about (jsonb decoded or not, int8 as text or number) in `rowToTask`, so no
15
+ * adapter has to reconfigure its driver — and none has to change how the
16
+ * application's OWN columns come back in order to satisfy cairnq.
16
17
  */
17
18
  /** A row as the driver hands it back: column name -> value. */
18
19
  export type Row = Record<string, unknown>;
@@ -8,11 +8,12 @@
8
8
  * instead of opening a second one, which is what makes a task's settlement
9
9
  * commit in the same transaction as the rows the task produced.
10
10
  *
11
- * Implementing one is small — see `createPoolExecutor` below for the reference
12
- * implementation over `pg`, and PROTOCOL.md for the two things an adapter must
13
- * get right that are easy to miss: int8 must come back as a JS number (every
14
- * cairnq bigint is an epoch-ms or a counter, all inside the safe range), and
15
- * jsonb must come back as an object, not a string.
11
+ * Implementing one is small — see `createPoolExecutor` in pg-pool.ts for the
12
+ * reference implementation over `pg`. An adapter passes rows through as its
13
+ * driver produced them: cairnq normalizes both column types the drivers disagree
14
+ * about (jsonb decoded or not, int8 as text or number) in `rowToTask`, so no
15
+ * adapter has to reconfigure its driver — and none has to change how the
16
+ * application's OWN columns come back in order to satisfy cairnq.
16
17
  */
17
18
  /**
18
19
  * LISTEN will not work on this connection, and retrying cannot change that.
@@ -14,11 +14,12 @@ async function loadPg() {
14
14
  throw new Error("PostgresStore requires the 'pg' package — install it (e.g. `npm i pg`)");
15
15
  }
16
16
  const pg = (mod.default ?? mod);
17
- // Postgres returns bigint (int8, OID 20) as a string to avoid precision loss.
18
- // Every cairnq bigint is an epoch-ms or a counter, all within Number's safe
19
- // integer range, so parse to number once (globally) to match the Task model
20
- // (*_ms typed as number, same as the SQLite SDK). Set before any query runs.
21
- pg.types.setTypeParser(pg.types.builtins.INT8, (v) => (v == null ? null : Number(v)));
17
+ // Deliberately NOT setting a global int8 type parser here. It would be the
18
+ // shortest fix for Postgres sending bigint as text, and it is what this file
19
+ // used to do — but pg's type parsers are process-global, so a library that
20
+ // installs one silently changes how the APPLICATION's own bigint columns come
21
+ // back. rowToTask normalizes instead, which costs nothing and leaves the
22
+ // caller's driver as they configured it.
22
23
  pgModule = pg;
23
24
  return pg;
24
25
  }
@@ -77,14 +78,39 @@ export async function createPoolExecutor(dsn, opts = {}) {
77
78
  const pool = new pg.Pool({ connectionString: dsn, max: opts.max });
78
79
  if (schema) {
79
80
  // Queued on the connection before it is handed out, so every statement this
80
- // pool ever runs — migrations included — resolves in the right schema. Pooled
81
- // connections come and go, which is why this is per-connection rather than a
82
- // one-off at startup.
81
+ // pool ever runs — migrations included — resolves in the right schema.
82
+ // Pooled connections come and go, which is why this is per-connection rather
83
+ // than a one-off at startup.
84
+ //
85
+ // Deliberately NOT the `options: '-c search_path=...'` connection parameter,
86
+ // which reads like the tidier answer: `pg` builds its config as
87
+ // `Object.assign({}, config, parse(connectionString))`, so a DSN carrying its
88
+ // own `?options=` (a statement_timeout, say — and cairnq's own SchemaMismatch
89
+ // message recommends putting `search_path` there) silently REPLACES it, and
90
+ // the pool then resolves somewhere else entirely. A startup parameter is also
91
+ // a narrower compatibility surface than a plain statement behind a pooler.
92
+ //
93
+ // The Python twin does get to be a startup parameter (asyncpg's
94
+ // `server_settings`), so there a bad schema fails the connection atomically
95
+ // and needs no handler below. That asymmetry is the driver's, not a drift.
83
96
  pool.on("connect", (client) => {
84
- void client.query(`set search_path to "${schema}"`);
97
+ // The rejection must be handled: `void` on a query that rejects is an
98
+ // unhandled rejection, which by default takes the process down. And it
99
+ // must not be swallowed either — a connection whose search_path never
100
+ // landed would go on serving statements against the wrong schema for its
101
+ // whole life, which no error anywhere would explain. Ending it is what
102
+ // makes that loud: this client is about to be handed to a caller, whose
103
+ // first statement then fails instead of quietly writing to `public`.
104
+ client.query(`set search_path to "${schema}"`).catch(() => {
105
+ // `end` is the underlying Client's; the pool hands the connect event a
106
+ // PoolClient whose `release` does not exist yet at this point. A client
107
+ // ended here is dropped from the pool when its holder releases it
108
+ // (pg-pool checks `_queryable`), so this costs a connection, not a slot.
109
+ void client.end().catch(() => {
110
+ // Already gone. The pool discards it either way.
111
+ });
112
+ });
85
113
  });
86
- }
87
- if (schema) {
88
114
  // Created once, here, rather than from the connect handler (which would ask
89
115
  // for CREATE privilege on every new connection) or from the migrations
90
116
  // (which name no schema, by design). Without it the first `create table`
@@ -50,6 +50,14 @@ export declare class PostgresStore extends TaskStore {
50
50
  /** Set once migrations have run and the protocol version checked out. */
51
51
  private executor;
52
52
  private connecting;
53
+ /**
54
+ * Bumped by close(). A doConnect that started before the bump must not publish
55
+ * what it built: close() nulls `connecting` but does not await it, so an
56
+ * in-flight connect outlives the close and would otherwise install an executor
57
+ * — and, worse, a LISTEN connection — on a store nobody will close again. That
58
+ * connection's socket keeps the process alive on its own.
59
+ */
60
+ private generation;
53
61
  private readonly statements;
54
62
  private listener;
55
63
  private listenerConnecting;
@@ -65,6 +73,27 @@ export declare class PostgresStore extends TaskStore {
65
73
  /** Queues notified while nobody was waiting; consumed by the next claimWake
66
74
  * so a wake that lands between polls is not lost. */
67
75
  private readonly pendingQueues;
76
+ /**
77
+ * The queues some caller has actually waited on — the only ones worth
78
+ * buffering into `pendingQueues`.
79
+ *
80
+ * Without it that buffer is a leak: every `cairnq_queued` notification lands in
81
+ * it, and only claimWake consumes, and only the queues it names. A client-side
82
+ * handle never calls claimWake at all, and a worker only ever names its own
83
+ * queues — so a deployment whose queue names carry an identity (`user:1234`)
84
+ * grows the set forever. This one is bounded by what this process claims from,
85
+ * which is its own configuration.
86
+ *
87
+ * Registered from `claimSession` as well as from `claimWake`, so a worker's
88
+ * queues are known from its first claim rather than from its first empty poll:
89
+ * a notification landing between those two would otherwise be dropped, costing
90
+ * a poll interval exactly when work has just arrived.
91
+ *
92
+ * The bound holds because a worker's queue list is its configuration. A caller
93
+ * driving `claimWake`/`claim` with per-tenant queue names directly still
94
+ * accumulates one entry per distinct name it passes.
95
+ */
96
+ private readonly wakeableQueues;
68
97
  /** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
69
98
  private readonly waiters;
70
99
  /** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
@@ -82,10 +111,45 @@ export declare class PostgresStore extends TaskStore {
82
111
  connect(): Promise<void>;
83
112
  close(): Promise<void>;
84
113
  private ensure;
114
+ /**
115
+ * A fresh connection gets a fresh verdict on LISTEN.
116
+ *
117
+ * Every path here connects lazily, so a store used again after close simply
118
+ * reconnects — and without this it would reconnect permanently degraded to
119
+ * polling, having decided "no push channel" about a connection that no longer
120
+ * exists. Called from the constructor for the initial state and from doConnect
121
+ * for every later one, so the three fields cannot be re-armed by halves. Only
122
+ * ever safe under doConnect's generation check.
123
+ */
124
+ private resetListenerState;
85
125
  private doConnect;
126
+ /**
127
+ * Refuse a connection pointed somewhere other than the deployment's cairnq.
128
+ *
129
+ * Two shapes, because `schema` means "the schema cairnq's tables live in" and
130
+ * cairnq can either arrange that (it built the connection) or only check it
131
+ * (the caller's executor did):
132
+ *
133
+ * - `schema` configured -> assert the connection actually resolves there. On
134
+ * the DSN path this is a cheap self-check; on an injected executor it is the
135
+ * only way to state the expectation at all.
136
+ * - `schema` not configured -> the dangerous case is being about to create a
137
+ * SECOND installation while one already exists elsewhere in this database,
138
+ * which is exactly what a mismatched pair of SDKs does. Joining an existing
139
+ * installation is fine no matter what else is around, so the check is
140
+ * deliberately narrow: it fires only when this schema has no cairnq and some
141
+ * other schema does.
142
+ *
143
+ * That narrowness is what keeps it from crying wolf. Two applications each
144
+ * running their own cairnq in their own schema are legitimate; the second one
145
+ * to be set up trips this once, and saying `schema` explicitly — which such a
146
+ * deployment should be doing anyway — is both the fix and the confirmation.
147
+ */
148
+ private checkSchema;
86
149
  private applyMigrations;
87
150
  private readProtocolVersion;
88
151
  protocolVersion(): Promise<number>;
152
+ protected registerWakeable(queues: string[]): void;
89
153
  claimWake(queues: string[], timeoutMs: number): Promise<void>;
90
154
  protected subscribePush(onSignal: (signal: WatchSignal) => void): () => void;
91
155
  protected warmPush(): void;
@@ -1,3 +1,4 @@
1
+ import { SchemaMismatch } from "../errors.js";
1
2
  import { loadMigrations, loadStatements } from "../sql.js";
2
3
  import { checkProtocolVersion, COMMENT, NAMED, statementParams, TaskStore, } from "./base.js";
3
4
  import { ListenUnavailable } from "./pg-executor.js";
@@ -69,6 +70,14 @@ export class PostgresStore extends TaskStore {
69
70
  /** Set once migrations have run and the protocol version checked out. */
70
71
  executor = null;
71
72
  connecting = null;
73
+ /**
74
+ * Bumped by close(). A doConnect that started before the bump must not publish
75
+ * what it built: close() nulls `connecting` but does not await it, so an
76
+ * in-flight connect outlives the close and would otherwise install an executor
77
+ * — and, worse, a LISTEN connection — on a store nobody will close again. That
78
+ * connection's socket keeps the process alive on its own.
79
+ */
80
+ generation = 0;
72
81
  statements;
73
82
  // ------------------------------------------------------- LISTEN/NOTIFY state
74
83
  // The executor subscribes one dedicated connection to both channels (see
@@ -87,9 +96,31 @@ export class PostgresStore extends TaskStore {
87
96
  * hammered from the poll loop. */
88
97
  listenerRetryAt = 0;
89
98
  listenerBackoffMs = LISTENER_RETRY_MS;
99
+ // NB: these three are re-armed together by resetListenerState().
90
100
  /** Queues notified while nobody was waiting; consumed by the next claimWake
91
101
  * so a wake that lands between polls is not lost. */
92
102
  pendingQueues = new Set();
103
+ /**
104
+ * The queues some caller has actually waited on — the only ones worth
105
+ * buffering into `pendingQueues`.
106
+ *
107
+ * Without it that buffer is a leak: every `cairnq_queued` notification lands in
108
+ * it, and only claimWake consumes, and only the queues it names. A client-side
109
+ * handle never calls claimWake at all, and a worker only ever names its own
110
+ * queues — so a deployment whose queue names carry an identity (`user:1234`)
111
+ * grows the set forever. This one is bounded by what this process claims from,
112
+ * which is its own configuration.
113
+ *
114
+ * Registered from `claimSession` as well as from `claimWake`, so a worker's
115
+ * queues are known from its first claim rather than from its first empty poll:
116
+ * a notification landing between those two would otherwise be dropped, costing
117
+ * a poll interval exactly when work has just arrived.
118
+ *
119
+ * The bound holds because a worker's queue list is its configuration. A caller
120
+ * driving `claimWake`/`claim` with per-tenant queue names directly still
121
+ * accumulates one entry per distinct name it passes.
122
+ */
123
+ wakeableQueues = new Set();
93
124
  /** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
94
125
  waiters = new Map();
95
126
  /** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
@@ -111,6 +142,7 @@ export class PostgresStore extends TaskStore {
111
142
  await this.ensure();
112
143
  }
113
144
  async close() {
145
+ this.generation++; // disown any connect still in flight
114
146
  this.listenerUnavailable = true; // no revival after close
115
147
  this.dropListener();
116
148
  const executor = this.executor;
@@ -135,10 +167,29 @@ export class PostgresStore extends TaskStore {
135
167
  }
136
168
  await this.connecting;
137
169
  }
170
+ /**
171
+ * A fresh connection gets a fresh verdict on LISTEN.
172
+ *
173
+ * Every path here connects lazily, so a store used again after close simply
174
+ * reconnects — and without this it would reconnect permanently degraded to
175
+ * polling, having decided "no push channel" about a connection that no longer
176
+ * exists. Called from the constructor for the initial state and from doConnect
177
+ * for every later one, so the three fields cannot be re-armed by halves. Only
178
+ * ever safe under doConnect's generation check.
179
+ */
180
+ resetListenerState() {
181
+ this.listenerUnavailable = false;
182
+ this.listenerRetryAt = 0;
183
+ this.listenerBackoffMs = LISTENER_RETRY_MS;
184
+ }
138
185
  async doConnect() {
186
+ const generation = this.generation;
139
187
  const executor = this.provided ??
140
188
  (await createPoolExecutor(this.dsn, { max: this.opts.max, schema: this.opts.schema }));
141
189
  try {
190
+ // Before migrations, which would otherwise create the very installation
191
+ // this is trying to warn about.
192
+ await this.checkSchema(executor);
142
193
  await this.applyMigrations(executor);
143
194
  checkProtocolVersion(await this.readProtocolVersion(executor));
144
195
  }
@@ -148,11 +199,73 @@ export class PostgresStore extends TaskStore {
148
199
  await executor.close().catch(() => { });
149
200
  throw e;
150
201
  }
202
+ // Closed while we were connecting: this executor has no owner, so drop it
203
+ // rather than publishing it. Nothing below may run — clearing
204
+ // `listenerUnavailable` here is exactly how a closed store would end up with
205
+ // a listener holding a socket forever.
206
+ if (generation !== this.generation) {
207
+ if (!this.provided)
208
+ await executor.close().catch(() => { });
209
+ throw new Error("store was closed while connecting");
210
+ }
151
211
  this.executor = executor; // publish only a fully-migrated, version-checked executor
212
+ this.resetListenerState();
152
213
  // Warm the LISTEN subscription in the background so the first idle sleep is
153
214
  // already wakeable. Fire-and-forget: failure just means polling.
154
215
  this.listenerReady();
155
216
  }
217
+ /**
218
+ * Refuse a connection pointed somewhere other than the deployment's cairnq.
219
+ *
220
+ * Two shapes, because `schema` means "the schema cairnq's tables live in" and
221
+ * cairnq can either arrange that (it built the connection) or only check it
222
+ * (the caller's executor did):
223
+ *
224
+ * - `schema` configured -> assert the connection actually resolves there. On
225
+ * the DSN path this is a cheap self-check; on an injected executor it is the
226
+ * only way to state the expectation at all.
227
+ * - `schema` not configured -> the dangerous case is being about to create a
228
+ * SECOND installation while one already exists elsewhere in this database,
229
+ * which is exactly what a mismatched pair of SDKs does. Joining an existing
230
+ * installation is fine no matter what else is around, so the check is
231
+ * deliberately narrow: it fires only when this schema has no cairnq and some
232
+ * other schema does.
233
+ *
234
+ * That narrowness is what keeps it from crying wolf. Two applications each
235
+ * running their own cairnq in their own schema are legitimate; the second one
236
+ * to be set up trips this once, and saying `schema` explicitly — which such a
237
+ * deployment should be doing anyway — is both the fix and the confirmation.
238
+ */
239
+ async checkSchema(executor) {
240
+ // One row per installation; the statement's LEFT JOIN guarantees at least
241
+ // one, so current_schema is readable even where cairnq lives nowhere yet.
242
+ const rows = await executor.query(this.statements.installations, []);
243
+ const current = rows[0]?.current_schema ?? null;
244
+ const installations = rows
245
+ .map((r) => r.schema)
246
+ .filter((s) => s != null);
247
+ const wanted = this.opts.schema;
248
+ if (wanted != null) {
249
+ if (current !== wanted) {
250
+ throw new SchemaMismatch(`cairnq is configured for schema ${JSON.stringify(wanted)} but this ` +
251
+ `connection resolves to ${JSON.stringify(current)} — check the ` +
252
+ `connection's search_path`);
253
+ }
254
+ return;
255
+ }
256
+ // A search_path naming nothing that exists: there is no "here" to compare
257
+ // against, and the migrations are about to fail with a clearer message.
258
+ if (current === null)
259
+ return;
260
+ if (installations.length === 0 || installations.includes(current))
261
+ return;
262
+ throw new SchemaMismatch(`cairnq tables already exist in schema ${installations.map((s) => JSON.stringify(s)).join(", ")} ` +
263
+ `of this database, but this connection resolves to ${JSON.stringify(current)}, where there are ` +
264
+ `none. Connecting would create a second, parallel installation that the other one can never see ` +
265
+ `— an API and a worker split this way agree about everything except where, and no task crosses. ` +
266
+ `Point this process at the same schema (\`schema\` option, or \`options=-c search_path=...\` in the ` +
267
+ `DSN), or pass \`schema\` explicitly to confirm a separate installation is what you meant.`);
268
+ }
156
269
  async applyMigrations(executor) {
157
270
  await executor.exec("create table if not exists cairnq_migrations " +
158
271
  "(name text primary key, applied_at_ms bigint not null)");
@@ -182,7 +295,12 @@ export class PostgresStore extends TaskStore {
182
295
  return this.readProtocolVersion(this.executor);
183
296
  }
184
297
  // ------------------------------------------------------------ wake channel
298
+ registerWakeable(queues) {
299
+ for (const q of queues)
300
+ this.wakeableQueues.add(q);
301
+ }
185
302
  async claimWake(queues, timeoutMs) {
303
+ this.registerWakeable(queues);
186
304
  if (!this.listenerReady())
187
305
  return super.claimWake(queues, timeoutMs);
188
306
  const deadline = Date.now() + timeoutMs;
@@ -297,7 +415,9 @@ export class PostgresStore extends TaskStore {
297
415
  onNotification(channel, payload) {
298
416
  let key;
299
417
  if (channel === QUEUED_CHANNEL && payload) {
300
- this.pendingQueues.add(payload);
418
+ // Buffered only for a queue someone waits on — see wakeableQueues.
419
+ if (this.wakeableQueues.has(payload))
420
+ this.pendingQueues.add(payload);
301
421
  key = "queued";
302
422
  this.publish({ reason: "queued", queue: payload });
303
423
  }
package/dist/worker.d.ts CHANGED
@@ -98,7 +98,7 @@ export declare class Worker {
98
98
  private readonly queues;
99
99
  private readonly opts;
100
100
  private readonly handlers;
101
- private readonly workerId;
101
+ private readonly ownId;
102
102
  /** Payload bytes charged to running handlers — see maxInFlightBytes. */
103
103
  private inFlightBytes;
104
104
  /** Calls in flight, for the names that cap their own concurrency. */
@@ -139,7 +139,9 @@ export declare class Worker {
139
139
  max?: number;
140
140
  schema?: string;
141
141
  }): Worker;
142
- get id(): string;
142
+ /** This worker's id — what `worker_id` on a running task points at, and what
143
+ * the Python SDK calls `worker_id` too. */
144
+ get workerId(): string;
143
145
  task(handler: Handler): this;
144
146
  task(name: string, handler: Handler): this;
145
147
  task<P, R>(def: TaskDef<P, R>, handler: TypedHandler<P, R>): this;