cairnq 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +9 -32
  2. package/dist/_protocol/sql/postgres/purge.sql +10 -10
  3. package/dist/_protocol/sql/sqlite/purge.sql +10 -10
  4. package/dist/backpressure.d.ts +0 -3
  5. package/dist/backpressure.js +1 -3
  6. package/dist/client.d.ts +42 -46
  7. package/dist/client.js +27 -43
  8. package/dist/context.d.ts +23 -1
  9. package/dist/context.js +18 -7
  10. package/dist/errors.d.ts +20 -26
  11. package/dist/errors.js +23 -35
  12. package/dist/index.d.ts +4 -10
  13. package/dist/index.js +3 -6
  14. package/dist/models.d.ts +14 -1
  15. package/dist/models.js +17 -5
  16. package/dist/retention.d.ts +22 -66
  17. package/dist/retention.js +53 -64
  18. package/dist/store/base.d.ts +29 -90
  19. package/dist/store/base.js +136 -123
  20. package/dist/store/pg-executor.d.ts +11 -0
  21. package/dist/store/pg-executor.js +11 -0
  22. package/dist/store/pg-pool.js +14 -0
  23. package/dist/store/postgres.d.ts +18 -12
  24. package/dist/store/postgres.js +23 -29
  25. package/dist/store/sqlite.d.ts +39 -2
  26. package/dist/store/sqlite.js +108 -29
  27. package/dist/worker.d.ts +38 -93
  28. package/dist/worker.js +181 -234
  29. package/package.json +1 -1
  30. package/src/backpressure.ts +1 -5
  31. package/src/client.ts +66 -71
  32. package/src/context.ts +28 -7
  33. package/src/errors.ts +25 -36
  34. package/src/index.ts +4 -27
  35. package/src/models.ts +17 -5
  36. package/src/retention.ts +50 -115
  37. package/src/store/base.ts +131 -166
  38. package/src/store/pg-executor.ts +11 -0
  39. package/src/store/pg-pool.ts +14 -0
  40. package/src/store/postgres.ts +24 -31
  41. package/src/store/sqlite.ts +108 -27
  42. package/src/worker.ts +208 -300
  43. package/dist/_protocol/sql/postgres/stats.sql +0 -23
  44. package/dist/_protocol/sql/sqlite/stats.sql +0 -23
package/src/store/base.ts CHANGED
@@ -18,6 +18,47 @@ import {
18
18
  } from "../models.js";
19
19
  import { type BackpressureOptions, QueueDepthGate } from "../backpressure.js";
20
20
 
21
+ /**
22
+ * Whether `JSON.stringify` would write this object without its contents.
23
+ *
24
+ * Every such value serializes to `{}` — or, for a typed array, to an object
25
+ * keyed by index — with everything it actually held silently gone. A handler
26
+ * that returns a `Map` would otherwise record `{}` as the task's result:
27
+ * succeeded, no error, and nothing left to recover the value from. That is the
28
+ * same class of silent mangle as `NaN` becoming `null`, so it is refused the
29
+ * same way.
30
+ *
31
+ * Structural rather than a list of built-in names, because a list is only ever
32
+ * as current as the day it was written. An enumeration of `Map`/`Set`/`Promise`/
33
+ * the typed arrays already missed `Float16Array` (ES2025, and `engines` here is
34
+ * node >=22), `Blob`, `Headers`, `URLSearchParams`, and — unfixably, since its
35
+ * tag is plain `[object Object]` — a class whose state is all private fields.
36
+ * The rule below catches those, and whatever the platform adds next, with
37
+ * nothing to maintain:
38
+ *
39
+ * - **A non-plain object with no own enumerable properties.** Its state lives
40
+ * somewhere `JSON.stringify` cannot reach (internal slots, private fields, a
41
+ * host binding), so `{}` is all that would be written. A *plain* `{}` is
42
+ * deliberately exempt: an empty object is a value, not a loss.
43
+ * - **Any `ArrayBuffer` view.** A typed array writes its indices as string keys
44
+ * and reads back as an object, not an array — non-empty, so the first rule
45
+ * does not see it.
46
+ *
47
+ * A class instance that carries ordinary properties still crosses: those
48
+ * properties are exactly what gets written, and nothing is lost. `Date` never
49
+ * reaches here at all — `toJSON` has already replaced it with its ISO string by
50
+ * the time a replacer is called, which is also why `toJSON` remains the escape
51
+ * hatch for anything that wants to define its own JSON form.
52
+ */
53
+ function emptiesItselfOut(v: object): boolean {
54
+ if (ArrayBuffer.isView(v)) return true;
55
+ const proto = Object.getPrototypeOf(v);
56
+ // Plain and null-prototype objects are the two shapes whose own enumerable
57
+ // properties ARE their contents, so an empty one is honestly empty.
58
+ if (proto === Object.prototype || proto === null) return false;
59
+ return Object.keys(v).length === 0;
60
+ }
61
+
21
62
  const rejectMangled = function (this: unknown, _key: string, v: unknown): unknown {
22
63
  if (typeof v === "number" && !Number.isFinite(v)) {
23
64
  throw new SerializationError(`non-finite number ${v} is not JSON-serializable`);
@@ -28,15 +69,22 @@ const rejectMangled = function (this: unknown, _key: string, v: unknown): unknow
28
69
  if (Array.isArray(this) && (v === undefined || typeof v === "function" || typeof v === "symbol")) {
29
70
  throw new SerializationError(`${typeof v} inside an array is not JSON-serializable`);
30
71
  }
72
+ if (v !== null && typeof v === "object" && !Array.isArray(v) && emptiesItselfOut(v)) {
73
+ const name = (v as { constructor?: { name?: string } }).constructor?.name ?? "object";
74
+ throw new SerializationError(
75
+ `${name} carries nothing JSON.stringify can see and would be written as an empty object`,
76
+ );
77
+ }
31
78
  return v;
32
79
  };
33
80
 
34
81
  /** Encode a value for a protocol JSON column, raising SerializationError on
35
82
  * anything JSON cannot represent. Refuses what JSON.stringify would silently
36
- * mangle into `null`: NaN/Infinity anywhere, undefined/function/symbol inside an
37
- * array, and a top-level undefined that disappears entirely — either way the
38
- * twin SDK reads back something other than what the caller meant (the Python
39
- * SDK rejects the same values, via allow_nan=False). */
83
+ * mangle: NaN/Infinity anywhere, undefined/function/symbol inside an array, a
84
+ * top-level undefined that disappears entirely, and the objects that would be
85
+ * written as an empty one (see emptiesItselfOut) — either way the twin SDK
86
+ * reads back something other than what the caller meant. The Python SDK rejects
87
+ * the same classes of value; see its dump_json. */
40
88
  export function dumpJson(value: unknown): string {
41
89
  let text: string | undefined;
42
90
  try {
@@ -48,10 +96,36 @@ export function dumpJson(value: unknown): string {
48
96
  if (text === undefined) {
49
97
  throw new SerializationError(`value of type ${typeof value} is not JSON-serializable`);
50
98
  }
51
- // Every mangled value reaches the output as the literal `null`, so a
52
- // null-free result needs no strict pass — this keeps the replacer (which
53
- // forfeits V8's native stringifier) off the hot path.
54
- if (text.includes("null")) JSON.stringify(value, rejectMangled);
99
+ // Every mangled value betrays itself in the output, so the strict pass — which
100
+ // forfeits V8's native stringifier — runs only when one of its traces is
101
+ // there: `null` for a mangled scalar, `{}` for an opaque container emptied
102
+ // out, `{"0":` for a typed array rewritten as an index-keyed object. Each is a
103
+ // superset (a payload carrying an empty object, or a literal "0" key, merely
104
+ // pays for a second pass that then finds nothing), and together they are
105
+ // exhaustive over everything rejectMangled rejects. Measured on a null-free
106
+ // payload the three checks cost ~20% of the encode, against ~130% for running
107
+ // the replacer unconditionally.
108
+ //
109
+ // The second pass builds a whole string and throws it away, so a hit costs
110
+ // 2.4x the encode (211us against 87us on a 42KB payload) where a validating
111
+ // WALK would cost 28us. It stays a second stringify anyway, twice over: a walk
112
+ // would have to reproduce seven of JSON.stringify's traversal semantics by
113
+ // hand — toJSON applied before the replacer sees a value, boxed primitives,
114
+ // sparse arrays, own-enumerable ordering, getters called once each, and a
115
+ // cycle guard the native pass does not need here but a walk would (an impure
116
+ // getter can introduce a cycle the first pass did not see, where stringify
117
+ // throws and a walk recurses forever) — and a divergence between the two is
118
+ // either a false rejection or the silent-data-loss bug coming back. It is also
119
+ // the wrong thing to optimise: every dumpJson sits next to a durable write, so
120
+ // 124us is noise beside the transaction it precedes. Revisit only with a
121
+ // profile showing this in double digits of submit latency, and pair any walk
122
+ // with a differential test asserting the two agree over generated values.
123
+ //
124
+ // The `{}` trace cannot be narrowed, either: an emptied-out Map and a
125
+ // legitimate empty object are byte-identical in the output.
126
+ if (text.includes("null") || text.includes("{}") || text.includes('{"0":')) {
127
+ JSON.stringify(value, rejectMangled);
128
+ }
55
129
  return text;
56
130
  }
57
131
 
@@ -106,7 +180,8 @@ export interface SubmitInput {
106
180
  parentId?: string | null;
107
181
  rootId?: string | null;
108
182
  correlationId?: string | null;
109
- runAtDelayMs?: number;
183
+ /** Run no earlier than this many ms from now. */
184
+ delayMs?: number;
110
185
  }
111
186
 
112
187
  export interface ListInput {
@@ -216,7 +291,9 @@ export function specialize(sql: string, params: Params): string {
216
291
  if (params[name] != null) active += name + ",";
217
292
  }
218
293
  if (!active) return sql;
219
- const key = active + sql;
294
+ // "\u0000" delimits: the names run together with no separator otherwise, and
295
+ // ("a," + "b,SELECT") and ("a,b," + "SELECT") would be one cache entry.
296
+ const key = active + "\u0000" + sql;
220
297
  let out = specialCache.get(key);
221
298
  if (out === undefined) {
222
299
  out = sql.replace(OPTIONAL_FILTER, (whole, name: string, column: string) =>
@@ -258,35 +335,6 @@ export function statementParams(sql: string): readonly string[] {
258
335
  * them in one place is what stops SQLite and Postgres from drifting apart in
259
336
  * behavior; the shared SQL already stops them from drifting in wording.
260
337
  */
261
- /**
262
- * Why `watch` is calling back.
263
- *
264
- * `queued` / `done` come from the store's push channel and name what moved;
265
- * `poll` is the timer saying the watch cannot rule out a change. None of them
266
- * carries state — the row is the truth.
267
- */
268
- export interface WatchSignal {
269
- reason: "queued" | "done" | "poll";
270
- /** The queue a task was queued on. Only on `queued`. */
271
- queue?: string;
272
- /** The task that reached a terminal status. Only on `done`. */
273
- taskId?: string;
274
- }
275
-
276
- export interface WatchOptions {
277
- /** Restrict `queued` signals to these queues. Unset watches every queue. */
278
- queues?: string[];
279
- /**
280
- * How often to signal in the absence of a push channel — and, where there is
281
- * one, how long a dropped listener can go unnoticed. The default trades a
282
- * dashboard's idle query rate against how stale it may look.
283
- */
284
- pollMs?: number;
285
- }
286
-
287
- /** See WatchOptions.pollMs. */
288
- export const DEFAULT_WATCH_POLL_MS = 2_000;
289
-
290
338
  export abstract class TaskStore {
291
339
  /** Set by useBackpressure; null means submit is ungated. */
292
340
  private gate: QueueDepthGate | null = null;
@@ -324,13 +372,6 @@ export abstract class TaskStore {
324
372
  */
325
373
  protected txWithSession?<T>(fn: (fetch: Fetch, session: unknown) => Promise<T>): Promise<T>;
326
374
 
327
- /**
328
- * Register for this store's push channel, if it has one; returns an
329
- * unsubscribe. A store without a push channel does not implement this, and
330
- * `watch` degrades to its timer alone.
331
- */
332
- protected subscribePush?(onSignal: (signal: WatchSignal) => void): () => void;
333
-
334
375
  /**
335
376
  * Tell a store with a push channel which queues this process will wait on, so
336
377
  * it can buffer their notifications and ignore everyone else's. Optional: a
@@ -338,14 +379,6 @@ export abstract class TaskStore {
338
379
  */
339
380
  protected registerWakeable?(queues: string[]): void;
340
381
 
341
- /**
342
- * Nudge the push channel back up if it has dropped. Called from `watch`'s
343
- * timer, which is the only thing keeping a client-side subscriber alive: a
344
- * process that never claims never calls claimWake, so without this a listener
345
- * that died once would never come back there.
346
- */
347
- protected warmPush?(): void;
348
-
349
382
  /**
350
383
  * Whether it is worth opening the claim transaction at all — the read-only
351
384
  * `claimable_probe`, which every dialect ships.
@@ -384,6 +417,28 @@ export abstract class TaskStore {
384
417
  }
385
418
 
386
419
  // --------------------------------------------------------------- internals
420
+ /**
421
+ * Whether this store's driver hands JSON columns back as text.
422
+ *
423
+ * Declared by the store rather than sniffed per value, because for a JSON
424
+ * *string* the two wire forms are indistinguishable: the text form of
425
+ * `"hello"` and the decoded form of `"hello"` are both strings, and treating
426
+ * the decoded one as text parses it twice — `"s3://…"` threw, `"42"` came back
427
+ * as the number 42. See rowToTask.
428
+ *
429
+ * True is the SQLite answer and the safe default: its JSON columns are TEXT,
430
+ * always. PostgresStore replaces it with what its driver actually does, which
431
+ * it measures rather than assumes — an injected executor's driver is the
432
+ * application's choice, not cairnq's.
433
+ */
434
+ protected jsonIsText = true;
435
+
436
+ /** rowToTask, told this store's wire form. Every row in this class goes
437
+ * through here so no call site has to remember to pass it. */
438
+ private toTask(row: any): Task {
439
+ return rowToTask(row, this.jsonIsText);
440
+ }
441
+
387
442
  /**
388
443
  * An ownership-checked worker write (heartbeat/progress/succeed/complete/fail).
389
444
  * Each statement's WHERE pins worker_id + a live lease, so 0 rows back means
@@ -392,11 +447,13 @@ export abstract class TaskStore {
392
447
  private async ownedWrite(name: string, taskId: string, params: Params): Promise<Task> {
393
448
  const rows = await this.fetch(name, params);
394
449
  if (!rows.length) throw new LostLease(taskId);
395
- return rowToTask(rows[0]);
450
+ return this.toTask(rows[0]);
396
451
  }
397
452
 
398
- private static one(rows: any[]): Task | null {
399
- return rows.length ? rowToTask(rows[0]) : null;
453
+ // An instance method, unlike oneRef: mapping a Task reads this store's JSON
454
+ // wire form, and a TaskRef has no JSON column to read.
455
+ private one(rows: any[]): Task | null {
456
+ return rows.length ? this.toTask(rows[0]) : null;
400
457
  }
401
458
 
402
459
  private static oneRef(rows: any[]): TaskRef | null {
@@ -426,7 +483,7 @@ export abstract class TaskStore {
426
483
  metadata: dumpJson(input.metadata ?? {}),
427
484
  max_attempts: input.maxAttempts ?? 3,
428
485
  priority: input.priority ?? 0,
429
- delay_ms: input.runAtDelayMs ?? 0,
486
+ delay_ms: input.delayMs ?? 0,
430
487
  parent_id: input.parentId ?? null,
431
488
  root_id: input.rootId ?? id,
432
489
  correlation_id: input.correlationId ?? null,
@@ -445,14 +502,14 @@ export abstract class TaskStore {
445
502
  if (input.maxAttempts != null && input.maxAttempts < 1) {
446
503
  throw new Error(`maxAttempts must be >= 1, got ${input.maxAttempts}`);
447
504
  }
448
- if (input.runAtDelayMs != null && input.runAtDelayMs < 0) {
449
- throw new Error(`runAtDelayMs must be >= 0, got ${input.runAtDelayMs}`);
505
+ if (input.delayMs != null && input.delayMs < 0) {
506
+ throw new Error(`delayMs must be >= 0, got ${input.delayMs}`);
450
507
  }
451
508
  // After validation and before the first write: bad arguments should fail
452
509
  // now, not after waiting out a full queue. Reads the resolved queue, so the
453
510
  // gate cannot throttle one queue while the row lands on another.
454
511
  if (this.gate) await this.gate.acquire(ins.queue as string);
455
- if (key === null) return rowToTask((await this.fetch("insert_task", ins))[0]);
512
+ if (key === null) return this.toTask((await this.fetch("insert_task", ins))[0]);
456
513
 
457
514
  // A key makes submit a read-then-write, so it has to be one transaction —
458
515
  // opened by taking the key's lock, because on Postgres the transaction alone
@@ -469,7 +526,7 @@ export abstract class TaskStore {
469
526
  const current = (await fetch("get", { id: existing[0].task_id }))[0];
470
527
  if (current) {
471
528
  if (conflict === "reject") throw new AlreadyExists(key);
472
- if (reusable(conflict, current.status as TaskStatus)) return rowToTask(current);
529
+ if (reusable(conflict, current.status as TaskStatus)) return this.toTask(current);
473
530
  // The strategy declined the recorded task, so the key repoints to the
474
531
  // fresh one inserted below. Cancel only what is still live: a terminal
475
532
  // task has nothing to stop, and cancelling it would rewrite a settled
@@ -481,16 +538,16 @@ export abstract class TaskStore {
481
538
  }
482
539
  const row = (await fetch("insert_task", ins))[0];
483
540
  await fetch("upsert_key", { key, task_id: id });
484
- return rowToTask(row);
541
+ return this.toTask(row);
485
542
  });
486
543
  }
487
544
 
488
545
  async get(taskId: string): Promise<Task | null> {
489
- return TaskStore.one(await this.fetch("get", { id: taskId }));
546
+ return this.one(await this.fetch("get", { id: taskId }));
490
547
  }
491
548
 
492
549
  async getByKey(key: string): Promise<Task | null> {
493
- return TaskStore.one(await this.fetch("get_by_key", { key }));
550
+ return this.one(await this.fetch("get_by_key", { key }));
494
551
  }
495
552
 
496
553
  /** The wait loop's probe: id + status alone, so polling a task with a large
@@ -523,15 +580,15 @@ export abstract class TaskStore {
523
580
  limit: input.limit ?? 100,
524
581
  offset: input.offset ?? 0,
525
582
  });
526
- return rows.map(rowToTask);
583
+ return rows.map((r) => this.toTask(r));
527
584
  }
528
585
 
529
586
  async cancel(taskId: string): Promise<Task | null> {
530
- return TaskStore.one(await this.fetch("cancel", { id: taskId }));
587
+ return this.one(await this.fetch("cancel", { id: taskId }));
531
588
  }
532
589
 
533
590
  async retry(taskId: string, opts: { resetAttempt?: boolean } = {}): Promise<Task | null> {
534
- return TaskStore.one(
591
+ return this.one(
535
592
  await this.fetch("retry", { id: taskId, reset_attempt: opts.resetAttempt ?? false }),
536
593
  );
537
594
  }
@@ -555,7 +612,7 @@ export abstract class TaskStore {
555
612
  await fetch("lock_key", { key });
556
613
  const existing = (await fetch("get_key", { key })) as { task_id: string }[];
557
614
  if (!existing.length) return null;
558
- return TaskStore.one(await fetch(name, { id: existing[0].task_id, ...params }));
615
+ return this.one(await fetch(name, { id: existing[0].task_id, ...params }));
559
616
  });
560
617
  }
561
618
 
@@ -580,104 +637,12 @@ export abstract class TaskStore {
580
637
  return rows.map((r) => r.id as string);
581
638
  }
582
639
 
583
- /**
584
- * Task counts per queue, keyed by status and zero-filled across all statuses —
585
- * `(await stats()).default.queued` is the backlog of a queue. A queue appears
586
- * only while it has rows; terminal tasks keep counting until `purge` removes
587
- * them.
588
- *
589
- * `queue` restricts the aggregate to one queue, which is also what stops the
590
- * caller paying for every other queue's rows: one installation carrying two
591
- * workloads is the coordination this project recommends, and the unfiltered
592
- * form reads the whole table. A named queue is always present in the result,
593
- * zero-filled if it has no rows at all — asking about a specific queue and
594
- * getting `undefined` back would make every caller write the same fallback.
595
- *
596
- * Filtered or not, this COUNTS, so it costs what it counts: a whole queue,
597
- * terminal rows included. Right for a dashboard, wrong on an interval — poll
598
- * `queueDepth`, which is bounded, and keep this for when the real numbers are
599
- * the point.
600
- */
601
- async stats(queue?: string): Promise<Record<string, Record<TaskStatus, number>>> {
602
- const zeros = (): Record<TaskStatus, number> =>
603
- Object.fromEntries(STATUSES.map((s) => [s, 0])) as Record<TaskStatus, number>;
604
- const out: Record<string, Record<TaskStatus, number>> = {};
605
- // Seed before the query, not after: a named queue with no rows returns no
606
- // rows to seed from, and that is exactly the case the promise is about.
607
- if (queue != null) out[queue] = zeros();
608
- for (const row of await this.fetch("stats", { queue: queue ?? null })) {
609
- const per = (out[row.queue] ??= zeros());
610
- per[row.status as TaskStatus] = Number(row.count);
611
- }
612
- return out;
613
- }
614
-
615
- /**
616
- * Call `onSignal` when the tasks on `queues` may have changed — something was
617
- * queued, or something finished.
618
- *
619
- * This is notify-ACCELERATED POLLING, not an event log, and the difference is
620
- * the whole contract. Where a push channel is available (Postgres LISTEN) an
621
- * idle watch costs nothing and a signal arrives within milliseconds of the
622
- * event. Where it is not — a transaction-mode pooler refuses LISTEN, SQLite has
623
- * no channel at all — the timer alone still delivers `poll` signals, so a
624
- * consumer that re-reads on every signal is correct in both cases and merely
625
- * less prompt in one.
626
- *
627
- * What it will NOT do is promise that a signal means something happened, or
628
- * that every event produces its own signal. Treat a signal as "re-read now"
629
- * and take the truth from `stats()` / `list()` / `get()`, which is where it
630
- * lives. `reason` is a hint for reading less: a `done` signal names the task,
631
- * so a dashboard can refresh that row instead of the list.
632
- *
633
- * Returns an unsubscribe. The timer is unref'd — watching does not hold a
634
- * process open.
635
- */
636
- watch(opts: WatchOptions, onSignal: (signal: WatchSignal) => void): () => void {
637
- const pollMs = Math.max(1, opts.pollMs ?? DEFAULT_WATCH_POLL_MS);
638
- const queues = opts.queues ?? null;
639
- let live = true;
640
- const emit = (signal: WatchSignal): void => {
641
- // A signal delivered after unsubscribe would have the consumer re-reading
642
- // a store it has stopped caring about, possibly a closed one.
643
- if (live) onSignal(signal);
644
- };
645
- const unsubscribe = this.subscribePush?.((signal) => {
646
- // A queued signal names its queue, so a watch scoped to some queues can
647
- // drop the rest. A done signal names only the task — which queue it was on
648
- // is not in the notification, so it is never filtered out.
649
- if (signal.reason === "queued" && queues && signal.queue && !queues.includes(signal.queue)) {
650
- return;
651
- }
652
- emit(signal);
653
- });
654
- const timer = setInterval(() => {
655
- this.warmPush?.();
656
- // Guarded for the same reason PostgresStore guards its push fan-out: a
657
- // consumer that throws must not take the timer down with it. Losing the
658
- // timer would silently retire the fallback that makes watch correct
659
- // where there is no push channel at all.
660
- try {
661
- emit({ reason: "poll" });
662
- } catch {
663
- // The consumer's problem, not the watch's.
664
- }
665
- }, pollMs);
666
- timer.unref?.();
667
- return () => {
668
- live = false;
669
- unsubscribe?.();
670
- clearInterval(timer);
671
- };
672
- }
673
-
674
640
  /**
675
641
  * How many more tasks fit on `queue` under `maxDepth` — 0 once it is full.
676
642
  *
677
- * The cheap half of backpressure: bounded at `maxDepth` index entries, unlike
678
- * `stats()`, which aggregates the whole table (terminal rows included) and so
679
- * costs more the longer a database has been running. Use it directly to shed
680
- * load or shape a producer; `QueueDepthGate` builds the blocking form on top.
643
+ * The cheap half of backpressure: bounded at `maxDepth` index entries, so it
644
+ * stays cheap to ask on every enqueue. Use it directly to shed load or shape
645
+ * a producer; `QueueDepthGate` builds the blocking form on top.
681
646
  */
682
647
  async queueDepth(queue: string, maxDepth: number): Promise<number> {
683
648
  if (!Number.isInteger(maxDepth) || maxDepth < 0) {
@@ -782,7 +747,7 @@ export abstract class TaskStore {
782
747
  name: oneName ? names![0] : null,
783
748
  limit,
784
749
  });
785
- return rows.map(rowToTask);
750
+ return rows.map((r) => this.toTask(r));
786
751
  });
787
752
  });
788
753
  }
@@ -888,7 +853,7 @@ export abstract class TaskStore {
888
853
  });
889
854
  // Rolls back `fn`'s writes along with the settlement that did not land.
890
855
  if (!rows.length) throw new LostLease(input.taskId);
891
- return { task: rowToTask(rows[0]), value };
856
+ return { task: this.toTask(rows[0]), value };
892
857
  });
893
858
  }
894
859
 
@@ -44,6 +44,17 @@
44
44
  * The reference implementation happens not to be affected — `pg` uses the
45
45
  * extended protocol by default — which is why this is stated here rather than
46
46
  * left to be discovered.
47
+ *
48
+ * A second thing, smaller but just as easy to get wrong: whatever this executor
49
+ * does with json/jsonb, it must do CONSISTENTLY. The store measures the wire
50
+ * form once at connect, with a `select '"cairnq"'::jsonb` probe, and maps every
51
+ * row it later reads according to the answer — so an executor that decodes
52
+ * jsonb for one statement and hands back text for another (or answers the probe
53
+ * differently from how it returns task rows) will have its payloads read the
54
+ * wrong way. Decoding or not are both fine; disagreeing with yourself is not.
55
+ * The probe exists because the two forms cannot be told apart from a value: a
56
+ * JSON string arrives as a string either way, so a guess parses the decoded one
57
+ * twice — `"s3://…"` throws and `"42"` silently becomes the number 42.
47
58
  */
48
59
 
49
60
  /** A row as the driver hands it back: column name -> value. */
@@ -160,6 +160,20 @@ export async function createPoolExecutor(
160
160
  // its connection for its whole life, and a pooled one would be a slot the
161
161
  // store never gives back. If `opts` ever grows connection-level settings
162
162
  // (ssl, application_name), the listener must receive them too.
163
+ //
164
+ // `schema` is deliberately NOT forwarded, and it is the exception worth
165
+ // stating rather than the omission it looks like: a NOTIFY channel name is
166
+ // scoped to the database, never to a schema, so a search_path would change
167
+ // nothing about what arrives here. The consequence is real but bounded —
168
+ // two cairnq installations in different schemas of ONE database hear each
169
+ // other's notifications, and a queue name they happen to share wakes the
170
+ // wrong workers. Correctness is unaffected (a wake is only ever a hint;
171
+ // the claim that follows re-reads the rows and finds none of them), so the
172
+ // cost is one wasted poll per foreign submit. Fixing it properly means
173
+ // putting the schema in the channel name, which is a migration and a
174
+ // mixed-version-fleet hazard: a worker still listening on the old channel
175
+ // would hear nothing and fall back to polling for the length of the
176
+ // rollout.
163
177
  const client = new pg.Client({ connectionString: dsn });
164
178
  await client.connect(); // failure here is transient: caller retries with backoff
165
179
  client.on("notification", (msg) => onNotify(msg.channel, msg.payload));
@@ -9,7 +9,6 @@ import {
9
9
  specialize,
10
10
  statementParams,
11
11
  TaskStore,
12
- type WatchSignal,
13
12
  } from "./base.js";
14
13
  import { ListenUnavailable, type PgExecutor, type PgSession } from "./pg-executor.js";
15
14
  import { createPoolExecutor } from "./pg-pool.js";
@@ -144,9 +143,6 @@ export class PostgresStore extends TaskStore {
144
143
  private readonly wakeableQueues = new Set<string>();
145
144
  /** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
146
145
  private readonly waiters = new Map<string, Set<() => void>>();
147
- /** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
148
- * consumes the notification, a subscriber is standing and only observes. */
149
- private readonly subscribers = new Set<(signal: WatchSignal) => void>();
150
146
 
151
147
  /**
152
148
  * `source` is either a libpq connection string — this store then owns a `pg`
@@ -199,8 +195,8 @@ export class PostgresStore extends TaskStore {
199
195
  * Every path here connects lazily, so a store used again after close simply
200
196
  * reconnects — and without this it would reconnect permanently degraded to
201
197
  * polling, having decided "no push channel" about a connection that no longer
202
- * exists. Called from the constructor for the initial state and from doConnect
203
- * for every later one, so the three fields cannot be re-armed by halves. Only
198
+ * exists. The fields' initializers give the first state and doConnect calls
199
+ * this for every later one, so the three cannot be re-armed by halves. Only
204
200
  * ever safe under doConnect's generation check.
205
201
  */
206
202
  private resetListenerState(): void {
@@ -220,6 +216,9 @@ export class PostgresStore extends TaskStore {
220
216
  await this.checkSchema(executor);
221
217
  await this.applyMigrations(executor);
222
218
  checkProtocolVersion(await this.readProtocolVersion(executor));
219
+ // Before publishing the executor, so no row is ever mapped against an
220
+ // unmeasured wire form.
221
+ this.jsonIsText = await this.detectJsonWireForm(executor);
223
222
  } catch (e) {
224
223
  // Never leak an executor we created; never close one we were handed.
225
224
  if (!this.provided) await executor.close().catch(() => {});
@@ -321,6 +320,25 @@ export class PostgresStore extends TaskStore {
321
320
  }
322
321
  }
323
322
 
323
+ /**
324
+ * Whether this driver hands json/jsonb back as text, measured rather than
325
+ * assumed.
326
+ *
327
+ * `pg` decodes jsonb by default; asyncpg hands back the text; an application's
328
+ * own driver — or a `pg` with a type parser the application installed — may do
329
+ * either, and an injected executor is exactly that case. The two are
330
+ * indistinguishable from a value once it arrives (see rowToTask), so the
331
+ * answer has to come from the driver itself.
332
+ *
333
+ * A JSON string is the probe because it is the only value whose two forms
334
+ * differ: as text it arrives with its quotes (`"cairnq"`, 8 characters), as a
335
+ * decoded value without them (`cairnq`, 6). One statement, once per connect.
336
+ */
337
+ private async detectJsonWireForm(s: PgSession): Promise<boolean> {
338
+ const rows = await s.query(`select '"cairnq"'::jsonb as probe`, []);
339
+ return rows[0]?.probe === '"cairnq"';
340
+ }
341
+
324
342
  // Takes an explicit session: during doConnect the executor is not published
325
343
  // yet, so this cannot go through fetch(). The statement binds nothing.
326
344
  private async readProtocolVersion(s: PgSession): Promise<number> {
@@ -355,16 +373,6 @@ export class PostgresStore extends TaskStore {
355
373
  }
356
374
  }
357
375
 
358
- protected subscribePush(onSignal: (signal: WatchSignal) => void): () => void {
359
- this.subscribers.add(onSignal);
360
- this.listenerReady(); // an API-side watcher is often the only thing asking
361
- return () => void this.subscribers.delete(onSignal);
362
- }
363
-
364
- protected warmPush(): void {
365
- if (this.subscribers.size > 0) this.listenerReady();
366
- }
367
-
368
376
  taskDoneWake(taskId: string, timeoutMs: number): Promise<void> {
369
377
  if (!this.listenerReady()) return super.taskDoneWake(taskId, timeoutMs);
370
378
  return this.wakeOn(`done:${taskId}`, timeoutMs);
@@ -453,10 +461,8 @@ export class PostgresStore extends TaskStore {
453
461
  // Buffered only for a queue someone waits on — see wakeableQueues.
454
462
  if (this.wakeableQueues.has(payload)) this.pendingQueues.add(payload);
455
463
  key = "queued";
456
- this.publish({ reason: "queued", queue: payload });
457
464
  } else if (channel === DONE_CHANNEL && payload) {
458
465
  key = `done:${payload}`;
459
- this.publish({ reason: "done", taskId: payload });
460
466
  } else {
461
467
  return;
462
468
  }
@@ -465,19 +471,6 @@ export class PostgresStore extends TaskStore {
465
471
  if (set) for (const w of set) w();
466
472
  }
467
473
 
468
- /** Hand a notification to every watch() subscriber. A throwing subscriber is
469
- * its own problem: it must not cost the others their signal, nor take down the
470
- * listener connection that delivered it. */
471
- private publish(signal: WatchSignal): void {
472
- for (const s of [...this.subscribers]) {
473
- try {
474
- s(signal);
475
- } catch {
476
- // Deliberately swallowed — see above.
477
- }
478
- }
479
- }
480
-
481
474
  private dropListener(): void {
482
475
  const stop = this.listener;
483
476
  this.listener = null;