cairnq 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -4
- package/dist/_protocol/migrations/postgres/0007_purge_status_index.sql +11 -0
- package/dist/_protocol/migrations/sqlite/0007_purge_status_index.sql +11 -0
- package/dist/_protocol/sql/postgres/get_status.sql +7 -0
- package/dist/_protocol/sql/postgres/get_status_by_key.sql +7 -0
- package/dist/_protocol/sql/postgres/purge.sql +8 -1
- package/dist/_protocol/sql/sqlite/get_status.sql +6 -0
- package/dist/_protocol/sql/sqlite/get_status_by_key.sql +7 -0
- package/dist/_protocol/sql/sqlite/purge.sql +7 -1
- package/dist/client.d.ts +32 -15
- package/dist/client.js +38 -12
- package/dist/context.d.ts +25 -0
- package/dist/context.js +33 -0
- package/dist/errors.d.ts +4 -2
- package/dist/errors.js +7 -4
- package/dist/index.d.ts +9 -5
- package/dist/index.js +4 -1
- package/dist/models.d.ts +14 -2
- package/dist/models.js +12 -1
- package/dist/retention.d.ts +15 -3
- package/dist/retention.js +36 -14
- package/dist/store/base.d.ts +118 -1
- package/dist/store/base.js +122 -20
- package/dist/store/pg-executor.d.ts +77 -0
- package/dist/store/pg-executor.js +26 -0
- package/dist/store/pg-pool.d.ts +16 -0
- package/dist/store/pg-pool.js +147 -0
- package/dist/store/postgres.d.ts +41 -13
- package/dist/store/postgres.js +135 -147
- package/dist/store/sqlite.js +20 -2
- package/dist/wait.d.ts +9 -4
- package/dist/wait.js +27 -13
- package/dist/worker.d.ts +6 -3
- package/dist/worker.js +6 -5
- package/package.json +6 -4
- package/src/client.ts +60 -21
- package/src/context.ts +37 -0
- package/src/errors.ts +7 -4
- package/src/index.ts +16 -4
- package/src/models.ts +24 -3
- package/src/retention.ts +45 -15
- package/src/store/base.ts +204 -9
- package/src/store/pg-executor.ts +90 -0
- package/src/store/pg-pool.ts +156 -0
- package/src/store/postgres.ts +144 -141
- package/src/store/sqlite.ts +23 -2
- package/src/wait.ts +35 -14
- package/src/worker.ts +8 -6
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { ListenUnavailable } from "./pg-executor.js";
|
|
2
|
+
// `pg` is an optional dependency: the SDK is SQLite-first, so it's loaded lazily
|
|
3
|
+
// the first time a pool-backed executor is built. Absent -> a clear install hint.
|
|
4
|
+
// An application that brings its own executor never reaches this.
|
|
5
|
+
let pgModule = null;
|
|
6
|
+
async function loadPg() {
|
|
7
|
+
if (pgModule)
|
|
8
|
+
return pgModule;
|
|
9
|
+
let mod;
|
|
10
|
+
try {
|
|
11
|
+
mod = (await import("pg"));
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
throw new Error("PostgresStore requires the 'pg' package — install it (e.g. `npm i pg`)");
|
|
15
|
+
}
|
|
16
|
+
const pg = (mod.default ?? mod);
|
|
17
|
+
// Postgres returns bigint (int8, OID 20) as a string to avoid precision loss.
|
|
18
|
+
// Every cairnq bigint is an epoch-ms or a counter, all within Number's safe
|
|
19
|
+
// integer range, so parse to number once (globally) to match the Task model
|
|
20
|
+
// (*_ms typed as number, same as the SQLite SDK). Set before any query runs.
|
|
21
|
+
pg.types.setTypeParser(pg.types.builtins.INT8, (v) => (v == null ? null : Number(v)));
|
|
22
|
+
pgModule = pg;
|
|
23
|
+
return pg;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Roll back on the way out of a failed transaction, without letting the rollback
|
|
27
|
+
* become the error the caller sees. A dropped connection fails both the statement
|
|
28
|
+
* and the rollback, and it is the first one that says what went wrong.
|
|
29
|
+
*/
|
|
30
|
+
async function rollbackQuietly(client) {
|
|
31
|
+
try {
|
|
32
|
+
await client.query("rollback");
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
// Already rolled back, or the connection is gone. Either way the original
|
|
36
|
+
// error is the one worth propagating.
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** The PgSession face of one pg client or pool. */
|
|
40
|
+
function session(q) {
|
|
41
|
+
return {
|
|
42
|
+
async query(text, values) {
|
|
43
|
+
return (await q.query(text, values)).rows;
|
|
44
|
+
},
|
|
45
|
+
async exec(sql) {
|
|
46
|
+
// No values: pg sends this over the simple query protocol, which is what
|
|
47
|
+
// makes a multi-statement migration script legal here and not in query().
|
|
48
|
+
await q.query(sql);
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* A Postgres identifier that is safe to interpolate — cairnq quotes the schema
|
|
54
|
+
* name, and a name that could close that quote could rewrite the statement.
|
|
55
|
+
* Deliberately narrower than what Postgres accepts: a schema cairnq is asked to
|
|
56
|
+
* live in is a deployment decision, not a place to be clever.
|
|
57
|
+
*/
|
|
58
|
+
const PLAIN_IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$/;
|
|
59
|
+
/**
|
|
60
|
+
* The built-in executor: a `pg.Pool` over a libpq DSN. What `CairnQ.postgres(dsn)`
|
|
61
|
+
* uses, and the reference for what an adapter over another driver must do.
|
|
62
|
+
*
|
|
63
|
+
* Creating it does not connect — `pg.Pool` is lazy, and the store's first
|
|
64
|
+
* statement (the migration ledger) is what proves the database is reachable.
|
|
65
|
+
*
|
|
66
|
+
* `schema` puts cairnq's tables in a schema of their own rather than in whatever
|
|
67
|
+
* the connection's search_path leads with. The protocol's SQL names no schema, so
|
|
68
|
+
* this is a per-connection `search_path` and not one statement changes.
|
|
69
|
+
*/
|
|
70
|
+
export async function createPoolExecutor(dsn, opts = {}) {
|
|
71
|
+
const pg = await loadPg();
|
|
72
|
+
const schema = opts.schema;
|
|
73
|
+
if (schema !== undefined && !PLAIN_IDENT.test(schema)) {
|
|
74
|
+
throw new Error(`schema must be a plain identifier (letters, digits, _ and $, not starting ` +
|
|
75
|
+
`with a digit), got ${JSON.stringify(schema)}`);
|
|
76
|
+
}
|
|
77
|
+
const pool = new pg.Pool({ connectionString: dsn, max: opts.max });
|
|
78
|
+
if (schema) {
|
|
79
|
+
// Queued on the connection before it is handed out, so every statement this
|
|
80
|
+
// pool ever runs — migrations included — resolves in the right schema. Pooled
|
|
81
|
+
// connections come and go, which is why this is per-connection rather than a
|
|
82
|
+
// one-off at startup.
|
|
83
|
+
pool.on("connect", (client) => {
|
|
84
|
+
void client.query(`set search_path to "${schema}"`);
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
if (schema) {
|
|
88
|
+
// Created once, here, rather than from the connect handler (which would ask
|
|
89
|
+
// for CREATE privilege on every new connection) or from the migrations
|
|
90
|
+
// (which name no schema, by design). Without it the first `create table`
|
|
91
|
+
// fails with "no schema has been selected to create in", which says nothing
|
|
92
|
+
// about the cause. This is the one thing that makes building the executor
|
|
93
|
+
// connect; without `schema` it stays lazy.
|
|
94
|
+
try {
|
|
95
|
+
await pool.query(`create schema if not exists "${schema}"`);
|
|
96
|
+
}
|
|
97
|
+
catch (e) {
|
|
98
|
+
await pool.end();
|
|
99
|
+
throw e;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const poolSession = session(pool);
|
|
103
|
+
return {
|
|
104
|
+
query: poolSession.query,
|
|
105
|
+
exec: poolSession.exec,
|
|
106
|
+
async tx(fn) {
|
|
107
|
+
const client = await pool.connect();
|
|
108
|
+
try {
|
|
109
|
+
await client.query("begin");
|
|
110
|
+
const out = await fn(session(client));
|
|
111
|
+
await client.query("commit");
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
catch (e) {
|
|
115
|
+
await rollbackQuietly(client);
|
|
116
|
+
throw e;
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
client.release();
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
async listen(channels, onNotify, onClose) {
|
|
123
|
+
// Built from the raw DSN rather than taken from the pool: a listener holds
|
|
124
|
+
// its connection for its whole life, and a pooled one would be a slot the
|
|
125
|
+
// store never gives back. If `opts` ever grows connection-level settings
|
|
126
|
+
// (ssl, application_name), the listener must receive them too.
|
|
127
|
+
const client = new pg.Client({ connectionString: dsn });
|
|
128
|
+
await client.connect(); // failure here is transient: caller retries with backoff
|
|
129
|
+
client.on("notification", (msg) => onNotify(msg.channel, msg.payload));
|
|
130
|
+
// A dropped listener degrades to polling; the store reconnects on the next wake.
|
|
131
|
+
client.on("error", () => onClose());
|
|
132
|
+
try {
|
|
133
|
+
await client.query(channels.map((c) => `listen ${c}`).join("; "));
|
|
134
|
+
}
|
|
135
|
+
catch (e) {
|
|
136
|
+
void client.end().catch(() => { });
|
|
137
|
+
// Connected, but LISTEN was refused (e.g. a transaction-mode pooler) —
|
|
138
|
+
// deterministic, so tell the store not to retry.
|
|
139
|
+
throw new ListenUnavailable(e instanceof Error ? e.message : undefined);
|
|
140
|
+
}
|
|
141
|
+
return () => void client.end().catch(() => { });
|
|
142
|
+
},
|
|
143
|
+
async close() {
|
|
144
|
+
await pool.end();
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
}
|
package/dist/store/postgres.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { type Fetch, type Params, TaskStore } from "./base.js";
|
|
1
|
+
import { type Fetch, type Params, TaskStore, type WatchSignal } from "./base.js";
|
|
2
|
+
import { type PgExecutor, type PgSession } from "./pg-executor.js";
|
|
2
3
|
/**
|
|
3
4
|
* The rewritten SQL and the order its `$n` slots must be filled in.
|
|
4
5
|
*
|
|
@@ -28,23 +29,32 @@ export declare function toPositional(sql: string, params: Params): {
|
|
|
28
29
|
* PostgresStore — the Postgres dialect of the shared cairnq-protocol SQL.
|
|
29
30
|
*
|
|
30
31
|
* Everything protocol-shaped lives in TaskStore; this file is only what Postgres
|
|
31
|
-
* does differently:
|
|
32
|
-
* the DB clock (`now()`) instead of from the SDK,
|
|
33
|
-
* multi-host — unlike SQLite it coordinates API
|
|
34
|
-
* machines, with no shared clock to agree on. claim
|
|
35
|
-
* and needs no claimable_probe, because PG readers
|
|
36
|
-
* columns are jsonb (bound as JSON text, read back as
|
|
32
|
+
* does differently: `:name` -> `$n` translation, the migration ledger, LISTEN
|
|
33
|
+
* policy, and time taken from the DB clock (`now()`) instead of from the SDK,
|
|
34
|
+
* which is what makes this backend multi-host — unlike SQLite it coordinates API
|
|
35
|
+
* and worker processes across machines, with no shared clock to agree on. claim
|
|
36
|
+
* uses FOR UPDATE SKIP LOCKED and needs no claimable_probe, because PG readers
|
|
37
|
+
* don't block writers. JSON columns are jsonb (bound as JSON text, read back as
|
|
38
|
+
* objects by rowToTask).
|
|
39
|
+
*
|
|
40
|
+
* What it deliberately does NOT own is the connection. Given a DSN it builds a
|
|
41
|
+
* `pg` pool (see pg-pool.ts); given a PgExecutor it runs inside the caller's
|
|
42
|
+
* session instead — no second driver, no second pool, and the caller's writes and
|
|
43
|
+
* cairnq's can share one transaction.
|
|
37
44
|
*/
|
|
38
45
|
export declare class PostgresStore extends TaskStore {
|
|
39
|
-
private readonly dsn;
|
|
40
46
|
private readonly opts;
|
|
41
|
-
private
|
|
47
|
+
private readonly dsn;
|
|
48
|
+
/** The caller's executor, if one was injected — never closed by this store. */
|
|
49
|
+
private readonly provided;
|
|
50
|
+
/** Set once migrations have run and the protocol version checked out. */
|
|
51
|
+
private executor;
|
|
42
52
|
private connecting;
|
|
43
53
|
private readonly statements;
|
|
44
54
|
private listener;
|
|
45
55
|
private listenerConnecting;
|
|
46
|
-
/** LISTEN is off for good: the store was closed,
|
|
47
|
-
*
|
|
56
|
+
/** LISTEN is off for good: the store was closed, the executor does not support
|
|
57
|
+
* it, or the server refused it (e.g. a transaction-mode pooler) —
|
|
48
58
|
* deterministic, so retrying would fail the same way every time. */
|
|
49
59
|
private listenerUnavailable;
|
|
50
60
|
/** A failure to even connect is transient (network blip, server restarting):
|
|
@@ -57,8 +67,17 @@ export declare class PostgresStore extends TaskStore {
|
|
|
57
67
|
private readonly pendingQueues;
|
|
58
68
|
/** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
|
|
59
69
|
private readonly waiters;
|
|
60
|
-
|
|
70
|
+
/** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
|
|
71
|
+
* consumes the notification, a subscriber is standing and only observes. */
|
|
72
|
+
private readonly subscribers;
|
|
73
|
+
/**
|
|
74
|
+
* `source` is either a libpq connection string — this store then owns a `pg`
|
|
75
|
+
* pool and requires the optional `pg` package — or a PgExecutor the caller
|
|
76
|
+
* already has, which this store uses and never closes.
|
|
77
|
+
*/
|
|
78
|
+
constructor(source: string | PgExecutor, opts?: {
|
|
61
79
|
max?: number;
|
|
80
|
+
schema?: string;
|
|
62
81
|
});
|
|
63
82
|
connect(): Promise<void>;
|
|
64
83
|
close(): Promise<void>;
|
|
@@ -68,19 +87,28 @@ export declare class PostgresStore extends TaskStore {
|
|
|
68
87
|
private readProtocolVersion;
|
|
69
88
|
protocolVersion(): Promise<number>;
|
|
70
89
|
claimWake(queues: string[], timeoutMs: number): Promise<void>;
|
|
90
|
+
protected subscribePush(onSignal: (signal: WatchSignal) => void): () => void;
|
|
91
|
+
protected warmPush(): void;
|
|
71
92
|
taskDoneWake(taskId: string, timeoutMs: number): Promise<void>;
|
|
72
93
|
/** A promise resolving on notification-or-timeout, deregistering either way.
|
|
73
94
|
* The timer is unref'd: while a listener exists its socket keeps the process
|
|
74
95
|
* alive, and dropListener wakes every waiter the moment it goes away. */
|
|
75
96
|
private wakeOn;
|
|
76
|
-
/** True once the LISTEN
|
|
97
|
+
/** True once the LISTEN subscription is up; starts establishing it otherwise
|
|
77
98
|
* (respecting the transient-failure backoff). Callers fall back to plain
|
|
78
99
|
* polling until it is ready (or forever, if it can't be established) —
|
|
79
100
|
* correctness never depends on it. */
|
|
80
101
|
private listenerReady;
|
|
81
102
|
private startListener;
|
|
82
103
|
private onNotification;
|
|
104
|
+
/** Hand a notification to every watch() subscriber. A throwing subscriber is
|
|
105
|
+
* its own problem: it must not cost the others their signal, nor take down the
|
|
106
|
+
* listener connection that delivered it. */
|
|
107
|
+
private publish;
|
|
83
108
|
private dropListener;
|
|
84
109
|
protected fetch(name: string, params: Params): Promise<any[]>;
|
|
85
110
|
protected tx<T>(fn: (fetch: Fetch) => Promise<T>): Promise<T>;
|
|
111
|
+
protected txWithSession<T>(fn: (fetch: Fetch, session: PgSession) => Promise<T>): Promise<T>;
|
|
112
|
+
/** A Fetch that runs the protocol's statements on one particular session. */
|
|
113
|
+
private boundFetch;
|
|
86
114
|
}
|
package/dist/store/postgres.js
CHANGED
|
@@ -1,45 +1,11 @@
|
|
|
1
1
|
import { loadMigrations, loadStatements } from "../sql.js";
|
|
2
2
|
import { checkProtocolVersion, COMMENT, NAMED, statementParams, TaskStore, } from "./base.js";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
let pgModule = null;
|
|
6
|
-
async function loadPg() {
|
|
7
|
-
if (pgModule)
|
|
8
|
-
return pgModule;
|
|
9
|
-
let mod;
|
|
10
|
-
try {
|
|
11
|
-
mod = (await import("pg"));
|
|
12
|
-
}
|
|
13
|
-
catch {
|
|
14
|
-
throw new Error("PostgresStore requires the 'pg' package — install it (e.g. `npm i pg`)");
|
|
15
|
-
}
|
|
16
|
-
const pg = (mod.default ?? mod);
|
|
17
|
-
// Postgres returns bigint (int8, OID 20) as a string to avoid precision loss.
|
|
18
|
-
// Every cairnq bigint is an epoch-ms or a counter, all within Number's safe
|
|
19
|
-
// integer range, so parse to number once (globally) to match the Task model
|
|
20
|
-
// (*_ms typed as number, same as the SQLite SDK). Set before any query runs.
|
|
21
|
-
pg.types.setTypeParser(pg.types.builtins.INT8, (v) => (v == null ? null : Number(v)));
|
|
22
|
-
pgModule = pg;
|
|
23
|
-
return pg;
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Roll back on the way out of a failed transaction, without letting the rollback
|
|
27
|
-
* become the error the caller sees. A dropped connection fails both the statement
|
|
28
|
-
* and the rollback, and it is the first one that says what went wrong.
|
|
29
|
-
*/
|
|
30
|
-
async function rollbackQuietly(client) {
|
|
31
|
-
try {
|
|
32
|
-
await client.query("rollback");
|
|
33
|
-
}
|
|
34
|
-
catch {
|
|
35
|
-
// Already rolled back, or the connection is gone. Either way the original
|
|
36
|
-
// error is the one worth propagating.
|
|
37
|
-
}
|
|
38
|
-
}
|
|
3
|
+
import { ListenUnavailable } from "./pg-executor.js";
|
|
4
|
+
import { createPoolExecutor } from "./pg-pool.js";
|
|
39
5
|
// Notification channels, emitted by the 0003_notify trigger.
|
|
40
6
|
const QUEUED_CHANNEL = "cairnq_queued";
|
|
41
7
|
const DONE_CHANNEL = "cairnq_done";
|
|
42
|
-
// Backoff between attempts to (re)
|
|
8
|
+
// Backoff between attempts to (re)establish the LISTEN subscription after a
|
|
43
9
|
// transient failure. Doubles per failure up to the cap; polling covers the gap.
|
|
44
10
|
const LISTENER_RETRY_MS = 1_000;
|
|
45
11
|
const LISTENER_RETRY_MAX_MS = 30_000;
|
|
@@ -82,27 +48,38 @@ export function toPositional(sql, params) {
|
|
|
82
48
|
* PostgresStore — the Postgres dialect of the shared cairnq-protocol SQL.
|
|
83
49
|
*
|
|
84
50
|
* Everything protocol-shaped lives in TaskStore; this file is only what Postgres
|
|
85
|
-
* does differently:
|
|
86
|
-
* the DB clock (`now()`) instead of from the SDK,
|
|
87
|
-
* multi-host — unlike SQLite it coordinates API
|
|
88
|
-
* machines, with no shared clock to agree on. claim
|
|
89
|
-
* and needs no claimable_probe, because PG readers
|
|
90
|
-
* columns are jsonb (bound as JSON text, read back as
|
|
51
|
+
* does differently: `:name` -> `$n` translation, the migration ledger, LISTEN
|
|
52
|
+
* policy, and time taken from the DB clock (`now()`) instead of from the SDK,
|
|
53
|
+
* which is what makes this backend multi-host — unlike SQLite it coordinates API
|
|
54
|
+
* and worker processes across machines, with no shared clock to agree on. claim
|
|
55
|
+
* uses FOR UPDATE SKIP LOCKED and needs no claimable_probe, because PG readers
|
|
56
|
+
* don't block writers. JSON columns are jsonb (bound as JSON text, read back as
|
|
57
|
+
* objects by rowToTask).
|
|
58
|
+
*
|
|
59
|
+
* What it deliberately does NOT own is the connection. Given a DSN it builds a
|
|
60
|
+
* `pg` pool (see pg-pool.ts); given a PgExecutor it runs inside the caller's
|
|
61
|
+
* session instead — no second driver, no second pool, and the caller's writes and
|
|
62
|
+
* cairnq's can share one transaction.
|
|
91
63
|
*/
|
|
92
64
|
export class PostgresStore extends TaskStore {
|
|
93
|
-
dsn;
|
|
94
65
|
opts;
|
|
95
|
-
|
|
66
|
+
dsn;
|
|
67
|
+
/** The caller's executor, if one was injected — never closed by this store. */
|
|
68
|
+
provided;
|
|
69
|
+
/** Set once migrations have run and the protocol version checked out. */
|
|
70
|
+
executor = null;
|
|
96
71
|
connecting = null;
|
|
97
72
|
statements;
|
|
98
73
|
// ------------------------------------------------------- LISTEN/NOTIFY state
|
|
99
|
-
//
|
|
100
|
-
// the claimWake/taskDoneWake contract on TaskStore).
|
|
101
|
-
// keep it silently degrades the store to the base
|
|
74
|
+
// The executor subscribes one dedicated connection to both channels (see
|
|
75
|
+
// 0003_notify.sql and the claimWake/taskDoneWake contract on TaskStore).
|
|
76
|
+
// Failure to establish or keep it silently degrades the store to the base
|
|
77
|
+
// class's plain polling. Held as the unsubscribe function, not the connection:
|
|
78
|
+
// whose connection it is, is the executor's business.
|
|
102
79
|
listener = null;
|
|
103
80
|
listenerConnecting = null;
|
|
104
|
-
/** LISTEN is off for good: the store was closed,
|
|
105
|
-
*
|
|
81
|
+
/** LISTEN is off for good: the store was closed, the executor does not support
|
|
82
|
+
* it, or the server refused it (e.g. a transaction-mode pooler) —
|
|
106
83
|
* deterministic, so retrying would fail the same way every time. */
|
|
107
84
|
listenerUnavailable = false;
|
|
108
85
|
/** A failure to even connect is transient (network blip, server restarting):
|
|
@@ -115,10 +92,19 @@ export class PostgresStore extends TaskStore {
|
|
|
115
92
|
pendingQueues = new Set();
|
|
116
93
|
/** Wake callbacks by key: "queued" (broadcast) or "done:<task id>". */
|
|
117
94
|
waiters = new Map();
|
|
118
|
-
|
|
95
|
+
/** watch() subscribers. Separate from `waiters`: a waiter is one-shot and
|
|
96
|
+
* consumes the notification, a subscriber is standing and only observes. */
|
|
97
|
+
subscribers = new Set();
|
|
98
|
+
/**
|
|
99
|
+
* `source` is either a libpq connection string — this store then owns a `pg`
|
|
100
|
+
* pool and requires the optional `pg` package — or a PgExecutor the caller
|
|
101
|
+
* already has, which this store uses and never closes.
|
|
102
|
+
*/
|
|
103
|
+
constructor(source, opts = {}) {
|
|
119
104
|
super();
|
|
120
|
-
this.dsn = dsn;
|
|
121
105
|
this.opts = opts;
|
|
106
|
+
this.dsn = typeof source === "string" ? source : null;
|
|
107
|
+
this.provided = typeof source === "string" ? null : source;
|
|
122
108
|
this.statements = loadStatements("postgres");
|
|
123
109
|
}
|
|
124
110
|
async connect() {
|
|
@@ -127,18 +113,20 @@ export class PostgresStore extends TaskStore {
|
|
|
127
113
|
async close() {
|
|
128
114
|
this.listenerUnavailable = true; // no revival after close
|
|
129
115
|
this.dropListener();
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
116
|
+
const executor = this.executor;
|
|
117
|
+
this.executor = null;
|
|
118
|
+
this.connecting = null;
|
|
119
|
+
// An injected executor belongs to the caller, whose other work would not
|
|
120
|
+
// survive cairnq closing it.
|
|
121
|
+
if (executor && !this.provided)
|
|
122
|
+
await executor.close();
|
|
136
123
|
}
|
|
137
124
|
async ensure() {
|
|
138
|
-
if (this.
|
|
125
|
+
if (this.executor)
|
|
139
126
|
return;
|
|
140
|
-
// Cache the in-flight connect so concurrent calls share one
|
|
141
|
-
// clear it so a later call retries instead of re-awaiting a
|
|
127
|
+
// Cache the in-flight connect so concurrent calls share one executor. On
|
|
128
|
+
// failure, clear it so a later call retries instead of re-awaiting a
|
|
129
|
+
// rejected promise.
|
|
142
130
|
if (!this.connecting) {
|
|
143
131
|
this.connecting = this.doConnect().catch((e) => {
|
|
144
132
|
this.connecting = null;
|
|
@@ -148,68 +136,50 @@ export class PostgresStore extends TaskStore {
|
|
|
148
136
|
await this.connecting;
|
|
149
137
|
}
|
|
150
138
|
async doConnect() {
|
|
151
|
-
const
|
|
152
|
-
|
|
139
|
+
const executor = this.provided ??
|
|
140
|
+
(await createPoolExecutor(this.dsn, { max: this.opts.max, schema: this.opts.schema }));
|
|
153
141
|
try {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
await this.applyMigrations(client);
|
|
157
|
-
checkProtocolVersion(await this.readProtocolVersion(client));
|
|
158
|
-
}
|
|
159
|
-
finally {
|
|
160
|
-
client.release();
|
|
161
|
-
}
|
|
142
|
+
await this.applyMigrations(executor);
|
|
143
|
+
checkProtocolVersion(await this.readProtocolVersion(executor));
|
|
162
144
|
}
|
|
163
145
|
catch (e) {
|
|
164
|
-
|
|
146
|
+
// Never leak an executor we created; never close one we were handed.
|
|
147
|
+
if (!this.provided)
|
|
148
|
+
await executor.close().catch(() => { });
|
|
165
149
|
throw e;
|
|
166
150
|
}
|
|
167
|
-
this.
|
|
168
|
-
// Warm the LISTEN
|
|
151
|
+
this.executor = executor; // publish only a fully-migrated, version-checked executor
|
|
152
|
+
// Warm the LISTEN subscription in the background so the first idle sleep is
|
|
169
153
|
// already wakeable. Fire-and-forget: failure just means polling.
|
|
170
154
|
this.listenerReady();
|
|
171
155
|
}
|
|
172
|
-
async applyMigrations(
|
|
173
|
-
await
|
|
156
|
+
async applyMigrations(executor) {
|
|
157
|
+
await executor.exec("create table if not exists cairnq_migrations " +
|
|
174
158
|
"(name text primary key, applied_at_ms bigint not null)");
|
|
175
159
|
for (const { name, sql } of loadMigrations("postgres")) {
|
|
176
160
|
// Check and apply inside one transaction, with the table lock taken up
|
|
177
161
|
// front: two processes cold-starting together would otherwise both see a
|
|
178
162
|
// migration as unapplied and both run it.
|
|
179
|
-
|
|
180
|
-
await
|
|
181
|
-
await
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
if (applied.rowCount === 0) {
|
|
186
|
-
await client.query(sql); // multi-statement DDL (simple-query, no params)
|
|
187
|
-
await client.query("insert into cairnq_migrations (name, applied_at_ms) values " +
|
|
163
|
+
await executor.tx(async (s) => {
|
|
164
|
+
await s.exec("lock table cairnq_migrations in exclusive mode");
|
|
165
|
+
const applied = await s.query("select 1 from cairnq_migrations where name = $1", [name]);
|
|
166
|
+
if (applied.length === 0) {
|
|
167
|
+
await s.exec(sql); // multi-statement DDL (simple-query, no params)
|
|
168
|
+
await s.query("insert into cairnq_migrations (name, applied_at_ms) values " +
|
|
188
169
|
"($1, (extract(epoch from now()) * 1000)::bigint)", [name]);
|
|
189
170
|
}
|
|
190
|
-
|
|
191
|
-
}
|
|
192
|
-
catch (e) {
|
|
193
|
-
await rollbackQuietly(client);
|
|
194
|
-
throw e;
|
|
195
|
-
}
|
|
171
|
+
});
|
|
196
172
|
}
|
|
197
173
|
}
|
|
198
|
-
// Takes an explicit
|
|
199
|
-
// this cannot go through fetch(). The statement binds nothing
|
|
200
|
-
async readProtocolVersion(
|
|
201
|
-
const
|
|
202
|
-
return
|
|
174
|
+
// Takes an explicit session: during doConnect the executor is not published
|
|
175
|
+
// yet, so this cannot go through fetch(). The statement binds nothing.
|
|
176
|
+
async readProtocolVersion(s) {
|
|
177
|
+
const rows = await s.query(this.statements.protocol_version, []);
|
|
178
|
+
return rows.length ? Number(rows[0].value) : 0;
|
|
203
179
|
}
|
|
204
180
|
async protocolVersion() {
|
|
205
181
|
await this.ensure();
|
|
206
|
-
|
|
207
|
-
try {
|
|
208
|
-
return await this.readProtocolVersion(client);
|
|
209
|
-
}
|
|
210
|
-
finally {
|
|
211
|
-
client.release();
|
|
212
|
-
}
|
|
182
|
+
return this.readProtocolVersion(this.executor);
|
|
213
183
|
}
|
|
214
184
|
// ------------------------------------------------------------ wake channel
|
|
215
185
|
async claimWake(queues, timeoutMs) {
|
|
@@ -232,6 +202,15 @@ export class PostgresStore extends TaskStore {
|
|
|
232
202
|
await this.wakeOn("queued", remaining);
|
|
233
203
|
}
|
|
234
204
|
}
|
|
205
|
+
subscribePush(onSignal) {
|
|
206
|
+
this.subscribers.add(onSignal);
|
|
207
|
+
this.listenerReady(); // an API-side watcher is often the only thing asking
|
|
208
|
+
return () => void this.subscribers.delete(onSignal);
|
|
209
|
+
}
|
|
210
|
+
warmPush() {
|
|
211
|
+
if (this.subscribers.size > 0)
|
|
212
|
+
this.listenerReady();
|
|
213
|
+
}
|
|
235
214
|
taskDoneWake(taskId, timeoutMs) {
|
|
236
215
|
if (!this.listenerReady())
|
|
237
216
|
return super.taskDoneWake(taskId, timeoutMs);
|
|
@@ -258,7 +237,7 @@ export class PostgresStore extends TaskStore {
|
|
|
258
237
|
peers.add(waiter);
|
|
259
238
|
});
|
|
260
239
|
}
|
|
261
|
-
/** True once the LISTEN
|
|
240
|
+
/** True once the LISTEN subscription is up; starts establishing it otherwise
|
|
262
241
|
* (respecting the transient-failure backoff). Callers fall back to plain
|
|
263
242
|
* polling until it is ready (or forever, if it can't be established) —
|
|
264
243
|
* correctness never depends on it. */
|
|
@@ -272,38 +251,38 @@ export class PostgresStore extends TaskStore {
|
|
|
272
251
|
}
|
|
273
252
|
async startListener() {
|
|
274
253
|
try {
|
|
275
|
-
const
|
|
276
|
-
//
|
|
277
|
-
|
|
278
|
-
|
|
254
|
+
const executor = this.executor ?? this.provided;
|
|
255
|
+
// Not connected yet: transient by definition — doConnect calls back in.
|
|
256
|
+
if (!executor)
|
|
257
|
+
return;
|
|
258
|
+
if (!executor.listen) {
|
|
259
|
+
this.listenerUnavailable = true; // this executor will never push
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
let stop;
|
|
279
263
|
try {
|
|
280
|
-
await
|
|
264
|
+
stop = await executor.listen([QUEUED_CHANNEL, DONE_CHANNEL], (channel, payload) => this.onNotification(channel, payload),
|
|
265
|
+
// A dropped listener degrades to polling; the next wake reconnects.
|
|
266
|
+
() => this.dropListener());
|
|
281
267
|
}
|
|
282
|
-
catch {
|
|
283
|
-
|
|
268
|
+
catch (e) {
|
|
269
|
+
if (e instanceof ListenUnavailable) {
|
|
270
|
+
// Deterministic (e.g. a transaction-mode pooler): off for good rather
|
|
271
|
+
// than a reconnect loop that cannot succeed. Polling covers it.
|
|
272
|
+
this.listenerUnavailable = true;
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
// Could not establish it — transient. Schedule a backed-off retry;
|
|
284
276
|
// polling covers the gap.
|
|
285
277
|
this.listenerRetryAt = Date.now() + this.listenerBackoffMs;
|
|
286
278
|
this.listenerBackoffMs = Math.min(LISTENER_RETRY_MAX_MS, this.listenerBackoffMs * 2);
|
|
287
279
|
return;
|
|
288
280
|
}
|
|
289
|
-
client.on("notification", (msg) => this.onNotification(msg.channel, msg.payload));
|
|
290
|
-
// A dropped listener degrades to polling; the next wake call reconnects.
|
|
291
|
-
client.on("error", () => this.dropListener());
|
|
292
|
-
try {
|
|
293
|
-
await client.query(`listen ${QUEUED_CHANNEL}; listen ${DONE_CHANNEL}`);
|
|
294
|
-
}
|
|
295
|
-
catch {
|
|
296
|
-
// Connected, but LISTEN was refused (e.g. a transaction-mode pooler) —
|
|
297
|
-
// deterministic, so off for good. Polling covers it.
|
|
298
|
-
this.listenerUnavailable = true;
|
|
299
|
-
void client.end().catch(() => { });
|
|
300
|
-
return;
|
|
301
|
-
}
|
|
302
281
|
if (this.listenerUnavailable) {
|
|
303
|
-
|
|
282
|
+
stop(); // closed while we were subscribing
|
|
304
283
|
return;
|
|
305
284
|
}
|
|
306
|
-
this.listener =
|
|
285
|
+
this.listener = stop;
|
|
307
286
|
this.listenerBackoffMs = LISTENER_RETRY_MS;
|
|
308
287
|
}
|
|
309
288
|
catch {
|
|
@@ -320,9 +299,11 @@ export class PostgresStore extends TaskStore {
|
|
|
320
299
|
if (channel === QUEUED_CHANNEL && payload) {
|
|
321
300
|
this.pendingQueues.add(payload);
|
|
322
301
|
key = "queued";
|
|
302
|
+
this.publish({ reason: "queued", queue: payload });
|
|
323
303
|
}
|
|
324
304
|
else if (channel === DONE_CHANNEL && payload) {
|
|
325
305
|
key = `done:${payload}`;
|
|
306
|
+
this.publish({ reason: "done", taskId: payload });
|
|
326
307
|
}
|
|
327
308
|
else {
|
|
328
309
|
return;
|
|
@@ -333,11 +314,23 @@ export class PostgresStore extends TaskStore {
|
|
|
333
314
|
for (const w of set)
|
|
334
315
|
w();
|
|
335
316
|
}
|
|
317
|
+
/** Hand a notification to every watch() subscriber. A throwing subscriber is
|
|
318
|
+
* its own problem: it must not cost the others their signal, nor take down the
|
|
319
|
+
* listener connection that delivered it. */
|
|
320
|
+
publish(signal) {
|
|
321
|
+
for (const s of [...this.subscribers]) {
|
|
322
|
+
try {
|
|
323
|
+
s(signal);
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
// Deliberately swallowed — see above.
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
336
330
|
dropListener() {
|
|
337
|
-
const
|
|
331
|
+
const stop = this.listener;
|
|
338
332
|
this.listener = null;
|
|
339
|
-
|
|
340
|
-
void client.end().catch(() => { });
|
|
333
|
+
stop?.();
|
|
341
334
|
// Release everyone promptly; their fallback poll takes over.
|
|
342
335
|
for (const set of [...this.waiters.values()])
|
|
343
336
|
for (const w of [...set])
|
|
@@ -347,26 +340,21 @@ export class PostgresStore extends TaskStore {
|
|
|
347
340
|
async fetch(name, params) {
|
|
348
341
|
await this.ensure();
|
|
349
342
|
const { text, values } = toPositional(this.statements[name], params);
|
|
350
|
-
return
|
|
343
|
+
return this.executor.query(text, values);
|
|
351
344
|
}
|
|
352
345
|
async tx(fn) {
|
|
353
346
|
await this.ensure();
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
throw e;
|
|
367
|
-
}
|
|
368
|
-
finally {
|
|
369
|
-
client.release();
|
|
370
|
-
}
|
|
347
|
+
return this.executor.tx((s) => fn(this.boundFetch(s)));
|
|
348
|
+
}
|
|
349
|
+
async txWithSession(fn) {
|
|
350
|
+
await this.ensure();
|
|
351
|
+
return this.executor.tx((s) => fn(this.boundFetch(s), s));
|
|
352
|
+
}
|
|
353
|
+
/** A Fetch that runs the protocol's statements on one particular session. */
|
|
354
|
+
boundFetch(s) {
|
|
355
|
+
return async (name, params) => {
|
|
356
|
+
const { text, values } = toPositional(this.statements[name], params);
|
|
357
|
+
return s.query(text, values);
|
|
358
|
+
};
|
|
371
359
|
}
|
|
372
360
|
}
|