@bastani/atomic-storage-postgres 0.0.0-stage → 0.9.88-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/LICENSE +24 -0
- package/README.md +149 -2
- package/auth.d.ts +38 -0
- package/auth.js +237 -0
- package/listen.d.ts +30 -0
- package/listen.js +255 -0
- package/package.json +85 -4
- package/postgres.d.ts +51 -0
- package/postgres.js +1183 -0
package/listen.js
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @typedef {import("@bastani/atomic/storage").StorageListenerState} StorageListenerState
|
|
6
|
+
* @typedef {import("@bastani/atomic/storage").StorageNotificationEvent} StorageNotificationEvent
|
|
7
|
+
* @typedef {import("@bastani/atomic/storage").StorageNotifications} StorageNotifications
|
|
8
|
+
* @typedef {(event: StorageNotificationEvent) => void} NotificationListener
|
|
9
|
+
* @typedef {{
|
|
10
|
+
* query(sql: string): Promise<unknown>,
|
|
11
|
+
* on(event: "notification", listener: (message: { channel: string, payload?: string }) => void): unknown,
|
|
12
|
+
* on(event: "error" | "end", listener: () => void): unknown,
|
|
13
|
+
* end(): Promise<void>,
|
|
14
|
+
* }} ListenClient
|
|
15
|
+
* @typedef {{
|
|
16
|
+
* connect: () => Promise<ListenClient>,
|
|
17
|
+
* send: (channel: string, payload: string) => Promise<void>,
|
|
18
|
+
* channelName: (channel: string) => string,
|
|
19
|
+
* probeTimeoutMs?: number,
|
|
20
|
+
* retry?: { readonly firstMs: number, readonly maxMs: number },
|
|
21
|
+
* healthIntervalMs?: number,
|
|
22
|
+
* }} ListenerOptions
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* How long the listening connection waits for its probe notification, sent through the database's pool, before it
|
|
27
|
+
* decides the endpoint is a transaction pooler that does not deliver notifications to the listening session.
|
|
28
|
+
*/
|
|
29
|
+
export const LISTEN_PROBE_TIMEOUT_MS = 2_000;
|
|
30
|
+
/** The wait before the listening connection is opened again after it failed, and the longest such wait. */
|
|
31
|
+
export const LISTEN_RETRY = /** @type {const} */ ({ firstMs: 1_000, maxMs: 30_000 });
|
|
32
|
+
/**
|
|
33
|
+
* How often a listening connection sends itself a probe notification through the pool, which must arrive within
|
|
34
|
+
* `LISTEN_PROBE_TIMEOUT_MS`; otherwise the connection is taken for half-open (its peer gone without a reset) and
|
|
35
|
+
* replaced.
|
|
36
|
+
*/
|
|
37
|
+
export const LISTEN_HEALTH_INTERVAL_MS = 30_000;
|
|
38
|
+
|
|
39
|
+
/** Quotes `name` as an SQL identifier. */
|
|
40
|
+
function quoted(/** @type {string} */ name) {
|
|
41
|
+
return `"${name.replaceAll('"', '""')}"`;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Postgres `LISTEN`/`NOTIFY` wake-ups (#5103). Every channel shares one listening connection, opened on the first
|
|
46
|
+
* `listen` or `state` call and again, with a growing wait, after it fails or breaks. Once connected it listens to a
|
|
47
|
+
* probe channel of its own and sends a probe notification through the database's pool. When the probe never arrives,
|
|
48
|
+
* as through a transaction pooler, it unlistens, closes the connection, and stays `pooler` for good: it listens to
|
|
49
|
+
* no other channel there, so no later notification reaches a pooled server connection. While listening it repeats
|
|
50
|
+
* the probe every `healthIntervalMs` and replaces a connection the probe no longer reaches. Callers keep their polls;
|
|
51
|
+
* this only makes them find work sooner.
|
|
52
|
+
* @param {ListenerOptions} options
|
|
53
|
+
* @returns {StorageNotifications & { close(): Promise<void> }}
|
|
54
|
+
*/
|
|
55
|
+
export function createPostgresListener(options) {
|
|
56
|
+
const probeTimeoutMs = options.probeTimeoutMs ?? LISTEN_PROBE_TIMEOUT_MS;
|
|
57
|
+
const retry = options.retry ?? LISTEN_RETRY;
|
|
58
|
+
const healthIntervalMs = options.healthIntervalMs ?? LISTEN_HEALTH_INTERVAL_MS;
|
|
59
|
+
const probeChannel = `atomic_listen_probe_${randomBytes(6).toString("hex")}`;
|
|
60
|
+
/** @type {Map<string, Set<NotificationListener>>} */
|
|
61
|
+
const channels = new Map();
|
|
62
|
+
/** @type {StorageListenerState} */
|
|
63
|
+
let state = { status: "connecting" };
|
|
64
|
+
/** @type {ListenClient | undefined} */
|
|
65
|
+
let client;
|
|
66
|
+
/** The connection while it listens: `client` once its probe arrived. */
|
|
67
|
+
/** @type {ListenClient | undefined} */
|
|
68
|
+
let live;
|
|
69
|
+
/** @type {Promise<void> | undefined} */
|
|
70
|
+
let firstAttempt;
|
|
71
|
+
/** @type {NodeJS.Timeout | undefined} */
|
|
72
|
+
let retryTimer;
|
|
73
|
+
/** @type {NodeJS.Timeout | undefined} */
|
|
74
|
+
let healthTimer;
|
|
75
|
+
let failures = 0;
|
|
76
|
+
let closed = false;
|
|
77
|
+
/** A transaction pooler was found: it stays one, and probing it again would leave a LISTEN on it each time. */
|
|
78
|
+
let pooled = false;
|
|
79
|
+
/** The probes in flight, by nonce. */
|
|
80
|
+
/** @type {Map<string, () => void>} */
|
|
81
|
+
const probes = new Map();
|
|
82
|
+
|
|
83
|
+
/** @param {string} name @param {StorageNotificationEvent} event */
|
|
84
|
+
const dispatch = (name, event) => {
|
|
85
|
+
const listeners = channels.get(name);
|
|
86
|
+
// A notification can arrive for a channel whose last listener just left, before its UNLISTEN.
|
|
87
|
+
if (listeners === undefined) return;
|
|
88
|
+
for (const listener of [...listeners]) {
|
|
89
|
+
try {
|
|
90
|
+
listener(event);
|
|
91
|
+
} catch {
|
|
92
|
+
// A listener's failure is its own.
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const scheduleRetry = () => {
|
|
98
|
+
if (closed || pooled || retryTimer !== undefined) return;
|
|
99
|
+
const delayMs = Math.min(retry.maxMs, retry.firstMs * 2 ** failures);
|
|
100
|
+
failures++;
|
|
101
|
+
retryTimer = setTimeout(() => {
|
|
102
|
+
retryTimer = undefined;
|
|
103
|
+
void attempt();
|
|
104
|
+
}, delayMs);
|
|
105
|
+
retryTimer.unref?.();
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
/** @param {ListenClient} lost @param {"pooler" | "connection-failed"} reason */
|
|
109
|
+
const drop = (lost, reason) => {
|
|
110
|
+
if (client !== lost) return;
|
|
111
|
+
client = undefined;
|
|
112
|
+
live = undefined;
|
|
113
|
+
clearTimeout(healthTimer);
|
|
114
|
+
state = { status: "unavailable", reason };
|
|
115
|
+
void lost.end().catch(() => {});
|
|
116
|
+
scheduleRetry();
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Sends a probe notification through the pool, and resolves with whether it reached the listening connection within
|
|
121
|
+
* the probe timeout.
|
|
122
|
+
* @returns {Promise<boolean>}
|
|
123
|
+
*/
|
|
124
|
+
const probe = async () => {
|
|
125
|
+
const nonce = randomBytes(8).toString("hex");
|
|
126
|
+
/** @type {NodeJS.Timeout | undefined} */
|
|
127
|
+
let timer;
|
|
128
|
+
const arrived = new Promise((resolve) => {
|
|
129
|
+
probes.set(nonce, () => resolve(true));
|
|
130
|
+
timer = setTimeout(() => resolve(false), probeTimeoutMs);
|
|
131
|
+
});
|
|
132
|
+
try {
|
|
133
|
+
await options.send(probeChannel, nonce);
|
|
134
|
+
return /** @type {boolean} */ (await arrived);
|
|
135
|
+
} finally {
|
|
136
|
+
clearTimeout(timer);
|
|
137
|
+
probes.delete(nonce);
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
/** @param {ListenClient} opened */
|
|
142
|
+
const scheduleHealth = (opened) => {
|
|
143
|
+
// `drop` and `close` clear the timer, so it fires only while `opened` listens.
|
|
144
|
+
healthTimer = setTimeout(async () => {
|
|
145
|
+
let healthy = false;
|
|
146
|
+
try {
|
|
147
|
+
healthy = await probe();
|
|
148
|
+
} catch {
|
|
149
|
+
// The pool could not send the probe: the connection is not known to listen.
|
|
150
|
+
}
|
|
151
|
+
if (healthy) scheduleHealth(opened);
|
|
152
|
+
else drop(opened, "connection-failed");
|
|
153
|
+
}, healthIntervalMs);
|
|
154
|
+
healthTimer.unref?.();
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
const attempt = async () => {
|
|
158
|
+
if (closed) return;
|
|
159
|
+
/** @type {ListenClient} */
|
|
160
|
+
let opened;
|
|
161
|
+
try {
|
|
162
|
+
opened = await options.connect();
|
|
163
|
+
} catch {
|
|
164
|
+
state = { status: "unavailable", reason: "connection-failed" };
|
|
165
|
+
scheduleRetry();
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
if (closed) {
|
|
169
|
+
await opened.end().catch(() => {});
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
client = opened;
|
|
173
|
+
opened.on("error", () => drop(opened, "connection-failed"));
|
|
174
|
+
opened.on("end", () => drop(opened, "connection-failed"));
|
|
175
|
+
opened.on("notification", (message) => {
|
|
176
|
+
const payload = message.payload ?? "";
|
|
177
|
+
if (message.channel === probeChannel) probes.get(payload)?.();
|
|
178
|
+
else if (client === opened && state.status === "listening")
|
|
179
|
+
dispatch(message.channel, { kind: "notification", payload });
|
|
180
|
+
});
|
|
181
|
+
try {
|
|
182
|
+
// Only the probe channel until the probe arrives: a pooler keeps a LISTEN on whichever server connection ran
|
|
183
|
+
// it, and that connection's later notifications would reach no client.
|
|
184
|
+
await opened.query(`LISTEN ${quoted(probeChannel)}`);
|
|
185
|
+
if (!(await probe())) {
|
|
186
|
+
await opened.query("UNLISTEN *").catch(() => {});
|
|
187
|
+
pooled = true;
|
|
188
|
+
drop(opened, "pooler");
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
// Channels added meanwhile are in the map, and the loop reaches them.
|
|
192
|
+
for (const name of channels.keys()) await opened.query(`LISTEN ${quoted(name)}`);
|
|
193
|
+
} catch {
|
|
194
|
+
drop(opened, "connection-failed");
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
if (client !== opened) return;
|
|
198
|
+
failures = 0;
|
|
199
|
+
live = opened;
|
|
200
|
+
state = { status: "listening" };
|
|
201
|
+
scheduleHealth(opened);
|
|
202
|
+
for (const name of channels.keys()) dispatch(name, { kind: "listening" });
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
const start = () => {
|
|
206
|
+
firstAttempt ??= attempt();
|
|
207
|
+
return firstAttempt;
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
return {
|
|
211
|
+
notify(channel, payload) {
|
|
212
|
+
return { sql: "SELECT pg_notify(?, ?)", params: [options.channelName(channel), payload] };
|
|
213
|
+
},
|
|
214
|
+
listen(channel, listener) {
|
|
215
|
+
const name = options.channelName(channel);
|
|
216
|
+
let listeners = channels.get(name);
|
|
217
|
+
if (listeners === undefined) {
|
|
218
|
+
listeners = new Set();
|
|
219
|
+
channels.set(name, listeners);
|
|
220
|
+
const current = live;
|
|
221
|
+
// A channel added while listening is listened to on that connection; a failure drops it, and the next one
|
|
222
|
+
// listens to every channel. One added while connecting is listened to once the probe arrives.
|
|
223
|
+
if (current !== undefined)
|
|
224
|
+
current.query(`LISTEN ${quoted(name)}`).catch(() => drop(current, "connection-failed"));
|
|
225
|
+
}
|
|
226
|
+
listeners.add(listener);
|
|
227
|
+
void start();
|
|
228
|
+
return {
|
|
229
|
+
close() {
|
|
230
|
+
const current = channels.get(name);
|
|
231
|
+
// Closing again finds the listener gone.
|
|
232
|
+
if (!current?.delete(listener) || current.size > 0) return;
|
|
233
|
+
channels.delete(name);
|
|
234
|
+
const connected = live;
|
|
235
|
+
if (connected !== undefined)
|
|
236
|
+
connected.query(`UNLISTEN ${quoted(name)}`).catch(() => drop(connected, "connection-failed"));
|
|
237
|
+
},
|
|
238
|
+
};
|
|
239
|
+
},
|
|
240
|
+
async state() {
|
|
241
|
+
await start();
|
|
242
|
+
return state;
|
|
243
|
+
},
|
|
244
|
+
async close() {
|
|
245
|
+
closed = true;
|
|
246
|
+
clearTimeout(retryTimer);
|
|
247
|
+
clearTimeout(healthTimer);
|
|
248
|
+
retryTimer = undefined;
|
|
249
|
+
await firstAttempt;
|
|
250
|
+
const current = client;
|
|
251
|
+
client = undefined;
|
|
252
|
+
await current?.end().catch(() => {});
|
|
253
|
+
},
|
|
254
|
+
};
|
|
255
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,87 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bastani/atomic-storage-postgres",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.9.88-alpha.1",
|
|
4
|
+
"description": "PostgreSQL storage adapter for Atomic sessions, workflow journals, and Intercom.",
|
|
5
|
+
"contributors": [
|
|
6
|
+
"Norin Lavaee",
|
|
7
|
+
"Alex Lavaee"
|
|
8
|
+
],
|
|
9
|
+
"license": "MIT",
|
|
10
|
+
"type": "module",
|
|
11
|
+
"engines": {
|
|
12
|
+
"node": ">=22.19.0"
|
|
13
|
+
},
|
|
14
|
+
"main": "./postgres.js",
|
|
15
|
+
"types": "./postgres.d.ts",
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"types": "./postgres.d.ts",
|
|
19
|
+
"import": "./postgres.js",
|
|
20
|
+
"default": "./postgres.js"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"files": [
|
|
24
|
+
"postgres.js",
|
|
25
|
+
"postgres.d.ts",
|
|
26
|
+
"auth.js",
|
|
27
|
+
"auth.d.ts",
|
|
28
|
+
"listen.js",
|
|
29
|
+
"listen.d.ts",
|
|
30
|
+
"README.md",
|
|
31
|
+
"CHANGELOG.md",
|
|
32
|
+
"LICENSE"
|
|
33
|
+
],
|
|
34
|
+
"publishConfig": {
|
|
35
|
+
"access": "public"
|
|
36
|
+
},
|
|
37
|
+
"repository": {
|
|
38
|
+
"type": "git",
|
|
39
|
+
"url": "git+https://github.com/bastani-inc/atomic.git",
|
|
40
|
+
"directory": "packages/storage-postgres"
|
|
41
|
+
},
|
|
42
|
+
"homepage": "https://github.com/bastani-inc/atomic/tree/main/packages/storage-postgres#readme",
|
|
43
|
+
"bugs": {
|
|
44
|
+
"url": "https://github.com/bastani-inc/atomic/issues"
|
|
45
|
+
},
|
|
46
|
+
"keywords": [
|
|
47
|
+
"atomic",
|
|
48
|
+
"storage",
|
|
49
|
+
"postgres",
|
|
50
|
+
"postgresql",
|
|
51
|
+
"bastani"
|
|
52
|
+
],
|
|
53
|
+
"atomic": {
|
|
54
|
+
"storageAdapters": [
|
|
55
|
+
"./postgres.js"
|
|
56
|
+
]
|
|
57
|
+
},
|
|
58
|
+
"pi": {
|
|
59
|
+
"storageAdapters": [
|
|
60
|
+
"./postgres.js"
|
|
61
|
+
]
|
|
62
|
+
},
|
|
63
|
+
"dependencies": {
|
|
64
|
+
"pg": "8.23.1",
|
|
65
|
+
"pg-connection-string": "2.14.1"
|
|
66
|
+
},
|
|
67
|
+
"peerDependencies": {
|
|
68
|
+
"@aws-sdk/rds-signer": "^3.1169.0",
|
|
69
|
+
"@azure/identity": "^4.14.1",
|
|
70
|
+
"@bastani/atomic": "*",
|
|
71
|
+
"google-auth-library": "^10.9.1 || ^11.3.4"
|
|
72
|
+
},
|
|
73
|
+
"peerDependenciesMeta": {
|
|
74
|
+
"@aws-sdk/rds-signer": {
|
|
75
|
+
"optional": true
|
|
76
|
+
},
|
|
77
|
+
"@azure/identity": {
|
|
78
|
+
"optional": true
|
|
79
|
+
},
|
|
80
|
+
"@bastani/atomic": {
|
|
81
|
+
"optional": true
|
|
82
|
+
},
|
|
83
|
+
"google-auth-library": {
|
|
84
|
+
"optional": true
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
package/postgres.d.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { AdapterOptions, SqliteDatabase, StorageOpenContext } from "@bastani/atomic/storage";
|
|
2
|
+
|
|
3
|
+
export interface PostgresDatabaseOptions {
|
|
4
|
+
/** A `postgres://` or `postgresql://` URL without a password. */
|
|
5
|
+
url: string;
|
|
6
|
+
/** The password, or a function that returns it, called for each new connection the server asks one of. */
|
|
7
|
+
password?: string | (() => Promise<string>);
|
|
8
|
+
/**
|
|
9
|
+
* How the password is obtained. Default `"password"`. `"azure-entra"`, `"aws-iam"` and `"gcp-iam"` take a token from
|
|
10
|
+
* the platform's own credentials for each new connection, and refuse a `password` beside them.
|
|
11
|
+
*/
|
|
12
|
+
auth?: "password" | "azure-entra" | "aws-iam" | "gcp-iam";
|
|
13
|
+
/** Connections kept open at most, in this process. Default 8. */
|
|
14
|
+
maxConnections?: number;
|
|
15
|
+
/** How long one statement may run before the server cancels it, in milliseconds. Default 30000. */
|
|
16
|
+
statementTimeoutMs?: number;
|
|
17
|
+
/**
|
|
18
|
+
* A direct (session) endpoint for the connection that listens for wake-ups, when `url` goes through a transaction
|
|
19
|
+
* pooler. Default `url`. Without notifications, worker hosts and Intercom brokers find work by polling alone.
|
|
20
|
+
*/
|
|
21
|
+
listenUrl?: string;
|
|
22
|
+
/**
|
|
23
|
+
* The schema Atomic's tables live in (#5218). Default `atomic`, created by the first store that sets up its tables
|
|
24
|
+
* when the role may; a database whose `public` schema holds an earlier release's tables, and has no `atomic` schema,
|
|
25
|
+
* keeps using `public` until `atomic storage migrate` moves them.
|
|
26
|
+
*/
|
|
27
|
+
schema?: string;
|
|
28
|
+
/**
|
|
29
|
+
* npm prefixes to find a cloud `auth`'s SDK in when it is not found beside the adapter. The adapter's
|
|
30
|
+
* `openDatabase` passes `<agentDir>/npm`, where `atomic install npm:<package>` installs user packages.
|
|
31
|
+
*/
|
|
32
|
+
sdkSearchPaths?: readonly string[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export const DEFAULT_MAX_CONNECTIONS: 8;
|
|
36
|
+
export const DEFAULT_STATEMENT_TIMEOUT_MS: 30000;
|
|
37
|
+
export const DEFAULT_SCHEMA: "atomic";
|
|
38
|
+
|
|
39
|
+
/** `sql` with each positional `?` parameter outside literals, quoted identifiers and comments numbered `$1`, `$2`, .... */
|
|
40
|
+
export function numberPlaceholders(sql: string): string;
|
|
41
|
+
/** `sql` with every unquoted identifier longer than 63 bytes replaced by `postgresDialect.identifier` of it. */
|
|
42
|
+
export function physicalIdentifiers(sql: string): string;
|
|
43
|
+
|
|
44
|
+
export function openPostgresDatabase(options: PostgresDatabaseOptions): Promise<SqliteDatabase>;
|
|
45
|
+
|
|
46
|
+
declare const adapter: {
|
|
47
|
+
atomicStorageAdapter: 1;
|
|
48
|
+
liveSecrets: readonly string[];
|
|
49
|
+
openDatabase(options: AdapterOptions, context: StorageOpenContext): Promise<SqliteDatabase>;
|
|
50
|
+
};
|
|
51
|
+
export default adapter;
|