bunsane 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/SchedulerManager.ts +1 -0
- package/core/scheduler/DistributedLock.ts +147 -140
- package/core/scheduler/index.ts +13 -0
- package/core/scheduler/lockCoordinator.ts +1 -0
- package/core/scheduler/locks/AdvisoryLockBackend.ts +276 -0
- package/core/scheduler/locks/InProcessLockBackend.ts +85 -0
- package/core/scheduler/locks/LockBackend.ts +91 -0
- package/core/scheduler/locks/PostgresLeaseLockBackend.ts +159 -0
- package/core/scheduler/locks/index.ts +81 -0
- package/core/scheduler/withLock.ts +159 -98
- package/endpoints/archetypes.ts +1 -1
- package/package.json +1 -1
- package/types/scheduler.types.ts +13 -0
- package/utils/archetypeIndicator.ts +54 -0
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AdvisoryLockBackend — PostgreSQL session advisory locks (`pg_advisory_lock`).
|
|
3
|
+
*
|
|
4
|
+
* The framework's historical lock primitive, preserved as an OPT-IN backend.
|
|
5
|
+
*
|
|
6
|
+
* ⚠️ Advisory locks are bound to the PostgreSQL *session* that took them. This
|
|
7
|
+
* backend pins one connection via `sql.reserve()` and routes every lock/unlock
|
|
8
|
+
* through it, so a process's locks all live in one session and unlock always
|
|
9
|
+
* hits the acquiring session. If the process crashes, PostgreSQL drops the
|
|
10
|
+
* session and releases every lock automatically.
|
|
11
|
+
*
|
|
12
|
+
* This is ONLY safe when the client→server path preserves session affinity,
|
|
13
|
+
* i.e. a direct PostgreSQL connection or pgbouncer in `session`/`statement`
|
|
14
|
+
* mode. Behind pgbouncer `pool_mode = transaction` the reserved connection
|
|
15
|
+
* pins client→pgbouncer but NOT pgbouncer→backend, so lock and unlock land on
|
|
16
|
+
* different backends and the lock strands silently (BUNSANE-1). Prefer
|
|
17
|
+
* {@link PostgresLeaseLockBackend} unless you control a session-pinned lane.
|
|
18
|
+
*
|
|
19
|
+
* @see https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { randomUUID } from "crypto";
|
|
23
|
+
import type { ReservedSQL, SQL } from "bun";
|
|
24
|
+
import db from "../../../database";
|
|
25
|
+
import { logger } from "../../Logger";
|
|
26
|
+
import type { AcquireOptions, LockBackend, LockHandle } from "./LockBackend";
|
|
27
|
+
|
|
28
|
+
const loggerInstance = logger.child({ scope: "AdvisoryLockBackend" });
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Thrown on first use when the advisory backend detects it is NOT on a
|
|
32
|
+
* session-pinned connection (i.e. behind a transaction-pooling pooler), where
|
|
33
|
+
* advisory locks strand silently (BUNSANE-1/-7). This is a configuration error,
|
|
34
|
+
* not a transient lock failure — {@link DistributedLock} re-throws it so the
|
|
35
|
+
* misconfiguration fails loudly instead of silently skipping critical sections.
|
|
36
|
+
* Acknowledge and bypass with `BUNSANE_ALLOW_UNSAFE_ADVISORY_LOCK=true`.
|
|
37
|
+
*/
|
|
38
|
+
export class UnsafeAdvisoryPoolingError extends Error {
|
|
39
|
+
constructor() {
|
|
40
|
+
super(
|
|
41
|
+
"Advisory lock backend is running on a connection without session affinity " +
|
|
42
|
+
"(transaction-pooling pooler detected). Session advisory locks strand silently here. " +
|
|
43
|
+
"Use the default 'postgres' lease backend, point this app at a session-pinned lane, " +
|
|
44
|
+
"or set BUNSANE_ALLOW_UNSAFE_ADVISORY_LOCK=true to override at your own risk."
|
|
45
|
+
);
|
|
46
|
+
this.name = "UnsafeAdvisoryPoolingError";
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface AdvisoryBackendConfig {
|
|
51
|
+
/** Namespace prefix occupying the high 32 bits of the advisory key. */
|
|
52
|
+
lockKeyPrefix: number;
|
|
53
|
+
enableLogging: boolean;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const DEFAULT_PREFIX = 0x42554e53; // "BUNS"
|
|
57
|
+
|
|
58
|
+
interface AdvisoryInternal {
|
|
59
|
+
bigintKey: bigint;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export class AdvisoryLockBackend implements LockBackend {
|
|
63
|
+
readonly name = "advisory";
|
|
64
|
+
private config: AdvisoryBackendConfig;
|
|
65
|
+
private readonly sql: SQL;
|
|
66
|
+
|
|
67
|
+
private reservedConn: ReservedSQL | null = null;
|
|
68
|
+
private reservePromise: Promise<ReservedSQL> | null = null;
|
|
69
|
+
/** Outstanding handles; the reserved session is freed when this hits 0. */
|
|
70
|
+
private outstanding = 0;
|
|
71
|
+
/** Session-affinity probe runs once per reserved session. */
|
|
72
|
+
private safetyChecked = false;
|
|
73
|
+
|
|
74
|
+
constructor(config: Partial<AdvisoryBackendConfig> = {}, sql: SQL = db) {
|
|
75
|
+
this.config = {
|
|
76
|
+
lockKeyPrefix: config.lockKeyPrefix ?? DEFAULT_PREFIX,
|
|
77
|
+
enableLogging: config.enableLogging ?? false,
|
|
78
|
+
};
|
|
79
|
+
this.sql = sql;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* 32-bit string hash folded into the low half of the advisory key under a
|
|
84
|
+
* fixed 32-bit prefix. NOTE (BUNSANE-4): effective key space is ~2^32, so
|
|
85
|
+
* distinct keys can hash-collide onto the same advisory id → false
|
|
86
|
+
* contention. Acceptable for the opt-in advisory path; lease backends key
|
|
87
|
+
* on the raw text and have no such collision.
|
|
88
|
+
*/
|
|
89
|
+
private generateLockKey(key: string): bigint {
|
|
90
|
+
let hash = 0;
|
|
91
|
+
for (let i = 0; i < key.length; i++) {
|
|
92
|
+
const char = key.charCodeAt(i);
|
|
93
|
+
hash = (hash << 5) - hash + char;
|
|
94
|
+
hash = hash & hash;
|
|
95
|
+
}
|
|
96
|
+
hash = Math.abs(hash);
|
|
97
|
+
const prefix = BigInt(this.config.lockKeyPrefix);
|
|
98
|
+
const hashBigInt = BigInt(hash >>> 0);
|
|
99
|
+
return (prefix << 32n) | hashBigInt;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
private async ensureReserved(): Promise<ReservedSQL> {
|
|
103
|
+
if (this.reservedConn) return this.reservedConn;
|
|
104
|
+
if (!this.reservePromise) {
|
|
105
|
+
// On reject (pool exhausted / shutdown mid-reserve), null the
|
|
106
|
+
// promise so the next caller retries a fresh reserve rather than
|
|
107
|
+
// re-awaiting the same rejected one forever (H-DB-2).
|
|
108
|
+
this.reservePromise = this.sql.reserve().then(
|
|
109
|
+
(conn) => {
|
|
110
|
+
this.reservedConn = conn;
|
|
111
|
+
this.reservePromise = null;
|
|
112
|
+
return conn;
|
|
113
|
+
},
|
|
114
|
+
(err) => {
|
|
115
|
+
this.reservePromise = null;
|
|
116
|
+
throw err;
|
|
117
|
+
}
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
return this.reservePromise;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
private releaseReservationIfIdle(): void {
|
|
124
|
+
if (this.outstanding > 0 || !this.reservedConn) return;
|
|
125
|
+
try {
|
|
126
|
+
this.reservedConn.release();
|
|
127
|
+
} catch (error) {
|
|
128
|
+
loggerInstance.warn(
|
|
129
|
+
`Failed to release reserved connection: ${error instanceof Error ? error.message : String(error)}`
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
this.reservedConn = null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Verify the reserved connection has session affinity. Sets a session GUC
|
|
137
|
+
* then reads it back on a SEPARATE query: under a transaction pooler the two
|
|
138
|
+
* queries land on different backends so the value is lost, exposing the
|
|
139
|
+
* silently-unsafe config (BUNSANE-1/-7). Runs once per session. A probe that
|
|
140
|
+
* errors out (e.g. an engine that disallows the GUC) is treated as
|
|
141
|
+
* inconclusive → assume safe rather than block. Throws
|
|
142
|
+
* {@link UnsafeAdvisoryPoolingError} on an affirmative "not pinned" unless
|
|
143
|
+
* acknowledged via `BUNSANE_ALLOW_UNSAFE_ADVISORY_LOCK=true`.
|
|
144
|
+
*/
|
|
145
|
+
private async checkSessionAffinity(conn: ReservedSQL): Promise<void> {
|
|
146
|
+
if (this.safetyChecked) return;
|
|
147
|
+
|
|
148
|
+
if (process.env.BUNSANE_ALLOW_UNSAFE_ADVISORY_LOCK === "true") {
|
|
149
|
+
this.safetyChecked = true;
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const token = randomUUID();
|
|
154
|
+
let readBack: string | null = null;
|
|
155
|
+
try {
|
|
156
|
+
await conn`SELECT set_config('bunsane.lock_affinity_probe', ${token}, false)`;
|
|
157
|
+
const rows = await conn`
|
|
158
|
+
SELECT current_setting('bunsane.lock_affinity_probe', true) AS v
|
|
159
|
+
`;
|
|
160
|
+
readBack = rows[0]?.v ?? null;
|
|
161
|
+
} catch (error) {
|
|
162
|
+
// Inconclusive (engine rejected the GUC, etc.) — don't block.
|
|
163
|
+
loggerInstance.debug(
|
|
164
|
+
`Session-affinity probe inconclusive, assuming safe: ${error instanceof Error ? error.message : String(error)}`
|
|
165
|
+
);
|
|
166
|
+
this.safetyChecked = true;
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
if (readBack !== token) {
|
|
171
|
+
loggerInstance.error(
|
|
172
|
+
"Advisory lock backend has NO session affinity — a transaction-pooling pooler is " +
|
|
173
|
+
"multiplexing queries across backends. Session advisory locks WILL strand silently " +
|
|
174
|
+
"(see BUNSANE-1). Switch to the 'postgres' lease backend or a session-pinned lane."
|
|
175
|
+
);
|
|
176
|
+
throw new UnsafeAdvisoryPoolingError();
|
|
177
|
+
}
|
|
178
|
+
this.safetyChecked = true;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async acquire(key: string, _opts?: AcquireOptions): Promise<LockHandle | null> {
|
|
182
|
+
const bigintKey = this.generateLockKey(key);
|
|
183
|
+
|
|
184
|
+
// Reserve the session (transient failures → null, retried next call).
|
|
185
|
+
let conn: ReservedSQL;
|
|
186
|
+
try {
|
|
187
|
+
conn = await this.ensureReserved();
|
|
188
|
+
} catch (error) {
|
|
189
|
+
loggerInstance.error(
|
|
190
|
+
`Error reserving connection for advisory lock ${key}: ${error instanceof Error ? error.message : String(error)}`
|
|
191
|
+
);
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// Fail LOUD on an unsafe pooling config (throws past the catch below).
|
|
196
|
+
await this.checkSessionAffinity(conn);
|
|
197
|
+
|
|
198
|
+
try {
|
|
199
|
+
const result = await conn`
|
|
200
|
+
SELECT pg_try_advisory_lock(${bigintKey}::bigint) as locked
|
|
201
|
+
`;
|
|
202
|
+
const acquired = result[0]?.locked ?? false;
|
|
203
|
+
if (!acquired) {
|
|
204
|
+
this.releaseReservationIfIdle();
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
this.outstanding++;
|
|
208
|
+
if (this.config.enableLogging) {
|
|
209
|
+
loggerInstance.debug(`Acquired advisory lock ${key} (${bigintKey})`);
|
|
210
|
+
}
|
|
211
|
+
const internal: AdvisoryInternal = { bigintKey };
|
|
212
|
+
return { key, token: bigintKey.toString(), expiresAt: null, _internal: internal };
|
|
213
|
+
} catch (error) {
|
|
214
|
+
loggerInstance.error(
|
|
215
|
+
`Error acquiring advisory lock ${key}: ${error instanceof Error ? error.message : String(error)}`
|
|
216
|
+
);
|
|
217
|
+
this.releaseReservationIfIdle();
|
|
218
|
+
return null;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async release(handle: LockHandle): Promise<boolean> {
|
|
223
|
+
const { bigintKey } = (handle._internal as AdvisoryInternal) ?? {
|
|
224
|
+
bigintKey: this.generateLockKey(handle.key),
|
|
225
|
+
};
|
|
226
|
+
if (!this.reservedConn) {
|
|
227
|
+
loggerInstance.warn(
|
|
228
|
+
`No reserved connection to release advisory lock ${handle.key}`
|
|
229
|
+
);
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
try {
|
|
233
|
+
const result = await this.reservedConn`
|
|
234
|
+
SELECT pg_advisory_unlock(${bigintKey}::bigint) as unlocked
|
|
235
|
+
`;
|
|
236
|
+
const released = result[0]?.unlocked ?? false;
|
|
237
|
+
this.outstanding = Math.max(0, this.outstanding - 1);
|
|
238
|
+
// A false unlock is the canonical stranded-lock signal (BUNSANE-1/2);
|
|
239
|
+
// the DistributedLock facade promotes it to a loud ERROR + counter,
|
|
240
|
+
// so keep the backend log at debug to avoid double-logging.
|
|
241
|
+
if (this.config.enableLogging) {
|
|
242
|
+
loggerInstance.debug(
|
|
243
|
+
released
|
|
244
|
+
? `Released advisory lock ${handle.key} (${bigintKey})`
|
|
245
|
+
: `pg_advisory_unlock returned false for ${handle.key} (${bigintKey}) — may be stranded on another backend`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
this.releaseReservationIfIdle();
|
|
249
|
+
return released;
|
|
250
|
+
} catch (error) {
|
|
251
|
+
loggerInstance.error(
|
|
252
|
+
`Error releasing advisory lock ${handle.key}: ${error instanceof Error ? error.message : String(error)}`
|
|
253
|
+
);
|
|
254
|
+
this.outstanding = Math.max(0, this.outstanding - 1);
|
|
255
|
+
this.releaseReservationIfIdle();
|
|
256
|
+
return false;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
async dispose(): Promise<void> {
|
|
261
|
+
this.outstanding = 0;
|
|
262
|
+
this.safetyChecked = false; // new session must re-probe affinity
|
|
263
|
+
if (this.reservedConn) {
|
|
264
|
+
try {
|
|
265
|
+
this.reservedConn.release();
|
|
266
|
+
} catch {
|
|
267
|
+
/* best-effort on shutdown */
|
|
268
|
+
}
|
|
269
|
+
this.reservedConn = null;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
updateConfig(config: Partial<AdvisoryBackendConfig>): void {
|
|
274
|
+
this.config = { ...this.config, ...config };
|
|
275
|
+
}
|
|
276
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* InProcessLockBackend — single-process mutual exclusion via an in-memory Map.
|
|
3
|
+
*
|
|
4
|
+
* Correct for single-instance deployments and as a faithful test double:
|
|
5
|
+
* unlike the old `enabled:false` no-op (which pretended every acquire
|
|
6
|
+
* succeeded and so never exercised contention), this backend genuinely refuses
|
|
7
|
+
* a second holder of the same key and honours lease expiry. Two backend
|
|
8
|
+
* *instances* sharing one process do NOT share state — construct one per
|
|
9
|
+
* process and share it (this is what the singleton wiring does).
|
|
10
|
+
*
|
|
11
|
+
* It provides no cross-process exclusion. Use {@link PostgresLeaseLockBackend}
|
|
12
|
+
* or {@link RedisLockBackend} for multi-instance deployments.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { randomUUID } from "crypto";
|
|
16
|
+
import type { AcquireOptions, LockBackend, LockHandle } from "./LockBackend";
|
|
17
|
+
|
|
18
|
+
const DEFAULT_TTL_MS = 30_000;
|
|
19
|
+
|
|
20
|
+
interface Lease {
|
|
21
|
+
token: string;
|
|
22
|
+
expiresAt: number; // Date.now() scale
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export class InProcessLockBackend implements LockBackend {
|
|
26
|
+
readonly name = "in-process";
|
|
27
|
+
private readonly leases = new Map<string, Lease>();
|
|
28
|
+
|
|
29
|
+
private now(): number {
|
|
30
|
+
// Date.now is fine in normal runtime code (forbidden only in workflow
|
|
31
|
+
// scripts). Centralised so tests can stub if needed.
|
|
32
|
+
return Date.now();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
private isLive(lease: Lease | undefined): lease is Lease {
|
|
36
|
+
return !!lease && lease.expiresAt > this.now();
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async acquire(key: string, opts?: AcquireOptions): Promise<LockHandle | null> {
|
|
40
|
+
const existing = this.leases.get(key);
|
|
41
|
+
if (this.isLive(existing)) {
|
|
42
|
+
return null; // someone else holds a live lease
|
|
43
|
+
}
|
|
44
|
+
// No live holder (free, or the prior lease lapsed → steal it).
|
|
45
|
+
const ttlMs = opts?.ttlMs ?? DEFAULT_TTL_MS;
|
|
46
|
+
const token = randomUUID();
|
|
47
|
+
const expiresAt = this.now() + ttlMs;
|
|
48
|
+
this.leases.set(key, { token, expiresAt });
|
|
49
|
+
return { key, token, expiresAt };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async renew(handle: LockHandle, ttlMs: number): Promise<boolean> {
|
|
53
|
+
const lease = this.leases.get(handle.key);
|
|
54
|
+
// Only the current owner may renew; a lapsed-and-stolen lease fails.
|
|
55
|
+
if (!this.isLive(lease) || lease.token !== handle.token) {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
lease.expiresAt = this.now() + ttlMs;
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async release(handle: LockHandle): Promise<boolean> {
|
|
63
|
+
const lease = this.leases.get(handle.key);
|
|
64
|
+
if (!lease || lease.token !== handle.token) {
|
|
65
|
+
// Already gone, or we lost ownership (lapsed + restolen). The latter
|
|
66
|
+
// is the "stranded lease" signal.
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
this.leases.delete(handle.key);
|
|
70
|
+
return true;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async dispose(): Promise<void> {
|
|
74
|
+
this.leases.clear();
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Test helper: number of live leases currently held. */
|
|
78
|
+
liveCount(): number {
|
|
79
|
+
let n = 0;
|
|
80
|
+
for (const lease of this.leases.values()) {
|
|
81
|
+
if (this.isLive(lease)) n++;
|
|
82
|
+
}
|
|
83
|
+
return n;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LockBackend — pluggable mutual-exclusion primitive behind {@link DistributedLock}.
|
|
3
|
+
*
|
|
4
|
+
* Three concrete backends ship with the framework:
|
|
5
|
+
* - {@link InProcessLockBackend} — single-instance, in-memory. No infra.
|
|
6
|
+
* - {@link PostgresLeaseLockBackend} — lease-row table, pooler-safe. Default
|
|
7
|
+
* distributed backend (each acquire/renew/release is one short transaction,
|
|
8
|
+
* so it works behind pgbouncer `pool_mode = transaction`).
|
|
9
|
+
* - {@link AdvisoryLockBackend} — PostgreSQL session advisory locks. The
|
|
10
|
+
* historical default; opt-in only because it REQUIRES a session-pinned
|
|
11
|
+
* connection lane (breaks silently behind a transaction pooler — see
|
|
12
|
+
* docs/TICKETS_LOCK_POOLING_DATALOADER_2026-06-22.md, BUNSANE-1).
|
|
13
|
+
*
|
|
14
|
+
* The unifying model is a *lease*: a holder owns `key` until it releases it or
|
|
15
|
+
* `expiresAt` passes. Advisory locks have no TTL (they live for the session),
|
|
16
|
+
* so `ttlMs`/`renew` are best-effort: a backend that cannot honour a lease
|
|
17
|
+
* simply ignores `ttlMs` and omits `renew`.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Opaque proof of ownership returned by {@link LockBackend.acquire}. Must be
|
|
22
|
+
* passed back to `renew`/`release` so a backend can verify the caller still
|
|
23
|
+
* owns the lease (fencing) before mutating shared state.
|
|
24
|
+
*/
|
|
25
|
+
export interface LockHandle {
|
|
26
|
+
/** The logical lock key the caller asked for. */
|
|
27
|
+
key: string;
|
|
28
|
+
/**
|
|
29
|
+
* Unique-per-acquisition owner token. Lease backends compare-and-act on
|
|
30
|
+
* this so a holder whose lease already expired and was stolen cannot
|
|
31
|
+
* release/renew the new holder's lease.
|
|
32
|
+
*/
|
|
33
|
+
token: string;
|
|
34
|
+
/**
|
|
35
|
+
* Wall-clock ms (Date.now scale) at which the lease lapses, or `null` for
|
|
36
|
+
* session-bound backends (advisory) that have no TTL.
|
|
37
|
+
*/
|
|
38
|
+
expiresAt: number | null;
|
|
39
|
+
/**
|
|
40
|
+
* Backend-private scratch space (e.g. the bigint advisory key). Not part of
|
|
41
|
+
* the public contract — never read it outside the backend that set it.
|
|
42
|
+
*/
|
|
43
|
+
readonly _internal?: unknown;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface AcquireOptions {
|
|
47
|
+
/**
|
|
48
|
+
* Desired lease lifetime in ms. Lease backends use this as the TTL;
|
|
49
|
+
* session-bound backends ignore it. Renew before this elapses for work
|
|
50
|
+
* that outlives the lease.
|
|
51
|
+
*/
|
|
52
|
+
ttlMs?: number;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* A pluggable lock primitive. Implementations MUST be safe to call
|
|
57
|
+
* concurrently and MUST treat `acquire` as atomic: at most one live handle
|
|
58
|
+
* exists per `key` across every process sharing the backend's store.
|
|
59
|
+
*/
|
|
60
|
+
export interface LockBackend {
|
|
61
|
+
/** Stable identifier for logs/metrics (e.g. "postgres-lease"). */
|
|
62
|
+
readonly name: string;
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Try once to acquire `key`. Resolves to a {@link LockHandle} on success or
|
|
66
|
+
* `null` if another holder owns it. Never blocks/retries — the caller
|
|
67
|
+
* ({@link DistributedLock}) owns the wait loop.
|
|
68
|
+
*/
|
|
69
|
+
acquire(key: string, opts?: AcquireOptions): Promise<LockHandle | null>;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Extend an owned lease by `ttlMs`. Returns `false` if the handle no longer
|
|
73
|
+
* owns the lock (expired and stolen, or already released). Optional:
|
|
74
|
+
* session-bound backends omit it (their lock never lapses while held).
|
|
75
|
+
*/
|
|
76
|
+
renew?(handle: LockHandle, ttlMs: number): Promise<boolean>;
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Release an owned lock. Returns `true` if this handle released a lock it
|
|
80
|
+
* genuinely held, `false` if the lock was already gone / owned by someone
|
|
81
|
+
* else (a `false` here is the canonical "stranded/lost lease" signal —
|
|
82
|
+
* callers should surface it loudly, see BUNSANE-2).
|
|
83
|
+
*/
|
|
84
|
+
release(handle: LockHandle): Promise<boolean>;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Release any backend-held resources (pooled connection, redis client
|
|
88
|
+
* subscription). Called on shutdown. Idempotent.
|
|
89
|
+
*/
|
|
90
|
+
dispose?(): Promise<void>;
|
|
91
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PostgresLeaseLockBackend — pooler-safe distributed lock via a lease-row table.
|
|
3
|
+
*
|
|
4
|
+
* Each operation (acquire / renew / release) is a SINGLE autocommit statement,
|
|
5
|
+
* so it runs in exactly one transaction on one backend. That makes it correct
|
|
6
|
+
* behind pgbouncer `pool_mode = transaction` — unlike session advisory locks,
|
|
7
|
+
* which strand when lock and unlock land on different backends (BUNSANE-1). It
|
|
8
|
+
* also keys on the raw text `key`, so distinct keys never collide (no 32-bit
|
|
9
|
+
* hash folding — BUNSANE-4).
|
|
10
|
+
*
|
|
11
|
+
* Crash safety: a holder owns `key` until it deletes its row OR `expires_at`
|
|
12
|
+
* passes. A crashed holder's lease lapses after `ttlMs` and the next acquirer
|
|
13
|
+
* steals it. Long-running work must `renew` before the lease elapses (the
|
|
14
|
+
* DistributedLock/withLock layer drives a heartbeat).
|
|
15
|
+
*
|
|
16
|
+
* Fencing: `owner` is a per-acquisition random token. renew/release act only on
|
|
17
|
+
* `key = $key AND owner = $token`, so a holder whose lease expired and was
|
|
18
|
+
* stolen cannot renew or release the new holder's lease (returns false — the
|
|
19
|
+
* canonical lost-lease signal).
|
|
20
|
+
*
|
|
21
|
+
* The table is created lazily and idempotently on first use, so `withLock`
|
|
22
|
+
* works even without a full `App.init()` migration pass.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { randomUUID } from "crypto";
|
|
26
|
+
import type { SQL } from "bun";
|
|
27
|
+
import db from "../../../database";
|
|
28
|
+
import { logger } from "../../Logger";
|
|
29
|
+
import type { AcquireOptions, LockBackend, LockHandle } from "./LockBackend";
|
|
30
|
+
|
|
31
|
+
const loggerInstance = logger.child({ scope: "PostgresLeaseLockBackend" });
|
|
32
|
+
|
|
33
|
+
const TABLE = "bunsane_locks";
|
|
34
|
+
const DEFAULT_TTL_MS = 30_000;
|
|
35
|
+
|
|
36
|
+
export interface PostgresLeaseBackendConfig {
|
|
37
|
+
enableLogging: boolean;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export class PostgresLeaseLockBackend implements LockBackend {
|
|
41
|
+
readonly name = "postgres-lease";
|
|
42
|
+
private config: PostgresLeaseBackendConfig;
|
|
43
|
+
private readonly sql: SQL;
|
|
44
|
+
/** Runs the idempotent table create exactly once per backend instance. */
|
|
45
|
+
private tableReady: Promise<void> | null = null;
|
|
46
|
+
|
|
47
|
+
constructor(config: Partial<PostgresLeaseBackendConfig> = {}, sql: SQL = db) {
|
|
48
|
+
this.config = { enableLogging: config.enableLogging ?? false };
|
|
49
|
+
this.sql = sql;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
private ensureTable(): Promise<void> {
|
|
53
|
+
if (!this.tableReady) {
|
|
54
|
+
this.tableReady = this.sql
|
|
55
|
+
.unsafe(
|
|
56
|
+
`CREATE TABLE IF NOT EXISTS ${TABLE} (
|
|
57
|
+
key text PRIMARY KEY,
|
|
58
|
+
owner text NOT NULL,
|
|
59
|
+
expires_at timestamptz NOT NULL
|
|
60
|
+
)`
|
|
61
|
+
)
|
|
62
|
+
.then(() => undefined)
|
|
63
|
+
.catch((err) => {
|
|
64
|
+
// Reset so a transient failure (pool exhausted at boot)
|
|
65
|
+
// retries on the next call rather than caching the reject.
|
|
66
|
+
this.tableReady = null;
|
|
67
|
+
throw err;
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
return this.tableReady;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async acquire(key: string, opts?: AcquireOptions): Promise<LockHandle | null> {
|
|
74
|
+
const ttlMs = opts?.ttlMs ?? DEFAULT_TTL_MS;
|
|
75
|
+
const token = randomUUID();
|
|
76
|
+
try {
|
|
77
|
+
await this.ensureTable();
|
|
78
|
+
// Fresh insert OR steal of an expired lease both RETURN a row whose
|
|
79
|
+
// owner is us. A live foreign lease fails the conditional UPDATE and
|
|
80
|
+
// RETURNING yields zero rows.
|
|
81
|
+
const rows = await this.sql.unsafe(
|
|
82
|
+
`INSERT INTO ${TABLE} (key, owner, expires_at)
|
|
83
|
+
VALUES ($1, $2, now() + ($3::bigint * interval '1 millisecond'))
|
|
84
|
+
ON CONFLICT (key) DO UPDATE
|
|
85
|
+
SET owner = excluded.owner, expires_at = excluded.expires_at
|
|
86
|
+
WHERE ${TABLE}.expires_at < now()
|
|
87
|
+
RETURNING (owner = $2) AS acquired`,
|
|
88
|
+
[key, token, ttlMs]
|
|
89
|
+
);
|
|
90
|
+
const acquired = rows.length > 0 && rows[0]?.acquired === true;
|
|
91
|
+
if (!acquired) return null;
|
|
92
|
+
|
|
93
|
+
if (this.config.enableLogging) {
|
|
94
|
+
loggerInstance.debug(`Acquired lease ${key} (ttl ${ttlMs}ms)`);
|
|
95
|
+
}
|
|
96
|
+
return { key, token, expiresAt: Date.now() + ttlMs };
|
|
97
|
+
} catch (error) {
|
|
98
|
+
loggerInstance.error(
|
|
99
|
+
`Error acquiring lease ${key}: ${error instanceof Error ? error.message : String(error)}`
|
|
100
|
+
);
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async renew(handle: LockHandle, ttlMs: number): Promise<boolean> {
|
|
106
|
+
try {
|
|
107
|
+
const rows = await this.sql.unsafe(
|
|
108
|
+
`UPDATE ${TABLE}
|
|
109
|
+
SET expires_at = now() + ($3::bigint * interval '1 millisecond')
|
|
110
|
+
WHERE key = $1 AND owner = $2 AND expires_at > now()
|
|
111
|
+
RETURNING 1 AS renewed`,
|
|
112
|
+
[handle.key, handle.token, ttlMs]
|
|
113
|
+
);
|
|
114
|
+
return rows.length > 0;
|
|
115
|
+
} catch (error) {
|
|
116
|
+
loggerInstance.error(
|
|
117
|
+
`Error renewing lease ${handle.key}: ${error instanceof Error ? error.message : String(error)}`
|
|
118
|
+
);
|
|
119
|
+
return false;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async release(handle: LockHandle): Promise<boolean> {
|
|
124
|
+
try {
|
|
125
|
+
// Delete only our own row. A row stolen after our lease lapsed has a
|
|
126
|
+
// different owner → 0 rows → false (lost-lease signal).
|
|
127
|
+
const rows = await this.sql.unsafe(
|
|
128
|
+
`DELETE FROM ${TABLE}
|
|
129
|
+
WHERE key = $1 AND owner = $2
|
|
130
|
+
RETURNING 1 AS released`,
|
|
131
|
+
[handle.key, handle.token]
|
|
132
|
+
);
|
|
133
|
+
const released = rows.length > 0;
|
|
134
|
+
// A 0-row release (lease stolen / already gone) is surfaced loudly
|
|
135
|
+
// by the DistributedLock facade; keep the backend log at debug to
|
|
136
|
+
// avoid double-logging.
|
|
137
|
+
if (this.config.enableLogging) {
|
|
138
|
+
loggerInstance.debug(
|
|
139
|
+
released
|
|
140
|
+
? `Released lease ${handle.key}`
|
|
141
|
+
: `Lease release for ${handle.key} affected 0 rows (stolen or already released)`
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
return released;
|
|
145
|
+
} catch (error) {
|
|
146
|
+
loggerInstance.error(
|
|
147
|
+
`Error releasing lease ${handle.key}: ${error instanceof Error ? error.message : String(error)}`
|
|
148
|
+
);
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
async dispose(): Promise<void> {
|
|
154
|
+
// Stateless aside from the table-ready cache; nothing to release. Rows
|
|
155
|
+
// self-expire via TTL; an explicit releaseAll() at the DistributedLock
|
|
156
|
+
// layer deletes still-held leases on graceful shutdown.
|
|
157
|
+
this.tableReady = null;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lock backend selection + factory.
|
|
3
|
+
*
|
|
4
|
+
* Resolution order: explicit `config.kind` → `BUNSANE_LOCK_BACKEND` env →
|
|
5
|
+
* `'auto'`. `'auto'` resolves to the safest correct default for the deployment.
|
|
6
|
+
*
|
|
7
|
+
* `'auto'` resolves to `'postgres'` (pooler-safe lease): the default lock
|
|
8
|
+
* primitive is now correct behind pgbouncer transaction pooling. The old
|
|
9
|
+
* advisory primitive (silently broken behind a pooler — BUNSANE-1) is opt-in
|
|
10
|
+
* via `'advisory'`, and only safe on a session-pinned connection lane.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { logger } from "../../Logger";
|
|
14
|
+
import type { LockBackend } from "./LockBackend";
|
|
15
|
+
import { InProcessLockBackend } from "./InProcessLockBackend";
|
|
16
|
+
import { AdvisoryLockBackend } from "./AdvisoryLockBackend";
|
|
17
|
+
import { PostgresLeaseLockBackend } from "./PostgresLeaseLockBackend";
|
|
18
|
+
|
|
19
|
+
const loggerInstance = logger.child({ scope: "LockBackend" });
|
|
20
|
+
|
|
21
|
+
export type LockBackendKind =
|
|
22
|
+
| "auto"
|
|
23
|
+
| "in-process"
|
|
24
|
+
| "postgres"
|
|
25
|
+
| "redis"
|
|
26
|
+
| "advisory";
|
|
27
|
+
|
|
28
|
+
export interface LockBackendConfig {
|
|
29
|
+
/** Which backend to use. Default resolves via env then `'auto'`. */
|
|
30
|
+
kind?: LockBackendKind;
|
|
31
|
+
/** Advisory backend only: 32-bit namespace prefix for the advisory key. */
|
|
32
|
+
lockKeyPrefix?: number;
|
|
33
|
+
enableLogging?: boolean;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function resolveKind(kind: LockBackendKind | undefined): Exclude<LockBackendKind, "auto"> {
|
|
37
|
+
const requested =
|
|
38
|
+
kind ?? (process.env.BUNSANE_LOCK_BACKEND as LockBackendKind | undefined) ?? "auto";
|
|
39
|
+
|
|
40
|
+
if (requested === "auto") {
|
|
41
|
+
// Pooler-safe lease is the correct default behind PgBouncer transaction
|
|
42
|
+
// pooling. Single-instance deploys may opt down to 'in-process'.
|
|
43
|
+
return "postgres";
|
|
44
|
+
}
|
|
45
|
+
return requested;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function createLockBackend(config: LockBackendConfig = {}): LockBackend {
|
|
49
|
+
const kind = resolveKind(config.kind);
|
|
50
|
+
|
|
51
|
+
switch (kind) {
|
|
52
|
+
case "in-process":
|
|
53
|
+
return new InProcessLockBackend();
|
|
54
|
+
case "advisory":
|
|
55
|
+
return new AdvisoryLockBackend({
|
|
56
|
+
lockKeyPrefix: config.lockKeyPrefix,
|
|
57
|
+
enableLogging: config.enableLogging,
|
|
58
|
+
});
|
|
59
|
+
case "postgres":
|
|
60
|
+
return new PostgresLeaseLockBackend({
|
|
61
|
+
enableLogging: config.enableLogging,
|
|
62
|
+
});
|
|
63
|
+
case "redis":
|
|
64
|
+
// Implemented in Phase 4.
|
|
65
|
+
loggerInstance.error(
|
|
66
|
+
`Lock backend 'redis' is not implemented yet; falling back to postgres-lease.`
|
|
67
|
+
);
|
|
68
|
+
return new PostgresLeaseLockBackend({
|
|
69
|
+
enableLogging: config.enableLogging,
|
|
70
|
+
});
|
|
71
|
+
default: {
|
|
72
|
+
const exhaustive: never = kind;
|
|
73
|
+
throw new Error(`Unknown lock backend: ${String(exhaustive)}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export type { LockBackend, LockHandle, AcquireOptions } from "./LockBackend";
|
|
79
|
+
export { InProcessLockBackend } from "./InProcessLockBackend";
|
|
80
|
+
export { AdvisoryLockBackend, UnsafeAdvisoryPoolingError } from "./AdvisoryLockBackend";
|
|
81
|
+
export { PostgresLeaseLockBackend } from "./PostgresLeaseLockBackend";
|