@fullstackhouse/open-mercato-durable-work 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/om/config.js
CHANGED
|
@@ -18,6 +18,9 @@ function readConfig(env = process.env) {
|
|
|
18
18
|
redisUrl: env.DURABLE_WORK_REDIS_URL ?? env.QUEUE_REDIS_URL ?? env.REDIS_URL ?? null,
|
|
19
19
|
databaseUrl: env.DATABASE_URL ?? null,
|
|
20
20
|
pgBossSchema: env.DURABLE_WORK_PGBOSS_SCHEMA ?? "durable_work_boss",
|
|
21
|
+
// Deliberately far below pg-boss's own default: this pool is opened per process that
|
|
22
|
+
// touches the transport, and an in-process worker means the web process is one of them.
|
|
23
|
+
pgBossMaxConnections: num(env.DURABLE_WORK_PGBOSS_MAX, 4),
|
|
21
24
|
tickMs: num(env.DURABLE_WORK_TICK_MS, 15e3),
|
|
22
25
|
drainTimeoutMs: num(env.DURABLE_WORK_DRAIN_TIMEOUT_MS, 3e4),
|
|
23
26
|
reconcilerGraceMs: num(env.DURABLE_WORK_GRACE_MS, 2e4),
|
|
@@ -38,7 +41,11 @@ function createTransport(config, deps = {}) {
|
|
|
38
41
|
}
|
|
39
42
|
case "pgboss": {
|
|
40
43
|
if (!config.databaseUrl) throw new Error("DURABLE_WORK_TRANSPORT=pgboss requires DATABASE_URL.");
|
|
41
|
-
return new PgBossTransport({
|
|
44
|
+
return new PgBossTransport({
|
|
45
|
+
connectionString: config.databaseUrl,
|
|
46
|
+
schema: config.pgBossSchema,
|
|
47
|
+
max: config.pgBossMaxConnections
|
|
48
|
+
});
|
|
42
49
|
}
|
|
43
50
|
}
|
|
44
51
|
}
|
package/dist/om/config.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/om/config.ts"],
|
|
4
|
-
"sourcesContent": ["// How a host configures the mechanism: environment variables in, a transport out.\n\nimport { BullMQTransport } from '../transport/bullmq'\nimport { MemoryTransport } from '../transport/memory'\nimport { PgBossTransport } from '../transport/pgboss'\nimport type { TransportAdapter, TransportName } from '../transport/types'\n\nexport type DurableWorkConfig = {\n transport: TransportName\n redisUrl: string | null\n databaseUrl: string | null\n pgBossSchema: string\n tickMs: number\n drainTimeoutMs: number\n reconcilerGraceMs: number\n /** Hosts the worker inside the app process. Dev and ephemeral tests only. */\n inProcessWorker: boolean\n}\n\nconst num = (value: string | undefined, fallback: number): number => {\n const parsed = Number(value)\n return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback\n}\n\nconst bool = (value: string | undefined): boolean => value === '1' || value?.toLowerCase() === 'true'\n\nexport function readConfig(env: NodeJS.ProcessEnv = process.env): DurableWorkConfig {\n const raw = (env.DURABLE_WORK_TRANSPORT ?? 'pgboss').trim().toLowerCase()\n if (raw !== 'memory' && raw !== 'bullmq' && raw !== 'pgboss') {\n throw new Error(`DURABLE_WORK_TRANSPORT must be memory | bullmq | pgboss, got ${JSON.stringify(raw)}`)\n }\n return {\n transport: raw,\n // Falls back to the queue module's Redis, because an app that already runs one should not\n // have to configure a second.\n redisUrl: env.DURABLE_WORK_REDIS_URL ?? env.QUEUE_REDIS_URL ?? env.REDIS_URL ?? null,\n databaseUrl: env.DATABASE_URL ?? null,\n pgBossSchema: env.DURABLE_WORK_PGBOSS_SCHEMA ?? 'durable_work_boss',\n tickMs: num(env.DURABLE_WORK_TICK_MS, 15_000),\n drainTimeoutMs: num(env.DURABLE_WORK_DRAIN_TIMEOUT_MS, 30_000),\n reconcilerGraceMs: num(env.DURABLE_WORK_GRACE_MS, 20_000),\n inProcessWorker: bool(env.DURABLE_WORK_INPROCESS_WORKER),\n }\n}\n\nexport function createTransport(config: DurableWorkConfig, deps: { redisConnection?: unknown } = {}): TransportAdapter {\n switch (config.transport) {\n case 'memory':\n // Nothing survives the process, so this is a development convenience and is refused in\n // production rather than quietly losing every job on the next deploy.\n if (process.env.NODE_ENV === 'production') {\n throw new Error('DURABLE_WORK_TRANSPORT=memory keeps jobs in process memory and cannot be used in production.')\n }\n return new MemoryTransport()\n case 'bullmq': {\n const connection = deps.redisConnection ?? config.redisUrl\n if (!connection) throw new Error('DURABLE_WORK_TRANSPORT=bullmq requires DURABLE_WORK_REDIS_URL (or QUEUE_REDIS_URL).')\n return new BullMQTransport({ connection })\n }\n case 'pgboss': {\n if (!config.databaseUrl) throw new Error('DURABLE_WORK_TRANSPORT=pgboss requires DATABASE_URL.')\n return new PgBossTransport({
|
|
5
|
-
"mappings": "AAEA,SAAS,uBAAuB;AAChC,SAAS,uBAAuB;AAChC,SAAS,uBAAuB;
|
|
4
|
+
"sourcesContent": ["// How a host configures the mechanism: environment variables in, a transport out.\n\nimport { BullMQTransport } from '../transport/bullmq'\nimport { MemoryTransport } from '../transport/memory'\nimport { PgBossTransport } from '../transport/pgboss'\nimport type { TransportAdapter, TransportName } from '../transport/types'\n\nexport type DurableWorkConfig = {\n transport: TransportName\n redisUrl: string | null\n databaseUrl: string | null\n pgBossSchema: string\n /** Cap on pg-boss's own connection pool. Small on purpose \u2014 see the transport option. */\n pgBossMaxConnections: number\n tickMs: number\n drainTimeoutMs: number\n reconcilerGraceMs: number\n /** Hosts the worker inside the app process. Dev and ephemeral tests only. */\n inProcessWorker: boolean\n}\n\nconst num = (value: string | undefined, fallback: number): number => {\n const parsed = Number(value)\n return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback\n}\n\nconst bool = (value: string | undefined): boolean => value === '1' || value?.toLowerCase() === 'true'\n\nexport function readConfig(env: NodeJS.ProcessEnv = process.env): DurableWorkConfig {\n const raw = (env.DURABLE_WORK_TRANSPORT ?? 'pgboss').trim().toLowerCase()\n if (raw !== 'memory' && raw !== 'bullmq' && raw !== 'pgboss') {\n throw new Error(`DURABLE_WORK_TRANSPORT must be memory | bullmq | pgboss, got ${JSON.stringify(raw)}`)\n }\n return {\n transport: raw,\n // Falls back to the queue module's Redis, because an app that already runs one should not\n // have to configure a second.\n redisUrl: env.DURABLE_WORK_REDIS_URL ?? env.QUEUE_REDIS_URL ?? env.REDIS_URL ?? null,\n databaseUrl: env.DATABASE_URL ?? null,\n pgBossSchema: env.DURABLE_WORK_PGBOSS_SCHEMA ?? 'durable_work_boss',\n // Deliberately far below pg-boss's own default: this pool is opened per process that\n // touches the transport, and an in-process worker means the web process is one of them.\n pgBossMaxConnections: num(env.DURABLE_WORK_PGBOSS_MAX, 4),\n tickMs: num(env.DURABLE_WORK_TICK_MS, 15_000),\n drainTimeoutMs: num(env.DURABLE_WORK_DRAIN_TIMEOUT_MS, 30_000),\n reconcilerGraceMs: num(env.DURABLE_WORK_GRACE_MS, 20_000),\n inProcessWorker: bool(env.DURABLE_WORK_INPROCESS_WORKER),\n }\n}\n\nexport function createTransport(config: DurableWorkConfig, deps: { redisConnection?: unknown } = {}): TransportAdapter {\n switch (config.transport) {\n case 'memory':\n // Nothing survives the process, so this is a development convenience and is refused in\n // production rather than quietly losing every job on the next deploy.\n if (process.env.NODE_ENV === 'production') {\n throw new Error('DURABLE_WORK_TRANSPORT=memory keeps jobs in process memory and cannot be used in production.')\n }\n return new MemoryTransport()\n case 'bullmq': {\n const connection = deps.redisConnection ?? config.redisUrl\n if (!connection) throw new Error('DURABLE_WORK_TRANSPORT=bullmq requires DURABLE_WORK_REDIS_URL (or QUEUE_REDIS_URL).')\n return new BullMQTransport({ connection })\n }\n case 'pgboss': {\n if (!config.databaseUrl) throw new Error('DURABLE_WORK_TRANSPORT=pgboss requires DATABASE_URL.')\n return new PgBossTransport({\n connectionString: config.databaseUrl,\n schema: config.pgBossSchema,\n max: config.pgBossMaxConnections,\n })\n }\n }\n}\n"],
|
|
5
|
+
"mappings": "AAEA,SAAS,uBAAuB;AAChC,SAAS,uBAAuB;AAChC,SAAS,uBAAuB;AAiBhC,MAAM,MAAM,CAAC,OAA2B,aAA6B;AACnE,QAAM,SAAS,OAAO,KAAK;AAC3B,SAAO,OAAO,SAAS,MAAM,KAAK,SAAS,IAAI,SAAS;AAC1D;AAEA,MAAM,OAAO,CAAC,UAAuC,UAAU,OAAO,OAAO,YAAY,MAAM;AAExF,SAAS,WAAW,MAAyB,QAAQ,KAAwB;AAClF,QAAM,OAAO,IAAI,0BAA0B,UAAU,KAAK,EAAE,YAAY;AACxE,MAAI,QAAQ,YAAY,QAAQ,YAAY,QAAQ,UAAU;AAC5D,UAAM,IAAI,MAAM,gEAAgE,KAAK,UAAU,GAAG,CAAC,EAAE;AAAA,EACvG;AACA,SAAO;AAAA,IACL,WAAW;AAAA;AAAA;AAAA,IAGX,UAAU,IAAI,0BAA0B,IAAI,mBAAmB,IAAI,aAAa;AAAA,IAChF,aAAa,IAAI,gBAAgB;AAAA,IACjC,cAAc,IAAI,8BAA8B;AAAA;AAAA;AAAA,IAGhD,sBAAsB,IAAI,IAAI,yBAAyB,CAAC;AAAA,IACxD,QAAQ,IAAI,IAAI,sBAAsB,IAAM;AAAA,IAC5C,gBAAgB,IAAI,IAAI,+BAA+B,GAAM;AAAA,IAC7D,mBAAmB,IAAI,IAAI,uBAAuB,GAAM;AAAA,IACxD,iBAAiB,KAAK,IAAI,6BAA6B;AAAA,EACzD;AACF;AAEO,SAAS,gBAAgB,QAA2B,OAAsC,CAAC,GAAqB;AACrH,UAAQ,OAAO,WAAW;AAAA,IACxB,KAAK;AAGH,UAAI,QAAQ,IAAI,aAAa,cAAc;AACzC,cAAM,IAAI,MAAM,8FAA8F;AAAA,MAChH;AACA,aAAO,IAAI,gBAAgB;AAAA,IAC7B,KAAK,UAAU;AACb,YAAM,aAAa,KAAK,mBAAmB,OAAO;AAClD,UAAI,CAAC,WAAY,OAAM,IAAI,MAAM,qFAAqF;AACtH,aAAO,IAAI,gBAAgB,EAAE,WAAW,CAAC;AAAA,IAC3C;AAAA,IACA,KAAK,UAAU;AACb,UAAI,CAAC,OAAO,YAAa,OAAM,IAAI,MAAM,sDAAsD;AAC/F,aAAO,IAAI,gBAAgB;AAAA,QACzB,kBAAkB,OAAO;AAAA,QACzB,QAAQ,OAAO;AAAA,QACf,KAAK,OAAO;AAAA,MACd,CAAC;AAAA,IACH;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/transport/pgboss.js
CHANGED
|
@@ -40,7 +40,11 @@ class PgBossTransport {
|
|
|
40
40
|
if (!this.starting) {
|
|
41
41
|
this.starting = (async () => {
|
|
42
42
|
const { PgBoss } = await pgboss();
|
|
43
|
-
const instance = new PgBoss({
|
|
43
|
+
const instance = new PgBoss({
|
|
44
|
+
connectionString: this.options.connectionString,
|
|
45
|
+
schema: this.options.schema ?? "durable_work_boss",
|
|
46
|
+
...this.options.max ? { max: this.options.max } : {}
|
|
47
|
+
});
|
|
44
48
|
await instance.start();
|
|
45
49
|
this.boss = instance;
|
|
46
50
|
return instance;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/transport/pgboss.ts"],
|
|
4
|
-
"sourcesContent": ["// The pg-boss adapter: production without Redis, and the only adapter that can enqueue a\n// delivery inside the caller's transaction.\n//\n// That one capability is why it exists. `send(..., { db })` composes its statements on a\n// client the caller supplies, so a domain row, its job row and its delivery all commit or all\n// roll back. With any other transport there is a window between commit and enqueue where a\n// crash leaves a job nobody will ever deliver \u2014 the reconciler closes it, but closing it after\n// fifteen minutes is not the same as never opening it.\n//\n// pg-boss is a peer dependency and is imported lazily.\n\nimport { deliveryId } from '../core/ids'\nimport type { Delivery, SqlExecutor } from '../core/types'\nimport type {\n BindOptions,\n BoundWorker,\n DeliveryHandler,\n DeliveryState,\n EnqueueOptions,\n TransportAdapter,\n} from './types'\n\n// pg-boss's surface, described structurally rather than imported.\n//\n// `typeof import('pg-boss')` is more faithful and is the wrong tool here. This package's\n// `exports` map points its `types` condition at these sources, so a host typechecks this file \u2014\n// and pg-boss is an *optional* peer. A host running the BullMQ transport, which therefore never\n// installs pg-boss, failed its own typecheck on a file it never loads. Found in a real adopter,\n// not in this repo, because here the dependency is always present.\n//\n// Only what this adapter calls is described. pg-boss 12 exports the class by name, not as a\n// default. The runtime import below is unchanged and still a literal, so bundlers can see it.\ntype PgBossSendOptions = {\n singletonKey?: string\n singletonSeconds?: number\n startAfter?: number\n retryLimit?: number\n retryDelay?: number\n retryBackoff?: boolean\n retryDelayMax?: number\n db?: unknown\n}\n\ntype PgBossJob = { id: string; data: Delivery; state?: string; signal?: AbortSignal }\n\ntype PgBossInstance = {\n start(): Promise<unknown>\n stop(options?: { graceful?: boolean; close?: boolean; timeout?: number }): Promise<unknown>\n createQueue(name: string, options?: { expireInSeconds?: number }): Promise<unknown>\n send(name: string, data: object, options?: PgBossSendOptions): Promise<string | null>\n deleteJob(name: string, id: string): Promise<unknown>\n getJobById(name: string, id: string): Promise<{ state?: string } | null>\n work(name: string, options: { batchSize?: number }, handler: (jobs: PgBossJob[]) => Promise<unknown>): Promise<string>\n offWork(name: string, options?: { id?: string }): Promise<unknown>\n}\n\ntype PgBossModule = { PgBoss: new (options: { connectionString: string; schema?: string }) => PgBossInstance }\n\nexport type PgBossTransportOptions = {\n connectionString: string\n /** Keeps pg-boss's own tables out of `public`, so they are obviously not the app's. */\n schema?: string\n /** Reuse an already-started instance instead of owning its lifecycle. */\n instance?: PgBossInstance\n}\n\nlet cached: PgBossModule | null = null\nasync function pgboss(): Promise<PgBossModule> {\n if (cached) return cached\n try {\n // `@ts-expect-error` is the wrong tool here: in this repo the dependency IS installed, so\n // there is no error to expect and the build would fail on the assertion itself. The error\n // exists only in a host that never installed this optional peer \u2014 the case being suppressed.\n // eslint-disable-next-line @typescript-eslint/ban-ts-comment\n // @ts-ignore -- optional peer: absent in a host that runs another transport, and this file\n // must still typecheck there. The structural types above are why nothing else needs it.\n cached = (await import('pg-boss')) as unknown as PgBossModule\n return cached\n } catch (error) {\n throw new Error(\n 'The pgboss transport requires the optional peer dependency `pg-boss`. Install it, or use DURABLE_WORK_TRANSPORT=bullmq.',\n { cause: error },\n )\n }\n}\n\n/**\n * Adapts a `SqlExecutor` to the shape pg-boss expects from a caller-supplied client.\n *\n * pg-boss only ever calls `executeSql`, so the whole surface is one method. Passing our own\n * executor through \u2014 rather than requiring a raw `pg` client \u2014 is what lets the caller's\n * transaction be a MikroORM one, a node-postgres one, or the harness's, without any of them\n * knowing about the others.\n */\nfunction asDb(tx: SqlExecutor) {\n return {\n async executeSql(text: string, values: unknown[]) {\n const result = await tx.query(text, values)\n return { rows: result.rows as Record<string, unknown>[], rowCount: result.rowCount }\n },\n }\n}\n\n/** A tick, in the same shape as any other delivery. */\nconst tickDelivery = (id: string): Delivery => ({ jobId: id, seq: 0, redrives: 0 })\n\nexport class PgBossTransport implements TransportAdapter {\n readonly name = 'pgboss' as const\n readonly supportsTransactionalEnqueue = true\n\n private boss: PgBossInstance | null = null\n private starting: Promise<PgBossInstance> | null = null\n private readonly ownsInstance: boolean\n private readonly ensuredQueues = new Set<string>()\n private readonly workerIds: Array<{ queue: string; id: string }> = []\n private readonly ticks: NodeJS.Timeout[] = []\n private readonly shutdown = new AbortController()\n\n constructor(private readonly options: PgBossTransportOptions) {\n this.boss = options.instance ?? null\n this.ownsInstance = !options.instance\n }\n\n private async ready(): Promise<PgBossInstance> {\n if (this.boss) return this.boss\n if (!this.starting) {\n this.starting = (async () => {\n const { PgBoss } = await pgboss()\n const instance = new PgBoss({ connectionString: this.options.connectionString, schema: this.options.schema ?? 'durable_work_boss' })\n await instance.start()\n this.boss = instance\n return instance\n })()\n }\n return this.starting\n }\n\n /** pg-boss 10+ requires a queue to exist before anything is sent to it. `expireInSeconds`\n * belongs to the queue, not to the worker: it is how long a delivery may stay active before\n * pg-boss reclaims it, so it must exceed a whole slice or work that is still running gets\n * handed to a second worker \u2014 which the lease then refuses, wasting the slice. */\n private async ensureQueue(name: string, expireInSeconds?: number): Promise<PgBossInstance> {\n const boss = await this.ready()\n if (this.ensuredQueues.has(name)) return boss\n await boss.createQueue(name, expireInSeconds ? { expireInSeconds } : undefined)\n this.ensuredQueues.add(name)\n return boss\n }\n\n async enqueue(queue: string, delivery: Delivery, opts: EnqueueOptions): Promise<{ transportJobId: string }> {\n const boss = await this.ensureQueue(queue)\n const key = deliveryId(delivery)\n const sent = await boss.send(queue, delivery as unknown as object, {\n // pg-boss job ids are uuids, so the delivery identity travels as the singleton key \u2014\n // which is also what makes a re-enqueue of the same delivery a no-op.\n singletonKey: key,\n startAfter: opts.delayMs && opts.delayMs > 0 ? Math.ceil(opts.delayMs / 1000) : undefined,\n retryLimit: opts.retry.attempts,\n retryDelay: Math.max(1, Math.round(opts.retry.backoff.delayMs / 1000)),\n // pg-boss rejects a max delay unless backoff is on, so the cap travels only with it.\n ...(opts.retry.backoff.type === 'exponential'\n ? { retryBackoff: true, retryDelayMax: Math.max(1, Math.round(opts.retry.backoff.maxDelayMs / 1000)) }\n : { retryBackoff: false }),\n ...(opts.tx ? { db: asDb(opts.tx) } : {}),\n })\n // `send` returns null when the singleton key collapsed this into an existing job. That is\n // the intended outcome, not a failure: the delivery is already scheduled.\n return { transportJobId: sent ?? key }\n }\n\n async remove(queue: string, transportJobId: string): Promise<void> {\n const boss = await this.ready()\n await boss.deleteJob(queue, transportJobId).catch(() => undefined)\n }\n\n async getState(queue: string, transportJobId: string): Promise<DeliveryState> {\n const boss = await this.ready()\n const job = await boss.getJobById(queue, transportJobId).catch(() => null)\n if (!job) return 'unknown'\n switch (job.state) {\n case 'created':\n case 'retry':\n return 'waiting'\n case 'active':\n return 'active'\n case 'completed':\n return 'completed'\n case 'cancelled':\n case 'failed':\n return 'failed'\n default:\n return 'unknown'\n }\n }\n\n async upsertTick(opts: { id: string; queue: string; everyMs: number }): Promise<void> {\n await this.ensureQueue(opts.queue)\n // pg-boss's own scheduler is cron-based, so its finest granularity is a minute \u2014 too\n // coarse for a repair loop. A per-process timer with a singleton key gives the cadence we\n // need and still collapses the fleet's ticks into one job per window. The trade-off is\n // stated rather than hidden: with zero workers up there is no tick, exactly as with a\n // broker-owned schedule that nobody polls.\n const everySeconds = Math.max(1, Math.round(opts.everyMs / 1000))\n const fire = async () => {\n if (this.shutdown.signal.aborted) return\n const boss = await this.ready()\n // Delivery-shaped, like every other payload on every adapter: a tick is a delivery\n // whose handler happens to ignore it, not a second kind of message.\n await boss\n .send(opts.queue, tickDelivery(opts.id), { singletonKey: opts.id, singletonSeconds: everySeconds })\n .catch(() => undefined)\n }\n void fire()\n const timer = setInterval(() => void fire(), opts.everyMs)\n timer.unref?.()\n this.ticks.push(timer)\n }\n\n async bind(queue: string, handler: DeliveryHandler, opts: BindOptions): Promise<BoundWorker> {\n const boss = await this.ensureQueue(queue, Math.ceil(opts.activeTimeoutMs / 1000))\n\n const workerId = await boss.work(\n queue,\n { batchSize: opts.concurrency },\n async (jobs: PgBossJob[]) => {\n for (const job of jobs) {\n // Whatever arrived is handed on unexamined. An adapter that inspects payloads\n // decides what counts as a real delivery, and this one used to skip anything\n // without a `jobId` \u2014 which silently swallowed every reconciler tick.\n const delivery = job.data\n\n const combined = new AbortController()\n const relay = () => combined.abort()\n job.signal?.addEventListener('abort', relay, { once: true })\n this.shutdown.signal.addEventListener('abort', relay, { once: true })\n\n try {\n await handler(delivery, {\n transportJobId: job.id,\n // pg-boss does not expose the attempt on the job, so the adapter reports the\n // first attempt and lets its own retry policy carry the rest. The consequence is\n // narrow: the delay written to `next_run_at` is the base rather than a backed-off\n // one, and pg-boss's own `retryBackoff` still spaces the real deliveries.\n attempt: 1,\n maxAttempts: 1,\n signal: combined.signal,\n handBack: async (next, handBackOpts) => {\n // No native hand-back: send the next delivery and let this one complete. The\n // row is already at `seq + 1`, so the new key cannot collide with this job.\n await this.enqueue(queue, next, {\n delayMs: handBackOpts?.delayMs,\n retry: { attempts: 1, backoff: { type: 'fixed', delayMs: 0, maxDelayMs: 0 } },\n })\n },\n })\n } catch (error) {\n if ((error as { name?: string })?.name === 'NoFurtherAttempts') continue // settled; no retry wanted\n throw error\n } finally {\n job.signal?.removeEventListener('abort', relay)\n this.shutdown.signal.removeEventListener('abort', relay)\n }\n }\n },\n )\n\n this.workerIds.push({ queue, id: workerId })\n return {\n queue,\n close: async () => {\n const instance = await this.ready()\n await instance.offWork(queue, { id: workerId }).catch(() => undefined)\n },\n }\n }\n\n async close(opts: { timeoutMs?: number } = {}): Promise<void> {\n this.shutdown.abort()\n for (const timer of this.ticks) clearInterval(timer)\n this.ticks.length = 0\n if (!this.boss) return\n if (!this.ownsInstance) return\n await this.boss.stop({ graceful: true, close: true, timeout: opts.timeoutMs ?? 30_000 }).catch(() => undefined)\n this.boss = null\n this.starting = null\n }\n}\n"],
|
|
5
|
-
"mappings": "AAWA,SAAS,kBAAkB;
|
|
4
|
+
"sourcesContent": ["// The pg-boss adapter: production without Redis, and the only adapter that can enqueue a\n// delivery inside the caller's transaction.\n//\n// That one capability is why it exists. `send(..., { db })` composes its statements on a\n// client the caller supplies, so a domain row, its job row and its delivery all commit or all\n// roll back. With any other transport there is a window between commit and enqueue where a\n// crash leaves a job nobody will ever deliver \u2014 the reconciler closes it, but closing it after\n// fifteen minutes is not the same as never opening it.\n//\n// pg-boss is a peer dependency and is imported lazily.\n\nimport { deliveryId } from '../core/ids'\nimport type { Delivery, SqlExecutor } from '../core/types'\nimport type {\n BindOptions,\n BoundWorker,\n DeliveryHandler,\n DeliveryState,\n EnqueueOptions,\n TransportAdapter,\n} from './types'\n\n// pg-boss's surface, described structurally rather than imported.\n//\n// `typeof import('pg-boss')` is more faithful and is the wrong tool here. This package's\n// `exports` map points its `types` condition at these sources, so a host typechecks this file \u2014\n// and pg-boss is an *optional* peer. A host running the BullMQ transport, which therefore never\n// installs pg-boss, failed its own typecheck on a file it never loads. Found in a real adopter,\n// not in this repo, because here the dependency is always present.\n//\n// Only what this adapter calls is described. pg-boss 12 exports the class by name, not as a\n// default. The runtime import below is unchanged and still a literal, so bundlers can see it.\ntype PgBossSendOptions = {\n singletonKey?: string\n singletonSeconds?: number\n startAfter?: number\n retryLimit?: number\n retryDelay?: number\n retryBackoff?: boolean\n retryDelayMax?: number\n db?: unknown\n}\n\ntype PgBossJob = { id: string; data: Delivery; state?: string; signal?: AbortSignal }\n\ntype PgBossInstance = {\n start(): Promise<unknown>\n stop(options?: { graceful?: boolean; close?: boolean; timeout?: number }): Promise<unknown>\n createQueue(name: string, options?: { expireInSeconds?: number }): Promise<unknown>\n send(name: string, data: object, options?: PgBossSendOptions): Promise<string | null>\n deleteJob(name: string, id: string): Promise<unknown>\n getJobById(name: string, id: string): Promise<{ state?: string } | null>\n work(name: string, options: { batchSize?: number }, handler: (jobs: PgBossJob[]) => Promise<unknown>): Promise<string>\n offWork(name: string, options?: { id?: string }): Promise<unknown>\n}\n\ntype PgBossModule = { PgBoss: new (options: { connectionString: string; schema?: string; max?: number }) => PgBossInstance }\n\nexport type PgBossTransportOptions = {\n connectionString: string\n /** Keeps pg-boss's own tables out of `public`, so they are obviously not the app's. */\n schema?: string\n /** Reuse an already-started instance instead of owning its lifecycle. */\n instance?: PgBossInstance\n /**\n * Cap on pg-boss's own connection pool.\n *\n * It matters more than it looks. This transport opens a pool *besides* the app's \u2014 the\n * mechanism's own SQL rides the host's EntityManager, but pg-boss does not \u2014 and a host that\n * runs the worker in-process gets one per process that touches the transport: the web process\n * and any spawned queue worker. Multiplied by replicas during a rolling deploy, pg-boss's own\n * default is enough to eat a Postgres `max_connections` budget that was sized without it.\n */\n max?: number\n}\n\nlet cached: PgBossModule | null = null\nasync function pgboss(): Promise<PgBossModule> {\n if (cached) return cached\n try {\n // `@ts-expect-error` is the wrong tool here: in this repo the dependency IS installed, so\n // there is no error to expect and the build would fail on the assertion itself. The error\n // exists only in a host that never installed this optional peer \u2014 the case being suppressed.\n // eslint-disable-next-line @typescript-eslint/ban-ts-comment\n // @ts-ignore -- optional peer: absent in a host that runs another transport, and this file\n // must still typecheck there. The structural types above are why nothing else needs it.\n cached = (await import('pg-boss')) as unknown as PgBossModule\n return cached\n } catch (error) {\n throw new Error(\n 'The pgboss transport requires the optional peer dependency `pg-boss`. Install it, or use DURABLE_WORK_TRANSPORT=bullmq.',\n { cause: error },\n )\n }\n}\n\n/**\n * Adapts a `SqlExecutor` to the shape pg-boss expects from a caller-supplied client.\n *\n * pg-boss only ever calls `executeSql`, so the whole surface is one method. Passing our own\n * executor through \u2014 rather than requiring a raw `pg` client \u2014 is what lets the caller's\n * transaction be a MikroORM one, a node-postgres one, or the harness's, without any of them\n * knowing about the others.\n */\nfunction asDb(tx: SqlExecutor) {\n return {\n async executeSql(text: string, values: unknown[]) {\n const result = await tx.query(text, values)\n return { rows: result.rows as Record<string, unknown>[], rowCount: result.rowCount }\n },\n }\n}\n\n/** A tick, in the same shape as any other delivery. */\nconst tickDelivery = (id: string): Delivery => ({ jobId: id, seq: 0, redrives: 0 })\n\nexport class PgBossTransport implements TransportAdapter {\n readonly name = 'pgboss' as const\n readonly supportsTransactionalEnqueue = true\n\n private boss: PgBossInstance | null = null\n private starting: Promise<PgBossInstance> | null = null\n private readonly ownsInstance: boolean\n private readonly ensuredQueues = new Set<string>()\n private readonly workerIds: Array<{ queue: string; id: string }> = []\n private readonly ticks: NodeJS.Timeout[] = []\n private readonly shutdown = new AbortController()\n\n constructor(private readonly options: PgBossTransportOptions) {\n this.boss = options.instance ?? null\n this.ownsInstance = !options.instance\n }\n\n private async ready(): Promise<PgBossInstance> {\n if (this.boss) return this.boss\n if (!this.starting) {\n this.starting = (async () => {\n const { PgBoss } = await pgboss()\n const instance = new PgBoss({\n connectionString: this.options.connectionString,\n schema: this.options.schema ?? 'durable_work_boss',\n ...(this.options.max ? { max: this.options.max } : {}),\n })\n await instance.start()\n this.boss = instance\n return instance\n })()\n }\n return this.starting\n }\n\n /** pg-boss 10+ requires a queue to exist before anything is sent to it. `expireInSeconds`\n * belongs to the queue, not to the worker: it is how long a delivery may stay active before\n * pg-boss reclaims it, so it must exceed a whole slice or work that is still running gets\n * handed to a second worker \u2014 which the lease then refuses, wasting the slice. */\n private async ensureQueue(name: string, expireInSeconds?: number): Promise<PgBossInstance> {\n const boss = await this.ready()\n if (this.ensuredQueues.has(name)) return boss\n await boss.createQueue(name, expireInSeconds ? { expireInSeconds } : undefined)\n this.ensuredQueues.add(name)\n return boss\n }\n\n async enqueue(queue: string, delivery: Delivery, opts: EnqueueOptions): Promise<{ transportJobId: string }> {\n const boss = await this.ensureQueue(queue)\n const key = deliveryId(delivery)\n const sent = await boss.send(queue, delivery as unknown as object, {\n // pg-boss job ids are uuids, so the delivery identity travels as the singleton key \u2014\n // which is also what makes a re-enqueue of the same delivery a no-op.\n singletonKey: key,\n startAfter: opts.delayMs && opts.delayMs > 0 ? Math.ceil(opts.delayMs / 1000) : undefined,\n retryLimit: opts.retry.attempts,\n retryDelay: Math.max(1, Math.round(opts.retry.backoff.delayMs / 1000)),\n // pg-boss rejects a max delay unless backoff is on, so the cap travels only with it.\n ...(opts.retry.backoff.type === 'exponential'\n ? { retryBackoff: true, retryDelayMax: Math.max(1, Math.round(opts.retry.backoff.maxDelayMs / 1000)) }\n : { retryBackoff: false }),\n ...(opts.tx ? { db: asDb(opts.tx) } : {}),\n })\n // `send` returns null when the singleton key collapsed this into an existing job. That is\n // the intended outcome, not a failure: the delivery is already scheduled.\n return { transportJobId: sent ?? key }\n }\n\n async remove(queue: string, transportJobId: string): Promise<void> {\n const boss = await this.ready()\n await boss.deleteJob(queue, transportJobId).catch(() => undefined)\n }\n\n async getState(queue: string, transportJobId: string): Promise<DeliveryState> {\n const boss = await this.ready()\n const job = await boss.getJobById(queue, transportJobId).catch(() => null)\n if (!job) return 'unknown'\n switch (job.state) {\n case 'created':\n case 'retry':\n return 'waiting'\n case 'active':\n return 'active'\n case 'completed':\n return 'completed'\n case 'cancelled':\n case 'failed':\n return 'failed'\n default:\n return 'unknown'\n }\n }\n\n async upsertTick(opts: { id: string; queue: string; everyMs: number }): Promise<void> {\n await this.ensureQueue(opts.queue)\n // pg-boss's own scheduler is cron-based, so its finest granularity is a minute \u2014 too\n // coarse for a repair loop. A per-process timer with a singleton key gives the cadence we\n // need and still collapses the fleet's ticks into one job per window. The trade-off is\n // stated rather than hidden: with zero workers up there is no tick, exactly as with a\n // broker-owned schedule that nobody polls.\n const everySeconds = Math.max(1, Math.round(opts.everyMs / 1000))\n const fire = async () => {\n if (this.shutdown.signal.aborted) return\n const boss = await this.ready()\n // Delivery-shaped, like every other payload on every adapter: a tick is a delivery\n // whose handler happens to ignore it, not a second kind of message.\n await boss\n .send(opts.queue, tickDelivery(opts.id), { singletonKey: opts.id, singletonSeconds: everySeconds })\n .catch(() => undefined)\n }\n void fire()\n const timer = setInterval(() => void fire(), opts.everyMs)\n timer.unref?.()\n this.ticks.push(timer)\n }\n\n async bind(queue: string, handler: DeliveryHandler, opts: BindOptions): Promise<BoundWorker> {\n const boss = await this.ensureQueue(queue, Math.ceil(opts.activeTimeoutMs / 1000))\n\n const workerId = await boss.work(\n queue,\n { batchSize: opts.concurrency },\n async (jobs: PgBossJob[]) => {\n for (const job of jobs) {\n // Whatever arrived is handed on unexamined. An adapter that inspects payloads\n // decides what counts as a real delivery, and this one used to skip anything\n // without a `jobId` \u2014 which silently swallowed every reconciler tick.\n const delivery = job.data\n\n const combined = new AbortController()\n const relay = () => combined.abort()\n job.signal?.addEventListener('abort', relay, { once: true })\n this.shutdown.signal.addEventListener('abort', relay, { once: true })\n\n try {\n await handler(delivery, {\n transportJobId: job.id,\n // pg-boss does not expose the attempt on the job, so the adapter reports the\n // first attempt and lets its own retry policy carry the rest. The consequence is\n // narrow: the delay written to `next_run_at` is the base rather than a backed-off\n // one, and pg-boss's own `retryBackoff` still spaces the real deliveries.\n attempt: 1,\n maxAttempts: 1,\n signal: combined.signal,\n handBack: async (next, handBackOpts) => {\n // No native hand-back: send the next delivery and let this one complete. The\n // row is already at `seq + 1`, so the new key cannot collide with this job.\n await this.enqueue(queue, next, {\n delayMs: handBackOpts?.delayMs,\n retry: { attempts: 1, backoff: { type: 'fixed', delayMs: 0, maxDelayMs: 0 } },\n })\n },\n })\n } catch (error) {\n if ((error as { name?: string })?.name === 'NoFurtherAttempts') continue // settled; no retry wanted\n throw error\n } finally {\n job.signal?.removeEventListener('abort', relay)\n this.shutdown.signal.removeEventListener('abort', relay)\n }\n }\n },\n )\n\n this.workerIds.push({ queue, id: workerId })\n return {\n queue,\n close: async () => {\n const instance = await this.ready()\n await instance.offWork(queue, { id: workerId }).catch(() => undefined)\n },\n }\n }\n\n async close(opts: { timeoutMs?: number } = {}): Promise<void> {\n this.shutdown.abort()\n for (const timer of this.ticks) clearInterval(timer)\n this.ticks.length = 0\n if (!this.boss) return\n if (!this.ownsInstance) return\n await this.boss.stop({ graceful: true, close: true, timeout: opts.timeoutMs ?? 30_000 }).catch(() => undefined)\n this.boss = null\n this.starting = null\n }\n}\n"],
|
|
5
|
+
"mappings": "AAWA,SAAS,kBAAkB;AAiE3B,IAAI,SAA8B;AAClC,eAAe,SAAgC;AAC7C,MAAI,OAAQ,QAAO;AACnB,MAAI;AAOF,aAAU,MAAM,OAAO,SAAS;AAChC,WAAO;AAAA,EACT,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR;AAAA,MACA,EAAE,OAAO,MAAM;AAAA,IACjB;AAAA,EACF;AACF;AAUA,SAAS,KAAK,IAAiB;AAC7B,SAAO;AAAA,IACL,MAAM,WAAW,MAAc,QAAmB;AAChD,YAAM,SAAS,MAAM,GAAG,MAAM,MAAM,MAAM;AAC1C,aAAO,EAAE,MAAM,OAAO,MAAmC,UAAU,OAAO,SAAS;AAAA,IACrF;AAAA,EACF;AACF;AAGA,MAAM,eAAe,CAAC,QAA0B,EAAE,OAAO,IAAI,KAAK,GAAG,UAAU,EAAE;AAE1E,MAAM,gBAA4C;AAAA,EAYvD,YAA6B,SAAiC;AAAjC;AAX7B,SAAS,OAAO;AAChB,SAAS,+BAA+B;AAExC,SAAQ,OAA8B;AACtC,SAAQ,WAA2C;AAEnD,SAAiB,gBAAgB,oBAAI,IAAY;AACjD,SAAiB,YAAkD,CAAC;AACpE,SAAiB,QAA0B,CAAC;AAC5C,SAAiB,WAAW,IAAI,gBAAgB;AAG9C,SAAK,OAAO,QAAQ,YAAY;AAChC,SAAK,eAAe,CAAC,QAAQ;AAAA,EAC/B;AAAA,EAEA,MAAc,QAAiC;AAC7C,QAAI,KAAK,KAAM,QAAO,KAAK;AAC3B,QAAI,CAAC,KAAK,UAAU;AAClB,WAAK,YAAY,YAAY;AAC3B,cAAM,EAAE,OAAO,IAAI,MAAM,OAAO;AAChC,cAAM,WAAW,IAAI,OAAO;AAAA,UAC1B,kBAAkB,KAAK,QAAQ;AAAA,UAC/B,QAAQ,KAAK,QAAQ,UAAU;AAAA,UAC/B,GAAI,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC;AAAA,QACtD,CAAC;AACD,cAAM,SAAS,MAAM;AACrB,aAAK,OAAO;AACZ,eAAO;AAAA,MACT,GAAG;AAAA,IACL;AACA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,YAAY,MAAc,iBAAmD;AACzF,UAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,QAAI,KAAK,cAAc,IAAI,IAAI,EAAG,QAAO;AACzC,UAAM,KAAK,YAAY,MAAM,kBAAkB,EAAE,gBAAgB,IAAI,MAAS;AAC9E,SAAK,cAAc,IAAI,IAAI;AAC3B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAAQ,OAAe,UAAoB,MAA2D;AAC1G,UAAM,OAAO,MAAM,KAAK,YAAY,KAAK;AACzC,UAAM,MAAM,WAAW,QAAQ;AAC/B,UAAM,OAAO,MAAM,KAAK,KAAK,OAAO,UAA+B;AAAA;AAAA;AAAA,MAGjE,cAAc;AAAA,MACd,YAAY,KAAK,WAAW,KAAK,UAAU,IAAI,KAAK,KAAK,KAAK,UAAU,GAAI,IAAI;AAAA,MAChF,YAAY,KAAK,MAAM;AAAA,MACvB,YAAY,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,MAAM,QAAQ,UAAU,GAAI,CAAC;AAAA;AAAA,MAErE,GAAI,KAAK,MAAM,QAAQ,SAAS,gBAC5B,EAAE,cAAc,MAAM,eAAe,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,MAAM,QAAQ,aAAa,GAAI,CAAC,EAAE,IACnG,EAAE,cAAc,MAAM;AAAA,MAC1B,GAAI,KAAK,KAAK,EAAE,IAAI,KAAK,KAAK,EAAE,EAAE,IAAI,CAAC;AAAA,IACzC,CAAC;AAGD,WAAO,EAAE,gBAAgB,QAAQ,IAAI;AAAA,EACvC;AAAA,EAEA,MAAM,OAAO,OAAe,gBAAuC;AACjE,UAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,UAAM,KAAK,UAAU,OAAO,cAAc,EAAE,MAAM,MAAM,MAAS;AAAA,EACnE;AAAA,EAEA,MAAM,SAAS,OAAe,gBAAgD;AAC5E,UAAM,OAAO,MAAM,KAAK,MAAM;AAC9B,UAAM,MAAM,MAAM,KAAK,WAAW,OAAO,cAAc,EAAE,MAAM,MAAM,IAAI;AACzE,QAAI,CAAC,IAAK,QAAO;AACjB,YAAQ,IAAI,OAAO;AAAA,MACjB,KAAK;AAAA,MACL,KAAK;AACH,eAAO;AAAA,MACT,KAAK;AACH,eAAO;AAAA,MACT,KAAK;AACH,eAAO;AAAA,MACT,KAAK;AAAA,MACL,KAAK;AACH,eAAO;AAAA,MACT;AACE,eAAO;AAAA,IACX;AAAA,EACF;AAAA,EAEA,MAAM,WAAW,MAAqE;AACpF,UAAM,KAAK,YAAY,KAAK,KAAK;AAMjC,UAAM,eAAe,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,UAAU,GAAI,CAAC;AAChE,UAAM,OAAO,YAAY;AACvB,UAAI,KAAK,SAAS,OAAO,QAAS;AAClC,YAAM,OAAO,MAAM,KAAK,MAAM;AAG9B,YAAM,KACH,KAAK,KAAK,OAAO,aAAa,KAAK,EAAE,GAAG,EAAE,cAAc,KAAK,IAAI,kBAAkB,aAAa,CAAC,EACjG,MAAM,MAAM,MAAS;AAAA,IAC1B;AACA,SAAK,KAAK;AACV,UAAM,QAAQ,YAAY,MAAM,KAAK,KAAK,GAAG,KAAK,OAAO;AACzD,UAAM,QAAQ;AACd,SAAK,MAAM,KAAK,KAAK;AAAA,EACvB;AAAA,EAEA,MAAM,KAAK,OAAe,SAA0B,MAAyC;AAC3F,UAAM,OAAO,MAAM,KAAK,YAAY,OAAO,KAAK,KAAK,KAAK,kBAAkB,GAAI,CAAC;AAEjF,UAAM,WAAW,MAAM,KAAK;AAAA,MAC1B;AAAA,MACA,EAAE,WAAW,KAAK,YAAY;AAAA,MAC9B,OAAO,SAAsB;AAC3B,mBAAW,OAAO,MAAM;AAItB,gBAAM,WAAW,IAAI;AAErB,gBAAM,WAAW,IAAI,gBAAgB;AACrC,gBAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,cAAI,QAAQ,iBAAiB,SAAS,OAAO,EAAE,MAAM,KAAK,CAAC;AAC3D,eAAK,SAAS,OAAO,iBAAiB,SAAS,OAAO,EAAE,MAAM,KAAK,CAAC;AAEpE,cAAI;AACF,kBAAM,QAAQ,UAAU;AAAA,cACtB,gBAAgB,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA,cAKpB,SAAS;AAAA,cACT,aAAa;AAAA,cACb,QAAQ,SAAS;AAAA,cACjB,UAAU,OAAO,MAAM,iBAAiB;AAGtC,sBAAM,KAAK,QAAQ,OAAO,MAAM;AAAA,kBAC9B,SAAS,cAAc;AAAA,kBACvB,OAAO,EAAE,UAAU,GAAG,SAAS,EAAE,MAAM,SAAS,SAAS,GAAG,YAAY,EAAE,EAAE;AAAA,gBAC9E,CAAC;AAAA,cACH;AAAA,YACF,CAAC;AAAA,UACH,SAAS,OAAO;AACd,gBAAK,OAA6B,SAAS,oBAAqB;AAChE,kBAAM;AAAA,UACR,UAAE;AACA,gBAAI,QAAQ,oBAAoB,SAAS,KAAK;AAC9C,iBAAK,SAAS,OAAO,oBAAoB,SAAS,KAAK;AAAA,UACzD;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,UAAU,KAAK,EAAE,OAAO,IAAI,SAAS,CAAC;AAC3C,WAAO;AAAA,MACL;AAAA,MACA,OAAO,YAAY;AACjB,cAAM,WAAW,MAAM,KAAK,MAAM;AAClC,cAAM,SAAS,QAAQ,OAAO,EAAE,IAAI,SAAS,CAAC,EAAE,MAAM,MAAM,MAAS;AAAA,MACvE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,OAA+B,CAAC,GAAkB;AAC5D,SAAK,SAAS,MAAM;AACpB,eAAW,SAAS,KAAK,MAAO,eAAc,KAAK;AACnD,SAAK,MAAM,SAAS;AACpB,QAAI,CAAC,KAAK,KAAM;AAChB,QAAI,CAAC,KAAK,aAAc;AACxB,UAAM,KAAK,KAAK,KAAK,EAAE,UAAU,MAAM,OAAO,MAAM,SAAS,KAAK,aAAa,IAAO,CAAC,EAAE,MAAM,MAAM,MAAS;AAC9G,SAAK,OAAO;AACZ,SAAK,WAAW;AAAA,EAClB;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fullstackhouse/open-mercato-durable-work",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Durable at-least-once background work for Open Mercato apps: a leased job record in Postgres with epoch fencing, bounded resumable slices, a server-side reconciler, fenced cancel and an operator API. Pluggable transport (BullMQ or pg-boss).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest'
|
|
2
|
+
|
|
3
|
+
import { readConfig } from '../config'
|
|
4
|
+
|
|
5
|
+
describe('the pg-boss connection cap', () => {
|
|
6
|
+
it('defaults well below pg-boss own default, because this pool is per process', () => {
|
|
7
|
+
// The mechanism's SQL rides the host's EntityManager; pg-boss does not, so it opens a pool
|
|
8
|
+
// besides the app's — in the web process too, once the worker runs in-process. Multiplied by
|
|
9
|
+
// replicas during a rolling deploy, the library default is enough to eat a `max_connections`
|
|
10
|
+
// budget that was sized without it.
|
|
11
|
+
expect(readConfig({ DATABASE_URL: 'postgres://x/y' } as NodeJS.ProcessEnv).pgBossMaxConnections).toBe(4)
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
it('is tunable for a host that has the headroom', () => {
|
|
15
|
+
const config = readConfig({ DATABASE_URL: 'postgres://x/y', DURABLE_WORK_PGBOSS_MAX: '12' } as NodeJS.ProcessEnv)
|
|
16
|
+
expect(config.pgBossMaxConnections).toBe(12)
|
|
17
|
+
})
|
|
18
|
+
})
|
package/src/om/config.ts
CHANGED
|
@@ -10,6 +10,8 @@ export type DurableWorkConfig = {
|
|
|
10
10
|
redisUrl: string | null
|
|
11
11
|
databaseUrl: string | null
|
|
12
12
|
pgBossSchema: string
|
|
13
|
+
/** Cap on pg-boss's own connection pool. Small on purpose — see the transport option. */
|
|
14
|
+
pgBossMaxConnections: number
|
|
13
15
|
tickMs: number
|
|
14
16
|
drainTimeoutMs: number
|
|
15
17
|
reconcilerGraceMs: number
|
|
@@ -36,6 +38,9 @@ export function readConfig(env: NodeJS.ProcessEnv = process.env): DurableWorkCon
|
|
|
36
38
|
redisUrl: env.DURABLE_WORK_REDIS_URL ?? env.QUEUE_REDIS_URL ?? env.REDIS_URL ?? null,
|
|
37
39
|
databaseUrl: env.DATABASE_URL ?? null,
|
|
38
40
|
pgBossSchema: env.DURABLE_WORK_PGBOSS_SCHEMA ?? 'durable_work_boss',
|
|
41
|
+
// Deliberately far below pg-boss's own default: this pool is opened per process that
|
|
42
|
+
// touches the transport, and an in-process worker means the web process is one of them.
|
|
43
|
+
pgBossMaxConnections: num(env.DURABLE_WORK_PGBOSS_MAX, 4),
|
|
39
44
|
tickMs: num(env.DURABLE_WORK_TICK_MS, 15_000),
|
|
40
45
|
drainTimeoutMs: num(env.DURABLE_WORK_DRAIN_TIMEOUT_MS, 30_000),
|
|
41
46
|
reconcilerGraceMs: num(env.DURABLE_WORK_GRACE_MS, 20_000),
|
|
@@ -59,7 +64,11 @@ export function createTransport(config: DurableWorkConfig, deps: { redisConnecti
|
|
|
59
64
|
}
|
|
60
65
|
case 'pgboss': {
|
|
61
66
|
if (!config.databaseUrl) throw new Error('DURABLE_WORK_TRANSPORT=pgboss requires DATABASE_URL.')
|
|
62
|
-
return new PgBossTransport({
|
|
67
|
+
return new PgBossTransport({
|
|
68
|
+
connectionString: config.databaseUrl,
|
|
69
|
+
schema: config.pgBossSchema,
|
|
70
|
+
max: config.pgBossMaxConnections,
|
|
71
|
+
})
|
|
63
72
|
}
|
|
64
73
|
}
|
|
65
74
|
}
|
package/src/transport/pgboss.ts
CHANGED
|
@@ -54,7 +54,7 @@ type PgBossInstance = {
|
|
|
54
54
|
offWork(name: string, options?: { id?: string }): Promise<unknown>
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
type PgBossModule = { PgBoss: new (options: { connectionString: string; schema?: string }) => PgBossInstance }
|
|
57
|
+
type PgBossModule = { PgBoss: new (options: { connectionString: string; schema?: string; max?: number }) => PgBossInstance }
|
|
58
58
|
|
|
59
59
|
export type PgBossTransportOptions = {
|
|
60
60
|
connectionString: string
|
|
@@ -62,6 +62,16 @@ export type PgBossTransportOptions = {
|
|
|
62
62
|
schema?: string
|
|
63
63
|
/** Reuse an already-started instance instead of owning its lifecycle. */
|
|
64
64
|
instance?: PgBossInstance
|
|
65
|
+
/**
|
|
66
|
+
* Cap on pg-boss's own connection pool.
|
|
67
|
+
*
|
|
68
|
+
* It matters more than it looks. This transport opens a pool *besides* the app's — the
|
|
69
|
+
* mechanism's own SQL rides the host's EntityManager, but pg-boss does not — and a host that
|
|
70
|
+
* runs the worker in-process gets one per process that touches the transport: the web process
|
|
71
|
+
* and any spawned queue worker. Multiplied by replicas during a rolling deploy, pg-boss's own
|
|
72
|
+
* default is enough to eat a Postgres `max_connections` budget that was sized without it.
|
|
73
|
+
*/
|
|
74
|
+
max?: number
|
|
65
75
|
}
|
|
66
76
|
|
|
67
77
|
let cached: PgBossModule | null = null
|
|
@@ -126,7 +136,11 @@ export class PgBossTransport implements TransportAdapter {
|
|
|
126
136
|
if (!this.starting) {
|
|
127
137
|
this.starting = (async () => {
|
|
128
138
|
const { PgBoss } = await pgboss()
|
|
129
|
-
const instance = new PgBoss({
|
|
139
|
+
const instance = new PgBoss({
|
|
140
|
+
connectionString: this.options.connectionString,
|
|
141
|
+
schema: this.options.schema ?? 'durable_work_boss',
|
|
142
|
+
...(this.options.max ? { max: this.options.max } : {}),
|
|
143
|
+
})
|
|
130
144
|
await instance.start()
|
|
131
145
|
this.boss = instance
|
|
132
146
|
return instance
|