@fabrika/platform-node 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/migrations/0001_jobs.sql +33 -0
- package/migrations/0002_deploy_locks.sql +24 -0
- package/package.json +37 -0
- package/src/asset-server-fs.ts +113 -0
- package/src/blob-s3.ts +79 -0
- package/src/index.ts +47 -0
- package/src/job-queue-postgres.ts +275 -0
- package/src/placeholders.ts +199 -0
- package/src/postgres-migrations.ts +355 -0
- package/src/service-postgres-migrations.ts +69 -0
- package/src/sql-postgres.ts +176 -0
- package/src/uuid.ts +94 -0
- package/src/wait-until.ts +69 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
-- Job queue — the table behind `PostgresJobQueue` (src/job-queue-postgres.ts).
|
|
2
|
+
--
|
|
3
|
+
-- On Workers a queue exists because there is no process to poll with. A long-running process has one,
|
|
4
|
+
-- so the whole primitive collapses to a table of due work plus `SELECT … FOR UPDATE SKIP LOCKED`.
|
|
5
|
+
--
|
|
6
|
+
-- Delivery is VISIBILITY-based, like SQS and Cloudflare Queues: a claim pushes `visible_at` into the
|
|
7
|
+
-- future and bumps `attempts`; a completed job deletes its row; a consumer that dies simply lets the
|
|
8
|
+
-- timeout lapse and the job becomes claimable again. There is no separate lock column because a lock
|
|
9
|
+
-- that outlives its holder is the bug the visibility timeout exists to prevent.
|
|
10
|
+
--
|
|
11
|
+
-- DIALECT. This DDL is in the SQLite ∩ Postgres common subset — no AUTOINCREMENT, no SERIAL, no
|
|
12
|
+
-- `unixepoch()`/`now()` default (ids and timestamps are stamped caller-side, per the repo convention),
|
|
13
|
+
-- no partial index. It applies unchanged to both. The Postgres-specific part of this feature is the
|
|
14
|
+
-- CLAIM STATEMENT, not the schema: `FOR UPDATE SKIP LOCKED` has no SQLite equivalent, which is fine —
|
|
15
|
+
-- SQLite has no concurrent writers to skip.
|
|
16
|
+
--
|
|
17
|
+
-- TYPE NOTE (this bites on Postgres only). Bun's Postgres client decodes `BIGINT` as a STRING, never a
|
|
18
|
+
-- number. The millisecond columns below are therefore BIGINT and are never read back into JS — every
|
|
19
|
+
-- comparison against them happens inside SQL. `attempts`/`max_attempts` are INTEGER (int4) precisely
|
|
20
|
+
-- so they DO come back as numbers.
|
|
21
|
+
CREATE TABLE IF NOT EXISTS jobs (
|
|
22
|
+
id TEXT PRIMARY KEY, -- UUIDv7, minted caller-side (time-ordered)
|
|
23
|
+
queue TEXT NOT NULL, -- queue name, so several queues share one table
|
|
24
|
+
payload TEXT NOT NULL, -- the JSON-serialized message
|
|
25
|
+
visible_at BIGINT NOT NULL, -- unix MILLIS: claimable once now >= this (delay + visibility timeout)
|
|
26
|
+
attempts INTEGER NOT NULL, -- claims consumed so far
|
|
27
|
+
max_attempts INTEGER NOT NULL, -- claims allowed before the job is abandoned
|
|
28
|
+
created_at BIGINT NOT NULL -- unix MILLIS, enqueued
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
-- The claim's access path: due rows of one queue, oldest first. `id` tie-breaks so the order is total
|
|
32
|
+
-- and two consumers scan in the same sequence (which is what makes SKIP LOCKED skip cheaply).
|
|
33
|
+
CREATE INDEX IF NOT EXISTS idx_jobs_due ON jobs(queue, visible_at, id);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
-- Deploy locks — the table behind `SqlDeployLocks` (src/deploy-locks-sql.ts).
|
|
2
|
+
--
|
|
3
|
+
-- One row per lease. The row IS the lock: acquire is a single conditional upsert whose `changes` count
|
|
4
|
+
-- is the answer, so there is no read-then-write window for two consumers to slip through. A MISSING
|
|
5
|
+
-- row means free; a present row whose `expires_at` is in the past means expired-and-takeable; a
|
|
6
|
+
-- release deletes the row, but only when the caller is still the named holder.
|
|
7
|
+
--
|
|
8
|
+
-- `SqlDeployLocks` is parameterised by table name, so a service that wants a second, independent lease
|
|
9
|
+
-- namespace applies this DDL under a different name rather than getting a second implementation.
|
|
10
|
+
--
|
|
11
|
+
-- DIALECT. Common subset — no defaults, no generated columns, no partial index; applies unchanged to
|
|
12
|
+
-- SQLite and Postgres.
|
|
13
|
+
--
|
|
14
|
+
-- TYPE NOTE (this bites on Postgres only). `expires_at` is a `Date.now()`-based DEADLINE in unix
|
|
15
|
+
-- MILLISECONDS, not a creation timestamp, so it does NOT follow the unix-seconds convention the `*_at`
|
|
16
|
+
-- columns elsewhere in this codebase use. It must be BIGINT: Postgres `INTEGER` is int4, which tops
|
|
17
|
+
-- out at 2147483647 and cannot hold a millisecond epoch at all (SQLite's INTEGER is 64-bit, so the
|
|
18
|
+
-- same DDL survives there and the overflow only appears after the port). Nothing reads the column into
|
|
19
|
+
-- JS — Bun would decode a BIGINT as a string — every comparison stays inside SQL.
|
|
20
|
+
CREATE TABLE IF NOT EXISTS deploy_locks (
|
|
21
|
+
lock_key TEXT PRIMARY KEY, -- the lease target, e.g. `<app_id>:<env>`
|
|
22
|
+
holder TEXT NOT NULL, -- the run id holding the lease; release is checked against it
|
|
23
|
+
expires_at BIGINT NOT NULL -- unix MILLIS after which the lease is stale and takeable
|
|
24
|
+
);
|
package/package.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@fabrika/platform-node",
|
|
3
|
+
"version": "0.0.1",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "git+https://github.com/contember/fabrika-platform.git",
|
|
8
|
+
"directory": "packages/platform-node"
|
|
9
|
+
},
|
|
10
|
+
"publishConfig": {
|
|
11
|
+
"access": "public"
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"src",
|
|
15
|
+
"migrations",
|
|
16
|
+
"!src/**/__tests__/**"
|
|
17
|
+
],
|
|
18
|
+
"exports": {
|
|
19
|
+
".": {
|
|
20
|
+
"bun": "./src/index.ts",
|
|
21
|
+
"types": "./src/index.ts",
|
|
22
|
+
"default": "./src/index.ts"
|
|
23
|
+
},
|
|
24
|
+
"./postgres-migrations": {
|
|
25
|
+
"bun": "./src/postgres-migrations.ts",
|
|
26
|
+
"types": "./src/postgres-migrations.ts",
|
|
27
|
+
"default": "./src/postgres-migrations.ts"
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"scripts": {
|
|
31
|
+
"typecheck": "tsc --noEmit",
|
|
32
|
+
"test": "bun test"
|
|
33
|
+
},
|
|
34
|
+
"dependencies": {
|
|
35
|
+
"@fabrika/platform": "^0.0.1"
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// `AssetServer` over a built SPA directory — the replacement for Cloudflare's `ASSETS` binding.
|
|
2
|
+
//
|
|
3
|
+
// Three things it has to get right, and they are the three things a hand-rolled static handler
|
|
4
|
+
// usually gets wrong:
|
|
5
|
+
// * PATH SAFETY. The request path is attacker-controlled. It is percent-decoded, resolved against
|
|
6
|
+
// the root, and then the RESOLVED path is checked to still be inside the root — so `..`, encoded
|
|
7
|
+
// `%2e%2e` and absolute paths all land outside the root and are refused.
|
|
8
|
+
// * CONTENT TYPES. Taken from `Bun.file(...).type`, which maps the extension (and appends a charset
|
|
9
|
+
// for text types). A wrong `content-type` on `.js` or `.css` breaks a SPA outright in a browser.
|
|
10
|
+
// * SPA FALLBACK. A client-routed path like `/apps/foo` has no file behind it and must serve
|
|
11
|
+
// `index.html` with 200, not 404 — but a MISSING ASSET must still 404, or a typo'd bundle URL
|
|
12
|
+
// silently returns HTML and the browser reports a syntax error instead of a missing file. The
|
|
13
|
+
// discriminator is "does this look like a document request": an explicit `text/html` in `Accept`
|
|
14
|
+
// (every browser navigation) or an extension-less path (curl, deep links). A request for
|
|
15
|
+
// `/assets/main-abc123.js` has an extension and does not accept HTML, so it 404s.
|
|
16
|
+
|
|
17
|
+
import type { AssetServer } from '@fabrika/platform'
|
|
18
|
+
import { join, resolve, sep } from 'node:path'
|
|
19
|
+
|
|
20
|
+
export interface FileSystemAssetServerOptions {
|
|
21
|
+
/** File served for a directory and for the SPA fallback. */
|
|
22
|
+
indexFile?: string
|
|
23
|
+
/** Serve `indexFile` (200) for an unmatched document request. Off = everything unmatched 404s. */
|
|
24
|
+
spaFallback?: boolean
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const DEFAULT_INDEX = 'index.html'
|
|
28
|
+
/** A trailing `.ext` on the last path segment — the "this is an asset, not a route" signal. */
|
|
29
|
+
const HAS_EXTENSION = /\.[^./]+$/
|
|
30
|
+
|
|
31
|
+
export class FileSystemAssetServer implements AssetServer {
|
|
32
|
+
private readonly root: string
|
|
33
|
+
private readonly indexFile: string
|
|
34
|
+
private readonly spaFallback: boolean
|
|
35
|
+
|
|
36
|
+
constructor(root: string, options: FileSystemAssetServerOptions = {}) {
|
|
37
|
+
this.root = resolve(root)
|
|
38
|
+
this.indexFile = options.indexFile ?? DEFAULT_INDEX
|
|
39
|
+
this.spaFallback = options.spaFallback ?? true
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async fetch(request: Request): Promise<Response> {
|
|
43
|
+
if (request.method !== 'GET' && request.method !== 'HEAD') {
|
|
44
|
+
return new Response('method not allowed', { status: 405, headers: { allow: 'GET, HEAD' } })
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const pathname = decodePathname(new URL(request.url).pathname)
|
|
48
|
+
if (pathname === null) {
|
|
49
|
+
return new Response('bad request', { status: 400 })
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const file = await this.locate(pathname)
|
|
53
|
+
if (file !== null) {
|
|
54
|
+
return serve(file, request.method)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Unmatched. A navigation gets the SPA shell; a missing asset gets an honest 404.
|
|
58
|
+
if (this.spaFallback && looksLikeDocument(request, pathname)) {
|
|
59
|
+
const fallback = await this.locate(`/${this.indexFile}`)
|
|
60
|
+
if (fallback !== null) {
|
|
61
|
+
return serve(fallback, request.method)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return new Response('not found', { status: 404 })
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The file backing `pathname`, or null when there is none inside the root. */
|
|
68
|
+
private async locate(pathname: string): Promise<Bun.BunFile | null> {
|
|
69
|
+
const candidate = resolve(this.root, `.${pathname}`)
|
|
70
|
+
if (candidate !== this.root && !candidate.startsWith(this.root + sep)) {
|
|
71
|
+
return null
|
|
72
|
+
}
|
|
73
|
+
const direct = Bun.file(candidate)
|
|
74
|
+
if (await direct.exists()) {
|
|
75
|
+
return direct
|
|
76
|
+
}
|
|
77
|
+
// A directory is not a `BunFile` that exists, so this also covers `/` and `/sub/`.
|
|
78
|
+
const index = Bun.file(join(candidate, this.indexFile))
|
|
79
|
+
return await index.exists() ? index : null
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function serve(file: Bun.BunFile, method: string): Response {
|
|
84
|
+
const headers = new Headers({
|
|
85
|
+
'content-type': file.type === '' ? 'application/octet-stream' : file.type,
|
|
86
|
+
'content-length': String(file.size),
|
|
87
|
+
})
|
|
88
|
+
// HEAD must carry the same headers as the GET it stands for, but no body. A `BunFile` body streams.
|
|
89
|
+
return method === 'HEAD' ? new Response(null, { status: 200, headers }) : new Response(file, { status: 200, headers })
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Percent-decode a request path. Null when it is malformed or smuggles a NUL. */
|
|
93
|
+
function decodePathname(pathname: string): string | null {
|
|
94
|
+
let decoded: string
|
|
95
|
+
try {
|
|
96
|
+
decoded = decodeURIComponent(pathname)
|
|
97
|
+
} catch {
|
|
98
|
+
return null
|
|
99
|
+
}
|
|
100
|
+
if (decoded.includes('\0')) {
|
|
101
|
+
return null
|
|
102
|
+
}
|
|
103
|
+
return decoded.startsWith('/') ? decoded : `/${decoded}`
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Whether this reads as a request for a page rather than for a build artifact. */
|
|
107
|
+
function looksLikeDocument(request: Request, pathname: string): boolean {
|
|
108
|
+
if (request.headers.get('accept')?.includes('text/html') === true) {
|
|
109
|
+
return true
|
|
110
|
+
}
|
|
111
|
+
const lastSegment = pathname.slice(pathname.lastIndexOf('/') + 1)
|
|
112
|
+
return !HAS_EXTENSION.test(lastSegment)
|
|
113
|
+
}
|
package/src/blob-s3.ts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// `BlobStore` over S3-compatible object storage, using Bun's built-in S3 client (`Bun.S3Client`).
|
|
2
|
+
//
|
|
3
|
+
// ONE implementation covers both targets the platform has: Cloudflare R2 and Zerops object storage
|
|
4
|
+
// (MinIO) are both S3-compatible, so the only difference is the endpoint and the credentials. That is
|
|
5
|
+
// also why this does not depend on `@aws-sdk/client-s3` — the built-in client speaks the same
|
|
6
|
+
// protocol, ships with the runtime, and keeps credentials in one object instead of a middleware stack.
|
|
7
|
+
//
|
|
8
|
+
// Divergence from the R2 binding worth knowing: R2's `get()` returns null for a missing key in the
|
|
9
|
+
// same round trip, whereas S3 needs a HEAD first. `get()` therefore costs one extra request on a hit.
|
|
10
|
+
// Nothing here logs the credentials, the endpoint, or an error object that might quote a signed URL.
|
|
11
|
+
|
|
12
|
+
import type { BlobStore } from '@fabrika/platform'
|
|
13
|
+
|
|
14
|
+
/** Connection details for an S3-compatible bucket. `endpoint` is what makes it R2, MinIO, or AWS. */
|
|
15
|
+
export interface S3BlobStoreOptions {
|
|
16
|
+
bucket: string
|
|
17
|
+
accessKeyId: string
|
|
18
|
+
secretAccessKey: string
|
|
19
|
+
/** e.g. `https://<account>.r2.cloudflarestorage.com` (R2) or `http://storage:9000` (Zerops/MinIO). */
|
|
20
|
+
endpoint?: string
|
|
21
|
+
/** Required by AWS; R2 wants `auto`; MinIO ignores it. */
|
|
22
|
+
region?: string
|
|
23
|
+
/** Temporary-credential session token, when the deployment uses one. */
|
|
24
|
+
sessionToken?: string
|
|
25
|
+
/** `https://<bucket>.<host>` addressing. MinIO normally wants this OFF (path-style). */
|
|
26
|
+
virtualHostedStyle?: boolean
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The blob body handed back by `get()` — a stream, opened lazily and only once, plus a text shortcut. */
|
|
30
|
+
interface BlobBody {
|
|
31
|
+
body: ReadableStream
|
|
32
|
+
text(): Promise<string>
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export class S3BlobStore implements BlobStore {
|
|
36
|
+
constructor(private readonly client: Bun.S3Client) {}
|
|
37
|
+
|
|
38
|
+
/** Build a store from plain connection options. Never logs them. */
|
|
39
|
+
static connect(options: S3BlobStoreOptions): S3BlobStore {
|
|
40
|
+
return new S3BlobStore(
|
|
41
|
+
new Bun.S3Client({
|
|
42
|
+
bucket: options.bucket,
|
|
43
|
+
accessKeyId: options.accessKeyId,
|
|
44
|
+
secretAccessKey: options.secretAccessKey,
|
|
45
|
+
...(options.endpoint !== undefined ? { endpoint: options.endpoint } : {}),
|
|
46
|
+
...(options.region !== undefined ? { region: options.region } : {}),
|
|
47
|
+
...(options.sessionToken !== undefined ? { sessionToken: options.sessionToken } : {}),
|
|
48
|
+
...(options.virtualHostedStyle !== undefined ? { virtualHostedStyle: options.virtualHostedStyle } : {}),
|
|
49
|
+
}),
|
|
50
|
+
)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async put(key: string, value: string | ArrayBuffer | ReadableStream): Promise<void> {
|
|
54
|
+
// The S3 client takes bytes or a Response, not a bare stream — wrapping a stream in a Response
|
|
55
|
+
// is what lets it be uploaded without buffering the whole body first.
|
|
56
|
+
const payload = value instanceof ReadableStream ? new Response(value) : value
|
|
57
|
+
await this.client.write(key, payload)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async get(key: string): Promise<BlobBody | null> {
|
|
61
|
+
const file = this.client.file(key)
|
|
62
|
+
if (!(await file.exists())) {
|
|
63
|
+
return null
|
|
64
|
+
}
|
|
65
|
+
let stream: ReadableStream | undefined
|
|
66
|
+
return {
|
|
67
|
+
// Lazy + memoised, matching R2: the body is one stream, consumable once.
|
|
68
|
+
get body(): ReadableStream {
|
|
69
|
+
stream ??= file.stream()
|
|
70
|
+
return stream
|
|
71
|
+
},
|
|
72
|
+
text: () => file.text(),
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async delete(key: string): Promise<void> {
|
|
77
|
+
await this.client.delete(key)
|
|
78
|
+
}
|
|
79
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// @fabrika/platform-node — every `@fabrika/platform` port implemented for a LONG-RUNNING BUN PROCESS.
|
|
2
|
+
//
|
|
3
|
+
// This is the second implementation set behind the ports (the first being the Cloudflare bindings that
|
|
4
|
+
// live with the workers using them), and the one ADR-0003 needs: on Zerops the control plane executes
|
|
5
|
+
// the deploy itself, in a process, so it needs Postgres instead of D1, S3 instead of R2, a job table
|
|
6
|
+
// instead of Cloudflare Queues, a lock row instead of a Durable Object, and a directory instead of an
|
|
7
|
+
// `ASSETS` binding.
|
|
8
|
+
//
|
|
9
|
+
// SqlDatabase → PostgresDatabase (Bun.SQL; `?` → `$n` via a real tokeniser)
|
|
10
|
+
// BlobStore → S3BlobStore (Bun.S3Client; one impl for R2 and MinIO alike)
|
|
11
|
+
// JobQueue → PostgresJobQueue (+ PostgresJobConsumer, the poller the port omits by design)
|
|
12
|
+
// DeployLocks → (none — @fabrika/platform's `SqlDeployLocks` needs nothing but the SqlDatabase
|
|
13
|
+
// port, so it is runtime-neutral and lives there; import it from @fabrika/platform)
|
|
14
|
+
// AssetServer → FileSystemAssetServer (built SPA directory + SPA fallback)
|
|
15
|
+
// WaitUntil → createBackgroundTasks (supervised promises; rejections logged, never fatal)
|
|
16
|
+
//
|
|
17
|
+
// Dependencies: none beyond `@fabrika/platform` itself. Everything else is a Bun built-in.
|
|
18
|
+
// Migrations for what this package owns (the jobs table, the Postgres lock table) live in
|
|
19
|
+
// `migrations/` — DDL is per-dialect even when the statement over it is portable.
|
|
20
|
+
|
|
21
|
+
export { FileSystemAssetServer, type FileSystemAssetServerOptions } from './asset-server-fs'
|
|
22
|
+
export { S3BlobStore, type S3BlobStoreOptions } from './blob-s3'
|
|
23
|
+
export { type Job, PostgresJobConsumer, type PostgresJobConsumerOptions, PostgresJobQueue, type PostgresJobQueueOptions } from './job-queue-postgres'
|
|
24
|
+
export { rewritePlaceholders, type RewrittenSql } from './placeholders'
|
|
25
|
+
export {
|
|
26
|
+
applyPostgresMigrations,
|
|
27
|
+
definePostgresMigrationPlan,
|
|
28
|
+
platformNodePostgresMigrationBundle,
|
|
29
|
+
type PostgresLegacyMigrationEffect,
|
|
30
|
+
type PostgresLegacyMigrationLedger,
|
|
31
|
+
type PostgresMigration,
|
|
32
|
+
type PostgresMigrationBundle,
|
|
33
|
+
postgresMigrationFilenames,
|
|
34
|
+
postgresMigrationIdentity,
|
|
35
|
+
type PostgresMigrationPlan,
|
|
36
|
+
readPostgresMigrationBundle,
|
|
37
|
+
type ReadPostgresMigrationBundleOptions,
|
|
38
|
+
} from './postgres-migrations'
|
|
39
|
+
export {
|
|
40
|
+
definePostgresServiceMigrations,
|
|
41
|
+
postgresMigrationResultMessage,
|
|
42
|
+
type PostgresServiceMigrations,
|
|
43
|
+
type PostgresServiceMigrationsConfig,
|
|
44
|
+
} from './service-postgres-migrations'
|
|
45
|
+
export { PostgresDatabase, PostgresStatement, type SqlBindValue } from './sql-postgres'
|
|
46
|
+
export { uuidv7 } from './uuid'
|
|
47
|
+
export { type BackgroundTasks, type BackgroundTasksOptions, createBackgroundTasks } from './wait-until'
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
// `JobQueue` as a Postgres table plus `SELECT … FOR UPDATE SKIP LOCKED`, and the in-process consumer
|
|
2
|
+
// the port deliberately does not model.
|
|
3
|
+
//
|
|
4
|
+
// WHY THE CONSUMER IS NOT IN THE PORT. On Workers the consumer side is not an API at all — it is an
|
|
5
|
+
// inversion of control: the platform delivers a batch to an exported `queue()` handler, with the
|
|
6
|
+
// platform owning retries, batching, concurrency and the ack protocol. There is nothing for fabrika to
|
|
7
|
+
// call and nothing for it to start or stop. In a long-running process fabrika owns the loop, the
|
|
8
|
+
// visibility timeout and the retry policy. Those two shapes have no honest common supertype: putting
|
|
9
|
+
// a `poll()`/`start()` on the port would force the Cloudflare implementation to expose a loop it can
|
|
10
|
+
// never run, and putting a `queue(batch)` on it would force this side to fake a delivery it never
|
|
11
|
+
// receives. `send()` — the only operation both sides genuinely share — is the whole port, and the
|
|
12
|
+
// consumer is a concrete class here (see ../../../docs/reference/portability-surface.md: "the reason
|
|
13
|
+
// Workers need a queue does not apply" once there is a process to poll with).
|
|
14
|
+
//
|
|
15
|
+
// The claim is a SINGLE statement — an `UPDATE … WHERE id IN (SELECT … FOR UPDATE SKIP LOCKED)
|
|
16
|
+
// RETURNING …` — so it runs through the `SqlDatabase` port with no explicit transaction: Postgres
|
|
17
|
+
// evaluates the sub-select's row locks and the update atomically. `SKIP LOCKED` is what lets N
|
|
18
|
+
// consumers drain one queue without blocking each other or handing the same job to two of them.
|
|
19
|
+
//
|
|
20
|
+
// Visibility, not locking, is the delivery model (same as SQS and Cloudflare Queues): claiming a job
|
|
21
|
+
// pushes `visible_at` into the future and bumps `attempts`. Finish it → `ack()` deletes the row.
|
|
22
|
+
// Crash → the timeout lapses and the job is claimable again. That is the ONLY reason a consumer that
|
|
23
|
+
// dies mid-job does not strand it, so `visibilityTimeoutMs` must exceed the slowest handler.
|
|
24
|
+
//
|
|
25
|
+
// `FOR UPDATE SKIP LOCKED` is the one Postgres-specific statement in this package; the DDL itself
|
|
26
|
+
// (migrations/0001_jobs.sql) is in the SQLite ∩ Postgres common subset.
|
|
27
|
+
|
|
28
|
+
import type { JobQueue, SqlDatabase } from '@fabrika/platform'
|
|
29
|
+
import { uuidv7 } from './uuid'
|
|
30
|
+
|
|
31
|
+
/** A claimed job, as handed to a handler. `payload` has been through the caller's `decode`. */
|
|
32
|
+
export interface Job<T> {
|
|
33
|
+
id: string
|
|
34
|
+
payload: T
|
|
35
|
+
/** How many times this job has been claimed, including now. First delivery is 1. */
|
|
36
|
+
attempts: number
|
|
37
|
+
/** Claims allowed before the job is abandoned. */
|
|
38
|
+
maxAttempts: number
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface PostgresJobQueueOptions {
|
|
42
|
+
/** Table holding the jobs. Must already exist; see migrations/0001_jobs.sql. */
|
|
43
|
+
table?: string
|
|
44
|
+
/** Queue name, so several queues can share one table. */
|
|
45
|
+
queue?: string
|
|
46
|
+
/** Claims allowed per job before it is abandoned. */
|
|
47
|
+
maxAttempts?: number
|
|
48
|
+
/** Injectable clock (unix millis) so delays and timeouts are testable without sleeping. */
|
|
49
|
+
now?: () => number
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const DEFAULT_TABLE = 'jobs'
|
|
53
|
+
const DEFAULT_QUEUE = 'default'
|
|
54
|
+
const DEFAULT_MAX_ATTEMPTS = 5
|
|
55
|
+
/** A table name is interpolated into SQL, so it is restricted to a bare unqualified identifier. */
|
|
56
|
+
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
57
|
+
|
|
58
|
+
/** The claim statement's projection. snake_case, mirroring the migration. */
|
|
59
|
+
interface ClaimedRow {
|
|
60
|
+
id: string
|
|
61
|
+
payload: string
|
|
62
|
+
attempts: number
|
|
63
|
+
max_attempts: number
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Producer + claim/ack primitives over one jobs table. Implements the `JobQueue` port (`send`); the
|
|
68
|
+
* rest is the consumer-side surface `PostgresJobConsumer` is built from and tests drive directly.
|
|
69
|
+
*/
|
|
70
|
+
export class PostgresJobQueue<T> implements JobQueue<T> {
|
|
71
|
+
private readonly queue: string
|
|
72
|
+
private readonly maxAttempts: number
|
|
73
|
+
private readonly now: () => number
|
|
74
|
+
private readonly sendSql: string
|
|
75
|
+
private readonly claimSql: string
|
|
76
|
+
private readonly ackSql: string
|
|
77
|
+
private readonly deferSql: string
|
|
78
|
+
|
|
79
|
+
constructor(private readonly db: SqlDatabase, options: PostgresJobQueueOptions = {}) {
|
|
80
|
+
const table = options.table ?? DEFAULT_TABLE
|
|
81
|
+
if (!IDENTIFIER.test(table)) {
|
|
82
|
+
throw new Error(`invalid jobs table name: ${table}`)
|
|
83
|
+
}
|
|
84
|
+
this.queue = options.queue ?? DEFAULT_QUEUE
|
|
85
|
+
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS
|
|
86
|
+
this.now = options.now ?? Date.now
|
|
87
|
+
|
|
88
|
+
this.sendSql = `INSERT INTO ${table} (id, queue, payload, visible_at, attempts, max_attempts, created_at)
|
|
89
|
+
VALUES (?, ?, ?, ?, 0, ?, ?)`
|
|
90
|
+
// One statement: the sub-select takes row locks (skipping rows another consumer already holds)
|
|
91
|
+
// and the UPDATE hides + counts them, so a claim can never be split across two consumers.
|
|
92
|
+
this.claimSql = `UPDATE ${table} SET visible_at = ?, attempts = attempts + 1
|
|
93
|
+
WHERE id IN (
|
|
94
|
+
SELECT id FROM ${table}
|
|
95
|
+
WHERE queue = ? AND visible_at <= ?
|
|
96
|
+
ORDER BY visible_at, id
|
|
97
|
+
LIMIT ?
|
|
98
|
+
FOR UPDATE SKIP LOCKED
|
|
99
|
+
)
|
|
100
|
+
RETURNING id, payload, attempts, max_attempts`
|
|
101
|
+
this.ackSql = `DELETE FROM ${table} WHERE id = ?`
|
|
102
|
+
this.deferSql = `UPDATE ${table} SET visible_at = ? WHERE id = ?`
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Enqueue a message. `delaySeconds` becomes a visible-after deadline, exactly as on Cloudflare Queues. */
|
|
106
|
+
async send(message: T, options: { delaySeconds?: number } = {}): Promise<void> {
|
|
107
|
+
const now = this.now()
|
|
108
|
+
const visibleAt = now + Math.max(0, options.delaySeconds ?? 0) * 1000
|
|
109
|
+
await this.db
|
|
110
|
+
.prepare(this.sendSql)
|
|
111
|
+
// The id is minted on the queue's own clock, so id order and enqueue order agree — which is
|
|
112
|
+
// what `claim()` sorts a batch by. The generator is monotonic (RFC 9562 §6.2 counter), so
|
|
113
|
+
// two sends inside one millisecond still order by which was sent first.
|
|
114
|
+
.bind(uuidv7(now), this.queue, JSON.stringify(message), visibleAt, this.maxAttempts, now)
|
|
115
|
+
.run()
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Take up to `limit` due jobs and hide them for `visibilityTimeoutMs`. `decode` narrows the stored
|
|
120
|
+
* JSON at the deserialization boundary — a row this consumer cannot understand throws here rather
|
|
121
|
+
* than reaching a handler that assumes a shape.
|
|
122
|
+
*/
|
|
123
|
+
async claim(options: { limit: number; visibilityTimeoutMs: number; decode: (payload: unknown) => T }): Promise<Job<T>[]> {
|
|
124
|
+
const now = this.now()
|
|
125
|
+
const { results } = await this.db
|
|
126
|
+
.prepare(this.claimSql)
|
|
127
|
+
.bind(now + options.visibilityTimeoutMs, this.queue, now, options.limit)
|
|
128
|
+
.all<ClaimedRow>()
|
|
129
|
+
// Postgres does NOT promise that `RETURNING` follows the sub-select's `ORDER BY` — that clause
|
|
130
|
+
// only decides WHICH rows the LIMIT takes, not the order they come back in. Sorting by id here
|
|
131
|
+
// restores enqueue order, because ids are monotonic UUIDv7 and therefore strictly time-ordered.
|
|
132
|
+
return results
|
|
133
|
+
.sort((left, right) => (left.id < right.id ? -1 : 1))
|
|
134
|
+
.map((row) => {
|
|
135
|
+
const parsed: unknown = JSON.parse(row.payload)
|
|
136
|
+
return { id: row.id, payload: options.decode(parsed), attempts: row.attempts, maxAttempts: row.max_attempts }
|
|
137
|
+
})
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** The job is done (or permanently abandoned) — drop the row. Idempotent. */
|
|
141
|
+
async ack(id: string): Promise<void> {
|
|
142
|
+
await this.db.prepare(this.ackSql).bind(id).run()
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Make the job claimable again after `delayMs`. Used for retries and for a deferred (locked-out) run. */
|
|
146
|
+
async defer(id: string, delayMs: number): Promise<void> {
|
|
147
|
+
await this.db.prepare(this.deferSql).bind(this.now() + delayMs, id).run()
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export interface PostgresJobConsumerOptions<T> {
|
|
152
|
+
/** Narrow the stored JSON to the message type. Throwing here abandons the job as undecodable. */
|
|
153
|
+
decode: (payload: unknown) => T
|
|
154
|
+
/** Handle one job. A throw schedules a retry until `maxAttempts` is spent. */
|
|
155
|
+
handler: (job: Job<T>) => Promise<void>
|
|
156
|
+
/** Jobs claimed per pass. Handlers still run one at a time. */
|
|
157
|
+
batchSize?: number
|
|
158
|
+
/** Sleep between passes that found nothing. A pass that found work polls again immediately. */
|
|
159
|
+
pollIntervalMs?: number
|
|
160
|
+
/** How long a claimed job stays hidden. MUST exceed the slowest handler, or the job is redelivered mid-flight. */
|
|
161
|
+
visibilityTimeoutMs?: number
|
|
162
|
+
/** Backoff before the next attempt, from the attempt count just consumed. */
|
|
163
|
+
retryDelayMs?: (attempts: number) => number
|
|
164
|
+
/** A handler threw and the job will be retried. Never log the error object (it may carry a token). */
|
|
165
|
+
onError?: (error: unknown, job: Job<T>) => void
|
|
166
|
+
/** A job spent its last attempt and was dropped. */
|
|
167
|
+
onAbandoned?: (job: Job<T>) => void
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const DEFAULT_BATCH_SIZE = 1
|
|
171
|
+
const DEFAULT_POLL_INTERVAL_MS = 1_000
|
|
172
|
+
const DEFAULT_VISIBILITY_TIMEOUT_MS = 15 * 60 * 1_000
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The in-process poller. One pass = claim a batch, run the handlers in order, ack or reschedule each.
|
|
176
|
+
* Handlers run SEQUENTIALLY: a deploy consumer is one run per message and ordering per queue is the
|
|
177
|
+
* cheapest correct default; raise `batchSize` only for genuinely independent work.
|
|
178
|
+
*
|
|
179
|
+
* `start()` is fire-and-forget and never throws; `stop()` waits for the pass in flight, so a shutdown
|
|
180
|
+
* does not orphan a job it has already made invisible.
|
|
181
|
+
*/
|
|
182
|
+
export class PostgresJobConsumer<T> {
|
|
183
|
+
private readonly options: Required<Omit<PostgresJobConsumerOptions<T>, 'onError' | 'onAbandoned'>>
|
|
184
|
+
private readonly onError: (error: unknown, job: Job<T>) => void
|
|
185
|
+
private readonly onAbandoned: (job: Job<T>) => void
|
|
186
|
+
private running = false
|
|
187
|
+
private timer: ReturnType<typeof setTimeout> | null = null
|
|
188
|
+
private pass: Promise<void> = Promise.resolve()
|
|
189
|
+
|
|
190
|
+
constructor(private readonly queue: PostgresJobQueue<T>, options: PostgresJobConsumerOptions<T>) {
|
|
191
|
+
this.options = {
|
|
192
|
+
decode: options.decode,
|
|
193
|
+
handler: options.handler,
|
|
194
|
+
batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE,
|
|
195
|
+
pollIntervalMs: options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
|
|
196
|
+
visibilityTimeoutMs: options.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS,
|
|
197
|
+
retryDelayMs: options.retryDelayMs ?? ((attempts) => Math.min(60_000, 1_000 * 2 ** attempts)),
|
|
198
|
+
}
|
|
199
|
+
this.onError = options.onError ?? ((error) => {
|
|
200
|
+
console.error('job handler failed:', error instanceof Error ? error.message : 'unknown error')
|
|
201
|
+
})
|
|
202
|
+
this.onAbandoned = options.onAbandoned ?? ((job) => {
|
|
203
|
+
console.warn(`job ${job.id} abandoned after ${job.attempts} attempt(s)`)
|
|
204
|
+
})
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Run one pass. Returns how many jobs were handled — 0 means the queue was empty. Exposed for tests. */
|
|
208
|
+
async poll(): Promise<number> {
|
|
209
|
+
const jobs = await this.queue.claim({
|
|
210
|
+
limit: this.options.batchSize,
|
|
211
|
+
visibilityTimeoutMs: this.options.visibilityTimeoutMs,
|
|
212
|
+
decode: this.options.decode,
|
|
213
|
+
})
|
|
214
|
+
for (const job of jobs) {
|
|
215
|
+
await this.handle(job)
|
|
216
|
+
}
|
|
217
|
+
return jobs.length
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Start polling in the background. Idempotent. */
|
|
221
|
+
start(): void {
|
|
222
|
+
if (this.running) {
|
|
223
|
+
return
|
|
224
|
+
}
|
|
225
|
+
this.running = true
|
|
226
|
+
this.schedule(0)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** Stop polling and wait for the pass in flight. Idempotent. */
|
|
230
|
+
async stop(): Promise<void> {
|
|
231
|
+
this.running = false
|
|
232
|
+
if (this.timer !== null) {
|
|
233
|
+
clearTimeout(this.timer)
|
|
234
|
+
this.timer = null
|
|
235
|
+
}
|
|
236
|
+
await this.pass
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
private schedule(delayMs: number): void {
|
|
240
|
+
this.timer = setTimeout(() => {
|
|
241
|
+
this.timer = null
|
|
242
|
+
// One pass at a time: the next is scheduled only once this one has settled, so a slow handler
|
|
243
|
+
// can never have two passes claiming against the same visibility window.
|
|
244
|
+
this.pass = this.runPass()
|
|
245
|
+
}, delayMs)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
private async runPass(): Promise<void> {
|
|
249
|
+
let handled = 0
|
|
250
|
+
try {
|
|
251
|
+
handled = await this.poll()
|
|
252
|
+
} catch (error) {
|
|
253
|
+
// A claim failure is usually the database being unreachable — log short and back off.
|
|
254
|
+
console.error('job poll failed:', error instanceof Error ? error.message : 'unknown error')
|
|
255
|
+
}
|
|
256
|
+
if (this.running) {
|
|
257
|
+
this.schedule(handled > 0 ? 0 : this.options.pollIntervalMs)
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
private async handle(job: Job<T>): Promise<void> {
|
|
262
|
+
try {
|
|
263
|
+
await this.options.handler(job)
|
|
264
|
+
await this.queue.ack(job.id)
|
|
265
|
+
} catch (error) {
|
|
266
|
+
this.onError(error, job)
|
|
267
|
+
if (job.attempts >= job.maxAttempts) {
|
|
268
|
+
await this.queue.ack(job.id)
|
|
269
|
+
this.onAbandoned(job)
|
|
270
|
+
return
|
|
271
|
+
}
|
|
272
|
+
await this.queue.defer(job.id, this.options.retryDelayMs(job.attempts))
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|