@astralarium/repo-postgres 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -0
- package/dist/src/queue.d.ts +86 -0
- package/dist/src/queue.js +201 -0
- package/dist/src/schema/index.d.ts +1 -1
- package/dist/src/schema/index.js +1 -1
- package/dist/src/schema/modules.d.ts +9 -0
- package/dist/src/schema/modules.js +23 -0
- package/dist/testkit/index.d.ts +3 -2
- package/dist/testkit/index.js +1 -0
- package/dist/testkit/locks.d.ts +27 -0
- package/dist/testkit/locks.js +57 -0
- package/package.json +8 -7
- package/sql/slug/schema.sql +25 -0
- package/sql/taskclaim/jobs/schema.sql +19 -0
- package/sql/taskclaim/schema.sql +41 -0
- package/sql/taskclaim/scrub.sql +3 -0
package/README.md
CHANGED
|
@@ -8,3 +8,32 @@ Roles:
|
|
|
8
8
|
- `app_service`: app service.
|
|
9
9
|
|
|
10
10
|
Roles are cluster-wide: one app per cluster.
|
|
11
|
+
|
|
12
|
+
```sql
|
|
13
|
+
SELECT slug.slug('Room 52'); -- room-52
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
```ts
|
|
17
|
+
// app modules: `taskclaimJobs` (requires `taskclaim`, `cron`) deletes expired claims hourly
|
|
18
|
+
postgresDatabase({ modules: [appModule, slug, taskclaimJobs], name: "app-db" });
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
// producer: one message per key per window
|
|
23
|
+
await enqueueClaimed(
|
|
24
|
+
sql,
|
|
25
|
+
env.REFRESH,
|
|
26
|
+
`project:${id}`,
|
|
27
|
+
{ id },
|
|
28
|
+
{ windowSeconds: 300 },
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
// consumer
|
|
32
|
+
await consumeClaimed(sql, batch, {
|
|
33
|
+
parse: parseRefreshTask,
|
|
34
|
+
run: refreshProject,
|
|
35
|
+
maxRetries: 3, // the consumer's max_retries
|
|
36
|
+
retryAfterMs: (e) =>
|
|
37
|
+
e instanceof RetryAfterError ? e.retryAfterMs : undefined,
|
|
38
|
+
});
|
|
39
|
+
```
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { type Sql, type TransactionSql } from "./sql.ts";
|
|
2
|
+
/** A Cloudflare Queues producer binding, as `enqueueClaimed` sends. */
|
|
3
|
+
export interface Queue<Body> {
|
|
4
|
+
send(body: Body): Promise<unknown>;
|
|
5
|
+
}
|
|
6
|
+
/** A Cloudflare Queues message, as `consumeClaimed` settles it. */
|
|
7
|
+
export interface QueueMessage<Body> {
|
|
8
|
+
readonly body: Body;
|
|
9
|
+
/** Deliveries so far, this one included: 1 on the first. */
|
|
10
|
+
readonly attempts: number;
|
|
11
|
+
ack(): void;
|
|
12
|
+
retry(options?: {
|
|
13
|
+
delaySeconds?: number;
|
|
14
|
+
}): void;
|
|
15
|
+
}
|
|
16
|
+
/** A Cloudflare Queues consumer batch. */
|
|
17
|
+
export interface MessageBatch<Body> {
|
|
18
|
+
/** Queue name. */
|
|
19
|
+
readonly queue: string;
|
|
20
|
+
readonly messages: readonly QueueMessage<Body>[];
|
|
21
|
+
}
|
|
22
|
+
/** A claimed task's queue message body. */
|
|
23
|
+
export interface ClaimedTask<Task> {
|
|
24
|
+
/** Claim key (`claimTask`). */
|
|
25
|
+
key: string;
|
|
26
|
+
/** Claim token (`claimTask`). */
|
|
27
|
+
claim: string;
|
|
28
|
+
/** Claim window (`claimTask`); a held retry's claim ends one window past
|
|
29
|
+
* its due time. */
|
|
30
|
+
windowSeconds: number;
|
|
31
|
+
task: Task;
|
|
32
|
+
}
|
|
33
|
+
/** Claims `key` for `windowSeconds`: a fresh token, or undefined while an
|
|
34
|
+
* unexpired claim holds it. */
|
|
35
|
+
export declare function claimTask(sql: Sql | TransactionSql, key: string, { windowSeconds }: {
|
|
36
|
+
windowSeconds: number;
|
|
37
|
+
}): Promise<string | undefined>;
|
|
38
|
+
/** Holds `key`'s claim by `token` through a retry due in `delaySeconds`,
|
|
39
|
+
* then `windowSeconds` more, reclaiming a freed key; a later claim
|
|
40
|
+
* stays. */
|
|
41
|
+
export declare function holdTask(sql: Sql | TransactionSql, key: string, token: string, { delaySeconds, windowSeconds, }: {
|
|
42
|
+
delaySeconds: number;
|
|
43
|
+
windowSeconds: number;
|
|
44
|
+
}): Promise<void>;
|
|
45
|
+
/** Frees `key`'s claim by `token`; a later claim stays. */
|
|
46
|
+
export declare function releaseTask(sql: Sql | TransactionSql, key: string, token: string): Promise<void>;
|
|
47
|
+
/** Sends `task` to `queue` under a fresh claim of `key` (`claimTask`);
|
|
48
|
+
* resolves to the claim token, undefined while a claim holds the key.
|
|
49
|
+
* A failed send releases the claim and throws the send's error; a failed
|
|
50
|
+
* release logs, its claim held for the window. Takes a client outside any
|
|
51
|
+
* transaction: the claim commits before the send. */
|
|
52
|
+
export declare function enqueueClaimed<Task>(sql: Sql, queue: Queue<ClaimedTask<Task>>, key: string, task: Task, options: {
|
|
53
|
+
windowSeconds: number;
|
|
54
|
+
}): Promise<string | undefined>;
|
|
55
|
+
/** How `consumeClaimed` runs a batch's tasks. */
|
|
56
|
+
export interface ConsumeClaimedOptions<Task> {
|
|
57
|
+
/** The task a message carries; undefined or a throw for one it cannot
|
|
58
|
+
* run. */
|
|
59
|
+
parse: (task: unknown) => Task | undefined;
|
|
60
|
+
/** Runs one task; a throw fails the attempt. Idempotent: a task can run
|
|
61
|
+
* more than once. */
|
|
62
|
+
run: (task: Task) => Promise<void>;
|
|
63
|
+
/** The consumer's `max_retries`: retries after the first delivery, an
|
|
64
|
+
* integer from 0. */
|
|
65
|
+
maxRetries: number;
|
|
66
|
+
/** Milliseconds a `run` throw asks to wait (e.g. `Retry-After`): a finite
|
|
67
|
+
* number from 0, or undefined for none. Another value or a throw counts
|
|
68
|
+
* as none. */
|
|
69
|
+
retryAfterMs?: (exception: unknown) => number | undefined;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Queue consumer of claimed tasks, messages in order. A run acks, keeping
|
|
73
|
+
* its claim until the window ends: a trigger inside the window coalesces.
|
|
74
|
+
* A failure retries after 30 s, doubling each attempt, or a longer
|
|
75
|
+
* `retryAfterMs`, up to 12 hours, holding the claim through the retry
|
|
76
|
+
* (`holdTask`). The last failure logs, releases the claim and acks; so does
|
|
77
|
+
* a task `parse` rejects or throws on. A body lacking a claim logs and acks.
|
|
78
|
+
* A failed claim write logs and the delivery settles as decided: a failed
|
|
79
|
+
* hold retries, a failed release acks, the key claimed for the window. The
|
|
80
|
+
* batch continues. Delivery is at-least-once (a queue redelivery reruns a
|
|
81
|
+
* task): `run` must be idempotent. Takes a client
|
|
82
|
+
* outside any transaction: each claim write commits before its message
|
|
83
|
+
* settles. Rejects with a `RangeError` for a `maxRetries` other than an
|
|
84
|
+
* integer from 0.
|
|
85
|
+
*/
|
|
86
|
+
export declare function consumeClaimed<Task>(sql: Sql, batch: MessageBatch<unknown>, options: ConsumeClaimedOptions<Task>): Promise<void>;
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claimed tasks on Cloudflare Queues: a task key queues once per claim
|
|
3
|
+
* window, its claim (`taskclaim.claim`) held through retries. Needs the
|
|
4
|
+
* `taskclaim` schema module; `taskclaimJobs` deletes expired claims.
|
|
5
|
+
*
|
|
6
|
+
* `enqueueClaimed` and `consumeClaimed` take a client outside any
|
|
7
|
+
* transaction. `Sql` rejects a `begin` transaction's client; a reserved
|
|
8
|
+
* connection inside a manual `BEGIN` type-checks, so the requirement rests
|
|
9
|
+
* on the caller.
|
|
10
|
+
*/
|
|
11
|
+
import { log } from "@astralarium/repo-telemetry";
|
|
12
|
+
import { msInterval } from "./sql.js";
|
|
13
|
+
/** Claims `key` for `windowSeconds`: a fresh token, or undefined while an
|
|
14
|
+
* unexpired claim holds it. */
|
|
15
|
+
export async function claimTask(sql, key, { windowSeconds }) {
|
|
16
|
+
const [claimed] = await sql `
|
|
17
|
+
INSERT INTO taskclaim.claim AS c (key, expires_at, token)
|
|
18
|
+
VALUES (
|
|
19
|
+
${key},
|
|
20
|
+
CURRENT_TIMESTAMP + ${msInterval(sql, windowSeconds * 1000)},
|
|
21
|
+
gen_random_uuid()
|
|
22
|
+
)
|
|
23
|
+
ON CONFLICT (key) DO UPDATE SET
|
|
24
|
+
expires_at = EXCLUDED.expires_at,
|
|
25
|
+
token = EXCLUDED.token
|
|
26
|
+
WHERE c.expires_at < CURRENT_TIMESTAMP
|
|
27
|
+
RETURNING token
|
|
28
|
+
`;
|
|
29
|
+
return claimed?.token;
|
|
30
|
+
}
|
|
31
|
+
/** Holds `key`'s claim by `token` through a retry due in `delaySeconds`,
|
|
32
|
+
* then `windowSeconds` more, reclaiming a freed key; a later claim
|
|
33
|
+
* stays. */
|
|
34
|
+
export async function holdTask(sql, key, token, { delaySeconds, windowSeconds, }) {
|
|
35
|
+
const held = msInterval(sql, (delaySeconds + windowSeconds) * 1000);
|
|
36
|
+
await sql `
|
|
37
|
+
INSERT INTO taskclaim.claim AS c (key, expires_at, token)
|
|
38
|
+
VALUES (${key}, CURRENT_TIMESTAMP + ${held}, ${token})
|
|
39
|
+
ON CONFLICT (key) DO UPDATE SET
|
|
40
|
+
expires_at = EXCLUDED.expires_at
|
|
41
|
+
WHERE c.token = EXCLUDED.token
|
|
42
|
+
`;
|
|
43
|
+
}
|
|
44
|
+
/** Frees `key`'s claim by `token`; a later claim stays. */
|
|
45
|
+
export async function releaseTask(sql, key, token) {
|
|
46
|
+
await sql `
|
|
47
|
+
DELETE FROM taskclaim.claim
|
|
48
|
+
WHERE key = ${key} AND token = ${token}
|
|
49
|
+
`;
|
|
50
|
+
}
|
|
51
|
+
/** Sends `task` to `queue` under a fresh claim of `key` (`claimTask`);
|
|
52
|
+
* resolves to the claim token, undefined while a claim holds the key.
|
|
53
|
+
* A failed send releases the claim and throws the send's error; a failed
|
|
54
|
+
* release logs, its claim held for the window. Takes a client outside any
|
|
55
|
+
* transaction: the claim commits before the send. */
|
|
56
|
+
export async function enqueueClaimed(sql, queue, key, task, options) {
|
|
57
|
+
const claim = await claimTask(sql, key, options);
|
|
58
|
+
if (!claim)
|
|
59
|
+
return undefined;
|
|
60
|
+
try {
|
|
61
|
+
await queue.send({
|
|
62
|
+
key,
|
|
63
|
+
claim,
|
|
64
|
+
windowSeconds: options.windowSeconds,
|
|
65
|
+
task,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
catch (sendException) {
|
|
69
|
+
try {
|
|
70
|
+
await releaseTask(sql, key, claim);
|
|
71
|
+
}
|
|
72
|
+
catch (exception) {
|
|
73
|
+
log.error("queue.claim_release_failed", "task {key} send failed, then its claim release; key claimed for the window", { key, exception });
|
|
74
|
+
}
|
|
75
|
+
throw sendException;
|
|
76
|
+
}
|
|
77
|
+
return claim;
|
|
78
|
+
}
|
|
79
|
+
/** Longest delay of a queue retry: Cloudflare Queues' 12 hours. */
|
|
80
|
+
const MAX_RETRY_DELAY_SECONDS = 43_200;
|
|
81
|
+
/** Seconds before the retry of a task's `attempts`th failed delivery: 30,
|
|
82
|
+
* doubling each attempt, or a longer `retryAfterMs`; at most
|
|
83
|
+
* `MAX_RETRY_DELAY_SECONDS`. A non-finite or negative `retryAfterMs`
|
|
84
|
+
* counts as none. */
|
|
85
|
+
function retryDelaySeconds(attempts, retryAfterMs) {
|
|
86
|
+
const backoff = 30 * 2 ** (attempts - 1);
|
|
87
|
+
const retryAfter = retryAfterMs !== undefined &&
|
|
88
|
+
Number.isFinite(retryAfterMs) &&
|
|
89
|
+
retryAfterMs >= 0
|
|
90
|
+
? Math.ceil(retryAfterMs / 1000)
|
|
91
|
+
: 0;
|
|
92
|
+
return Math.min(Math.max(backoff, retryAfter), MAX_RETRY_DELAY_SECONDS);
|
|
93
|
+
}
|
|
94
|
+
/** Milliseconds `retryAfterMs` asks to wait after `exception`; undefined
|
|
95
|
+
* for none or a throw. */
|
|
96
|
+
function requestedWaitMs(retryAfterMs, exception) {
|
|
97
|
+
try {
|
|
98
|
+
return retryAfterMs?.(exception);
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
return undefined;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/** A claim token: a uuid, as `taskclaim.claim.token` stores it. */
|
|
105
|
+
const CLAIM_TOKEN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
106
|
+
/** `body` as a claimed task envelope; undefined for another shape. */
|
|
107
|
+
function envelopeOf(body) {
|
|
108
|
+
if (typeof body !== "object" || body === null)
|
|
109
|
+
return undefined;
|
|
110
|
+
const { key, claim, windowSeconds, task } = body;
|
|
111
|
+
return typeof key === "string" &&
|
|
112
|
+
typeof claim === "string" &&
|
|
113
|
+
CLAIM_TOKEN.test(claim) &&
|
|
114
|
+
typeof windowSeconds === "number" &&
|
|
115
|
+
Number.isFinite(windowSeconds) &&
|
|
116
|
+
windowSeconds >= 0
|
|
117
|
+
? { key, claim, windowSeconds, task }
|
|
118
|
+
: undefined;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Queue consumer of claimed tasks, messages in order. A run acks, keeping
|
|
122
|
+
* its claim until the window ends: a trigger inside the window coalesces.
|
|
123
|
+
* A failure retries after 30 s, doubling each attempt, or a longer
|
|
124
|
+
* `retryAfterMs`, up to 12 hours, holding the claim through the retry
|
|
125
|
+
* (`holdTask`). The last failure logs, releases the claim and acks; so does
|
|
126
|
+
* a task `parse` rejects or throws on. A body lacking a claim logs and acks.
|
|
127
|
+
* A failed claim write logs and the delivery settles as decided: a failed
|
|
128
|
+
* hold retries, a failed release acks, the key claimed for the window. The
|
|
129
|
+
* batch continues. Delivery is at-least-once (a queue redelivery reruns a
|
|
130
|
+
* task): `run` must be idempotent. Takes a client
|
|
131
|
+
* outside any transaction: each claim write commits before its message
|
|
132
|
+
* settles. Rejects with a `RangeError` for a `maxRetries` other than an
|
|
133
|
+
* integer from 0.
|
|
134
|
+
*/
|
|
135
|
+
export async function consumeClaimed(sql, batch, options) {
|
|
136
|
+
const { maxRetries } = options;
|
|
137
|
+
if (!(Number.isInteger(maxRetries) && maxRetries >= 0))
|
|
138
|
+
throw new RangeError(`maxRetries must be an integer from 0: ${maxRetries}`);
|
|
139
|
+
const { queue } = batch;
|
|
140
|
+
for (const message of batch.messages) {
|
|
141
|
+
const envelope = envelopeOf(message.body);
|
|
142
|
+
if (!envelope) {
|
|
143
|
+
log.warn("queue.message_malformed", "{queue} message lacks a task claim; acked", { queue });
|
|
144
|
+
message.ack();
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
const { key, claim, windowSeconds } = envelope;
|
|
148
|
+
const settlement = await attempt(queue, envelope, message, options);
|
|
149
|
+
try {
|
|
150
|
+
if (!settlement.ack) {
|
|
151
|
+
await holdTask(sql, key, claim, {
|
|
152
|
+
delaySeconds: settlement.delaySeconds,
|
|
153
|
+
windowSeconds,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
else if (settlement.release) {
|
|
157
|
+
await releaseTask(sql, key, claim);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
catch (exception) {
|
|
161
|
+
if (settlement.ack)
|
|
162
|
+
log.error("queue.claim_release_failed", "{queue} task {key} claim release failed; acked, key claimed for the window", { queue, key, exception });
|
|
163
|
+
else
|
|
164
|
+
log.error("queue.claim_hold_failed", "{queue} task {key} claim hold failed; retrying", { queue, key, exception });
|
|
165
|
+
}
|
|
166
|
+
if (settlement.ack)
|
|
167
|
+
message.ack();
|
|
168
|
+
else
|
|
169
|
+
message.retry({ delaySeconds: settlement.delaySeconds });
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
/** Parses and runs `envelope`'s task, logging an unparseable or finally
|
|
173
|
+
* failed one; resolves to the delivery's settlement. */
|
|
174
|
+
async function attempt(queue, { key, task: body }, { attempts }, { parse, run, maxRetries, retryAfterMs }) {
|
|
175
|
+
let task;
|
|
176
|
+
try {
|
|
177
|
+
task = parse(body);
|
|
178
|
+
}
|
|
179
|
+
catch (exception) {
|
|
180
|
+
log.warn("queue.task_unparseable", "{queue} task {key} unparseable; releasing its claim and acking", { queue, key, exception });
|
|
181
|
+
return { ack: true, release: true };
|
|
182
|
+
}
|
|
183
|
+
if (task === undefined) {
|
|
184
|
+
log.warn("queue.task_unparseable", "{queue} task {key} unparseable; releasing its claim and acking", { queue, key });
|
|
185
|
+
return { ack: true, release: true };
|
|
186
|
+
}
|
|
187
|
+
try {
|
|
188
|
+
await run(task);
|
|
189
|
+
return { ack: true, release: false };
|
|
190
|
+
}
|
|
191
|
+
catch (exception) {
|
|
192
|
+
if (attempts <= maxRetries) {
|
|
193
|
+
return {
|
|
194
|
+
ack: false,
|
|
195
|
+
delaySeconds: retryDelaySeconds(attempts, requestedWaitMs(retryAfterMs, exception)),
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
log.error("queue.task_failed", "{queue} task {key} failed after {attempts, plural, one {# attempt} other {# attempts}}", { queue, key, attempts, exception });
|
|
199
|
+
return { ack: true, release: true };
|
|
200
|
+
}
|
|
201
|
+
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
/** Schema modules: each package's slice of the app database. */
|
|
2
2
|
export { defineSchemaModule, type SchemaModule, type SchemaModuleDeclaration, } from "./module.ts";
|
|
3
|
-
export { appSchema, cron, partman, trgm } from "./modules.ts";
|
|
3
|
+
export { appSchema, cron, partman, slug, taskclaim, taskclaimJobs, trgm, } from "./modules.ts";
|
package/dist/src/schema/index.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
/** Schema modules: each package's slice of the app database. */
|
|
2
2
|
export { defineSchemaModule, } from "./module.js";
|
|
3
|
-
export { appSchema, cron, partman, trgm } from "./modules.js";
|
|
3
|
+
export { appSchema, cron, partman, slug, taskclaim, taskclaimJobs, trgm, } from "./modules.js";
|
|
@@ -3,6 +3,13 @@ export declare const appSchema: import("./module.ts").SchemaModule;
|
|
|
3
3
|
/** pg_partman in schema `partman`, granted to `app_admin`; `reset.sql`
|
|
4
4
|
* clears its config for the dropped schemas. */
|
|
5
5
|
export declare const partman: import("./module.ts").SchemaModule;
|
|
6
|
+
/** Schema `slug`: `slug.slug(name)`, a name's URL slug (NFC, lowercase
|
|
7
|
+
* alphanumerics, spaces as `-`), immutable. */
|
|
8
|
+
export declare const slug: import("./module.ts").SchemaModule;
|
|
9
|
+
/** Schema `taskclaim`: table `taskclaim.claim`, one claim per task key,
|
|
10
|
+
* for `@astralarium/repo-postgres/queue`, and `taskclaim.delete_expired()`;
|
|
11
|
+
* a snapshot's scrub empties it. */
|
|
12
|
+
export declare const taskclaim: import("./module.ts").SchemaModule;
|
|
6
13
|
/** pg_trgm, for trigram (name) search. */
|
|
7
14
|
export declare const trgm: import("./module.ts").SchemaModule;
|
|
8
15
|
/**
|
|
@@ -13,3 +20,5 @@ export declare const trgm: import("./module.ts").SchemaModule;
|
|
|
13
20
|
* needs it idle.
|
|
14
21
|
*/
|
|
15
22
|
export declare const cron: import("./module.ts").SchemaModule;
|
|
23
|
+
/** pg_cron job `taskclaim-ttl`: `taskclaim.delete_expired()` hourly. */
|
|
24
|
+
export declare const taskclaimJobs: import("./module.ts").SchemaModule;
|
|
@@ -20,6 +20,22 @@ export const partman = defineSchemaModule({
|
|
|
20
20
|
extensions: ["partman"],
|
|
21
21
|
beforeReset: "before-reset.sql",
|
|
22
22
|
});
|
|
23
|
+
/** Schema `slug`: `slug.slug(name)`, a name's URL slug (NFC, lowercase
|
|
24
|
+
* alphanumerics, spaces as `-`), immutable. */
|
|
25
|
+
export const slug = defineSchemaModule({
|
|
26
|
+
name: "slug",
|
|
27
|
+
root: resolve(SQL_DIR, "slug"),
|
|
28
|
+
files: ["schema.sql"],
|
|
29
|
+
});
|
|
30
|
+
/** Schema `taskclaim`: table `taskclaim.claim`, one claim per task key,
|
|
31
|
+
* for `@astralarium/repo-postgres/queue`, and `taskclaim.delete_expired()`;
|
|
32
|
+
* a snapshot's scrub empties it. */
|
|
33
|
+
export const taskclaim = defineSchemaModule({
|
|
34
|
+
name: "taskclaim",
|
|
35
|
+
root: resolve(SQL_DIR, "taskclaim"),
|
|
36
|
+
files: ["schema.sql"],
|
|
37
|
+
scrub: "scrub.sql",
|
|
38
|
+
});
|
|
23
39
|
/** pg_trgm, for trigram (name) search. */
|
|
24
40
|
export const trgm = defineSchemaModule({
|
|
25
41
|
name: "trgm",
|
|
@@ -46,3 +62,10 @@ export const cron = defineSchemaModule({
|
|
|
46
62
|
"cron.use_background_workers=on",
|
|
47
63
|
],
|
|
48
64
|
});
|
|
65
|
+
/** pg_cron job `taskclaim-ttl`: `taskclaim.delete_expired()` hourly. */
|
|
66
|
+
export const taskclaimJobs = defineSchemaModule({
|
|
67
|
+
name: "taskclaim-jobs",
|
|
68
|
+
root: resolve(SQL_DIR, "taskclaim/jobs"),
|
|
69
|
+
files: ["schema.sql"],
|
|
70
|
+
requires: [cron, taskclaim],
|
|
71
|
+
});
|
package/dist/testkit/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
/** Test support: transactions that roll back, and Vitest
|
|
2
|
-
* Postgres.js under workerd. */
|
|
1
|
+
/** Test support: transactions that roll back, advisory locks, and Vitest
|
|
2
|
+
* hooks for Postgres.js under workerd. */
|
|
3
3
|
import type { Sql, TransactionSql } from "../src/sql.ts";
|
|
4
|
+
export { type AdvisoryLock, backendPid, type HeldTransaction, holdTransaction, lockHolders, lockWaiters, } from "./locks.ts";
|
|
4
5
|
/** Runs `fn` in a transaction that always rolls back; resolves to its result. */
|
|
5
6
|
export declare function rolledBack<T>(sql: Sql, fn: (tx: TransactionSql) => Promise<T>): Promise<T>;
|
|
6
7
|
/** Vitest `onUnhandledError` dropping only Postgres.js's close/read race
|
package/dist/testkit/index.js
CHANGED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** Observing and holding Postgres advisory locks. */
|
|
2
|
+
import type { Fragment, Sql, TransactionSql } from "../src/sql.ts";
|
|
3
|
+
/** SQL of an advisory lock's bigint key, as its lock call computes it
|
|
4
|
+
* (e.g. `hashtextextended(name, 0)`). */
|
|
5
|
+
export type AdvisoryLock = (sql: Sql | TransactionSql) => Fragment;
|
|
6
|
+
/** A transaction `holdTransaction` keeps open. */
|
|
7
|
+
export interface HeldTransaction {
|
|
8
|
+
/** Backend pid of the holding session. */
|
|
9
|
+
holder: number;
|
|
10
|
+
/** Lets the transaction end. */
|
|
11
|
+
release: () => void;
|
|
12
|
+
/** The transaction's end. */
|
|
13
|
+
done: Promise<unknown>;
|
|
14
|
+
}
|
|
15
|
+
/** Backend pid of `sql`'s session. */
|
|
16
|
+
export declare function backendPid(sql: Sql | TransactionSql): Promise<number>;
|
|
17
|
+
/** Backend pids of the sessions waiting on `lock` behind session `holder`
|
|
18
|
+
* (a backend pid). */
|
|
19
|
+
export declare function lockWaiters(sql: Sql, lock: AdvisoryLock, holder: number): Promise<number[]>;
|
|
20
|
+
/** Backend pids of the sessions holding `lock`, in either mode. */
|
|
21
|
+
export declare function lockHolders(sql: Sql, lock: AdvisoryLock): Promise<number[]>;
|
|
22
|
+
/**
|
|
23
|
+
* A transaction holding its locks until `release`. `transaction` opens it
|
|
24
|
+
* and calls `hold` once it takes them; resolves once `hold` runs, and
|
|
25
|
+
* rejects when the transaction ends first.
|
|
26
|
+
*/
|
|
27
|
+
export declare function holdTransaction(transaction: (hold: (sql: TransactionSql) => Promise<void>) => Promise<unknown>): Promise<HeldTransaction>;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/** Backend pid of `sql`'s session. */
|
|
2
|
+
export async function backendPid(sql) {
|
|
3
|
+
const [row] = await sql `SELECT pg_backend_pid() AS pid`;
|
|
4
|
+
if (!row)
|
|
5
|
+
throw new Error("no backend pid");
|
|
6
|
+
return row.pid;
|
|
7
|
+
}
|
|
8
|
+
/** `pg_locks` condition: the row is of `lock`. */
|
|
9
|
+
function isLock(sql, lock) {
|
|
10
|
+
const key = lock(sql);
|
|
11
|
+
return sql `
|
|
12
|
+
locktype = 'advisory'
|
|
13
|
+
AND classid = ((${key} >> 32) & 4294967295)::oid
|
|
14
|
+
AND objid = (${key} & 4294967295)::oid
|
|
15
|
+
AND objsubid = 1
|
|
16
|
+
`;
|
|
17
|
+
}
|
|
18
|
+
/** Backend pids of the sessions waiting on `lock` behind session `holder`
|
|
19
|
+
* (a backend pid). */
|
|
20
|
+
export async function lockWaiters(sql, lock, holder) {
|
|
21
|
+
const rows = await sql `
|
|
22
|
+
SELECT pid FROM pg_locks
|
|
23
|
+
WHERE ${isLock(sql, lock)}
|
|
24
|
+
AND NOT granted
|
|
25
|
+
AND ${holder}::integer = ANY(pg_blocking_pids(pid))
|
|
26
|
+
`;
|
|
27
|
+
return rows.map((x) => x.pid);
|
|
28
|
+
}
|
|
29
|
+
/** Backend pids of the sessions holding `lock`, in either mode. */
|
|
30
|
+
export async function lockHolders(sql, lock) {
|
|
31
|
+
const rows = await sql `
|
|
32
|
+
SELECT pid FROM pg_locks
|
|
33
|
+
WHERE ${isLock(sql, lock)}
|
|
34
|
+
AND granted
|
|
35
|
+
`;
|
|
36
|
+
return rows.map((x) => x.pid);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* A transaction holding its locks until `release`. `transaction` opens it
|
|
40
|
+
* and calls `hold` once it takes them; resolves once `hold` runs, and
|
|
41
|
+
* rejects when the transaction ends first.
|
|
42
|
+
*/
|
|
43
|
+
export async function holdTransaction(transaction) {
|
|
44
|
+
let release = () => undefined;
|
|
45
|
+
const released = new Promise((resolve) => (release = resolve));
|
|
46
|
+
let taken = (_) => undefined;
|
|
47
|
+
const holding = new Promise((resolve) => (taken = resolve));
|
|
48
|
+
const done = transaction(async (sql) => {
|
|
49
|
+
taken(await backendPid(sql));
|
|
50
|
+
await released;
|
|
51
|
+
});
|
|
52
|
+
const holder = await Promise.race([
|
|
53
|
+
holding,
|
|
54
|
+
done.then(() => Promise.reject(new Error("transaction ended before holding"))),
|
|
55
|
+
]);
|
|
56
|
+
return { holder, release, done };
|
|
57
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@astralarium/repo-postgres",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"homepage": "https://github.com/astralarium/repo/tree/main/packages/repo-postgres#readme",
|
|
6
6
|
"repository": {
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
},
|
|
22
22
|
"exports": {
|
|
23
23
|
"./provision": "./dist/src/provision/index.js",
|
|
24
|
+
"./queue": "./dist/src/queue.js",
|
|
24
25
|
"./schema": "./dist/src/schema/index.js",
|
|
25
26
|
"./sql": "./dist/src/sql.js",
|
|
26
27
|
"./testkit": "./dist/testkit/index.js",
|
|
@@ -32,19 +33,19 @@
|
|
|
32
33
|
"dependencies": {
|
|
33
34
|
"postgres": "^3.4.9",
|
|
34
35
|
"zod": "^4.6.7",
|
|
35
|
-
"@astralarium/repo-telemetry": "^0.1.0",
|
|
36
36
|
"@astralarium/repo-package-files": "^0.1.0",
|
|
37
|
-
"@astralarium/repo-
|
|
38
|
-
"@astralarium/repo-
|
|
37
|
+
"@astralarium/repo-deploy-info": "^0.1.1",
|
|
38
|
+
"@astralarium/repo-dev": "^0.1.1",
|
|
39
|
+
"@astralarium/repo-telemetry": "^0.2.0"
|
|
39
40
|
},
|
|
40
41
|
"devDependencies": {
|
|
41
42
|
"@types/node": "^24.2.0",
|
|
42
43
|
"eslint": "^10.2.1",
|
|
43
44
|
"typescript": "^6.0.3",
|
|
44
45
|
"vitest": "^5.0.3",
|
|
45
|
-
"@astralarium/repo-config-
|
|
46
|
-
"@astralarium/repo-config-
|
|
47
|
-
"@astralarium/repo-config-
|
|
46
|
+
"@astralarium/repo-config-vitest": "^0.2.0",
|
|
47
|
+
"@astralarium/repo-config-eslint": "^0.3.0",
|
|
48
|
+
"@astralarium/repo-config-typescript": "^0.1.0"
|
|
48
49
|
},
|
|
49
50
|
"peerDependencies": {
|
|
50
51
|
"vitest": "^5.0.3"
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
-- `slug.slug(name)`: the URL slug of a name: lowercase letters and digits,
|
|
2
|
+
-- spaces as `-`, other characters dropped. `O'Brien` → `obrien`;
|
|
3
|
+
-- `Room 52` → `room-52`. Names normalize to NFC first: decomposed and
|
|
4
|
+
-- composed spellings slug alike. Case and letter classes follow Unicode
|
|
5
|
+
-- (`pg_c_utf8`), whatever the database's or argument's collation.
|
|
6
|
+
SET ROLE app_admin;
|
|
7
|
+
|
|
8
|
+
CREATE SCHEMA slug AUTHORIZATION app_admin;
|
|
9
|
+
|
|
10
|
+
GRANT USAGE ON SCHEMA slug TO app_service;
|
|
11
|
+
|
|
12
|
+
CREATE FUNCTION slug.slug(name text) RETURNS text
|
|
13
|
+
LANGUAGE SQL IMMUTABLE PARALLEL SAFE
|
|
14
|
+
RETURN replace(
|
|
15
|
+
regexp_replace(
|
|
16
|
+
lower(normalize(name, NFC) COLLATE pg_c_utf8),
|
|
17
|
+
'[^[:alnum:] ]',
|
|
18
|
+
'',
|
|
19
|
+
'g'
|
|
20
|
+
),
|
|
21
|
+
' ',
|
|
22
|
+
'-'
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
SET ROLE NONE;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
-- taskclaim's pg_cron job: hourly `taskclaim.delete_expired()`. Skipped
|
|
2
|
+
-- where pg_cron is not loaded.
|
|
3
|
+
DO $$
|
|
4
|
+
BEGIN
|
|
5
|
+
IF current_setting('cron.database_name', true) IS NULL THEN
|
|
6
|
+
RAISE NOTICE 'pg_cron is not loaded; skipping taskclaim jobs';
|
|
7
|
+
RETURN;
|
|
8
|
+
END IF;
|
|
9
|
+
|
|
10
|
+
PERFORM cron.schedule(
|
|
11
|
+
'taskclaim-ttl',
|
|
12
|
+
'15 * * * *',
|
|
13
|
+
$job$
|
|
14
|
+
SET statement_timeout = '30s';
|
|
15
|
+
SELECT taskclaim.delete_expired();
|
|
16
|
+
$job$
|
|
17
|
+
);
|
|
18
|
+
END;
|
|
19
|
+
$$;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
-- Task claims in schema `taskclaim`: one row per task key, de-duplicating
|
|
2
|
+
-- queued tasks (`@astralarium/repo-postgres/queue`), and expired-claim
|
|
3
|
+
-- cleanup.
|
|
4
|
+
SET ROLE app_admin;
|
|
5
|
+
|
|
6
|
+
CREATE SCHEMA taskclaim AUTHORIZATION app_admin;
|
|
7
|
+
|
|
8
|
+
GRANT USAGE ON SCHEMA taskclaim TO app_service;
|
|
9
|
+
|
|
10
|
+
CREATE TABLE taskclaim.claim
|
|
11
|
+
(
|
|
12
|
+
key text PRIMARY KEY,
|
|
13
|
+
-- End of the claim; past it, a claim of the key replaces the row.
|
|
14
|
+
expires_at timestamptz NOT NULL,
|
|
15
|
+
-- Token of the claiming task, carried in its queue message.
|
|
16
|
+
token uuid NOT NULL
|
|
17
|
+
);
|
|
18
|
+
CREATE INDEX claim_expires_at_idx ON taskclaim.claim (expires_at);
|
|
19
|
+
|
|
20
|
+
GRANT SELECT, INSERT, UPDATE, DELETE ON taskclaim.claim TO app_service;
|
|
21
|
+
|
|
22
|
+
-- Deletes expired claims; their count. Reserved for the owner;
|
|
23
|
+
-- `taskclaimJobs` schedules it.
|
|
24
|
+
CREATE FUNCTION taskclaim.delete_expired()
|
|
25
|
+
RETURNS bigint
|
|
26
|
+
LANGUAGE plpgsql
|
|
27
|
+
SET search_path = pg_catalog
|
|
28
|
+
AS $$
|
|
29
|
+
DECLARE
|
|
30
|
+
deleted bigint;
|
|
31
|
+
BEGIN
|
|
32
|
+
DELETE FROM taskclaim.claim WHERE expires_at < CURRENT_TIMESTAMP;
|
|
33
|
+
GET DIAGNOSTICS deleted = ROW_COUNT;
|
|
34
|
+
RETURN deleted;
|
|
35
|
+
END;
|
|
36
|
+
$$;
|
|
37
|
+
|
|
38
|
+
COMMENT ON FUNCTION taskclaim.delete_expired() IS 'Deletes expired claims.';
|
|
39
|
+
REVOKE ALL ON FUNCTION taskclaim.delete_expired() FROM PUBLIC;
|
|
40
|
+
|
|
41
|
+
SET ROLE NONE;
|