cairnq 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/dist/_protocol/sql/postgres/installations.sql +33 -0
- package/dist/context.d.ts +4 -2
- package/dist/context.js +12 -8
- package/dist/errors.d.ts +17 -0
- package/dist/errors.js +20 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models.js +27 -0
- package/dist/store/pg-executor.d.ts +6 -5
- package/dist/store/pg-executor.js +6 -5
- package/dist/store/pg-pool.js +6 -5
- package/dist/store/postgres.d.ts +23 -0
- package/dist/store/postgres.js +56 -0
- package/dist/worker.d.ts +4 -2
- package/dist/worker.js +10 -8
- package/package.json +1 -1
- package/src/context.ts +10 -6
- package/src/errors.ts +21 -0
- package/src/index.ts +1 -0
- package/src/models.ts +27 -0
- package/src/store/pg-executor.ts +6 -5
- package/src/store/pg-pool.ts +6 -5
- package/src/store/postgres.ts +61 -0
- package/src/worker.ts +10 -8
package/README.md
CHANGED
|
@@ -114,10 +114,18 @@ const executor: PgExecutor = { /* ~30 lines over your driver */ };
|
|
|
114
114
|
const tasks = CairnQ.postgres(executor);
|
|
115
115
|
```
|
|
116
116
|
|
|
117
|
-
`schema`
|
|
117
|
+
`schema` puts cairnq's tables in a schema of their own:
|
|
118
118
|
`CairnQ.postgres(dsn, { schema: "cairnq" })` creates it if absent and sets
|
|
119
119
|
`search_path` per connection. The protocol's SQL names no schema, so nothing else
|
|
120
|
-
changes. With your own executor
|
|
120
|
+
changes. With your own executor the search_path is yours to set, and `schema`
|
|
121
|
+
becomes an assertion about where it lands.
|
|
122
|
+
|
|
123
|
+
**Every process in a deployment must agree on it.** A queue whose API and worker
|
|
124
|
+
resolve to different schemas is two empty queues, and — because every migration is
|
|
125
|
+
`create table if not exists` — both sides come up healthy, pass their protocol
|
|
126
|
+
version check, and never see each other's tasks. cairnq refuses to connect where
|
|
127
|
+
it can see that about to happen (`SchemaMismatch`); the Python SDK applies the
|
|
128
|
+
same rule.
|
|
121
129
|
|
|
122
130
|
Two things an adapter must get right: `int8` has to come back as a JS number
|
|
123
131
|
(every cairnq `*_ms` is an epoch or a counter, all inside the safe range), and
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
-- Where cairnq already lives in this database, and where this connection is
|
|
2
|
+
-- pointing. Read-only; runs once per connect, before migrations.
|
|
3
|
+
--
|
|
4
|
+
-- Exists because `search_path` is out-of-band configuration: two processes given
|
|
5
|
+
-- the same DSN can still resolve to different schemas, and because every
|
|
6
|
+
-- migration is `create table if not exists`, the second one to start does not
|
|
7
|
+
-- fail — it quietly builds a parallel, empty installation. Nothing downstream can
|
|
8
|
+
-- tell: protocol_version reads from whichever cairnq_meta the connection sees, so
|
|
9
|
+
-- the version check passes on both sides while the API's tasks are invisible to
|
|
10
|
+
-- the worker forever. The only way to catch that is to look OUTSIDE the
|
|
11
|
+
-- connection's own search_path, which is what this does.
|
|
12
|
+
--
|
|
13
|
+
-- One row per installation, never zero: the LEFT JOIN keeps `current_schema`
|
|
14
|
+
-- readable on a database that holds no cairnq yet, where `schema` is null.
|
|
15
|
+
-- Deliberately NOT an array column — pg_namespace.nspname is `name`, and which
|
|
16
|
+
-- drivers decode a `name[]` (or a text[]) into a list is exactly the kind of
|
|
17
|
+
-- disagreement this protocol keeps out of the SDKs. Scalar columns behave the
|
|
18
|
+
-- same everywhere.
|
|
19
|
+
--
|
|
20
|
+
-- `current_schema()` is null when the search_path names nothing that exists, in
|
|
21
|
+
-- which case the caller cannot conclude anything.
|
|
22
|
+
-- params: (none)
|
|
23
|
+
select
|
|
24
|
+
current_schema()::text as current_schema,
|
|
25
|
+
found.schema
|
|
26
|
+
from (select 1) one
|
|
27
|
+
left join (
|
|
28
|
+
select n.nspname::text as schema
|
|
29
|
+
from pg_class c
|
|
30
|
+
join pg_namespace n on n.oid = c.relnamespace
|
|
31
|
+
where c.relname = 'cairnq_tasks' and c.relkind = 'r'
|
|
32
|
+
) found on true
|
|
33
|
+
order by found.schema;
|
package/dist/context.d.ts
CHANGED
|
@@ -19,7 +19,7 @@ export interface TaskContextOptions {
|
|
|
19
19
|
export declare class TaskContext {
|
|
20
20
|
private readonly store;
|
|
21
21
|
private readonly task;
|
|
22
|
-
readonly
|
|
22
|
+
private readonly ownerId;
|
|
23
23
|
private readonly leaseMs;
|
|
24
24
|
private readonly abort;
|
|
25
25
|
private leaseLost;
|
|
@@ -27,8 +27,10 @@ export declare class TaskContext {
|
|
|
27
27
|
private isSettled;
|
|
28
28
|
private readonly backoffMs;
|
|
29
29
|
private readonly backoffMaxMs;
|
|
30
|
-
constructor(store: TaskStore, task: Task,
|
|
30
|
+
constructor(store: TaskStore, task: Task, ownerId: string, leaseMs: number, opts?: TaskContextOptions);
|
|
31
31
|
get taskId(): string;
|
|
32
|
+
/** The worker running this task — what `worker_id` on the row points at. */
|
|
33
|
+
get workerId(): string;
|
|
32
34
|
get name(): string;
|
|
33
35
|
get queue(): string;
|
|
34
36
|
get attempt(): number;
|
package/dist/context.js
CHANGED
|
@@ -14,7 +14,7 @@ import { pollWait } from "./wait.js";
|
|
|
14
14
|
export class TaskContext {
|
|
15
15
|
store;
|
|
16
16
|
task;
|
|
17
|
-
|
|
17
|
+
ownerId;
|
|
18
18
|
leaseMs;
|
|
19
19
|
abort = new AbortController();
|
|
20
20
|
leaseLost = false;
|
|
@@ -28,10 +28,10 @@ export class TaskContext {
|
|
|
28
28
|
isSettled = false;
|
|
29
29
|
backoffMs;
|
|
30
30
|
backoffMaxMs;
|
|
31
|
-
constructor(store, task,
|
|
31
|
+
constructor(store, task, ownerId, leaseMs, opts = {}) {
|
|
32
32
|
this.store = store;
|
|
33
33
|
this.task = task;
|
|
34
|
-
this.
|
|
34
|
+
this.ownerId = ownerId;
|
|
35
35
|
this.leaseMs = leaseMs;
|
|
36
36
|
this.backoffMs = opts.retryBackoffMs ?? DEFAULT_RETRY_BACKOFF_MS;
|
|
37
37
|
this.backoffMaxMs = opts.retryBackoffMaxMs ?? DEFAULT_RETRY_BACKOFF_MAX_MS;
|
|
@@ -39,6 +39,10 @@ export class TaskContext {
|
|
|
39
39
|
get taskId() {
|
|
40
40
|
return this.task.id;
|
|
41
41
|
}
|
|
42
|
+
/** The worker running this task — what `worker_id` on the row points at. */
|
|
43
|
+
get workerId() {
|
|
44
|
+
return this.ownerId;
|
|
45
|
+
}
|
|
42
46
|
get name() {
|
|
43
47
|
return this.task.name;
|
|
44
48
|
}
|
|
@@ -138,7 +142,7 @@ export class TaskContext {
|
|
|
138
142
|
async progress(value, message = null) {
|
|
139
143
|
return this.owned(() => this.store.progress({
|
|
140
144
|
taskId: this.task.id,
|
|
141
|
-
workerId: this.
|
|
145
|
+
workerId: this.ownerId,
|
|
142
146
|
progress: value,
|
|
143
147
|
message,
|
|
144
148
|
}));
|
|
@@ -146,7 +150,7 @@ export class TaskContext {
|
|
|
146
150
|
async heartbeat() {
|
|
147
151
|
return this.owned(() => this.store.heartbeat({
|
|
148
152
|
taskId: this.task.id,
|
|
149
|
-
workerId: this.
|
|
153
|
+
workerId: this.ownerId,
|
|
150
154
|
leaseMs: this.leaseMs,
|
|
151
155
|
}));
|
|
152
156
|
}
|
|
@@ -180,7 +184,7 @@ export class TaskContext {
|
|
|
180
184
|
async succeed(result = null) {
|
|
181
185
|
if (this.isSettled)
|
|
182
186
|
return null;
|
|
183
|
-
const task = await this.owned(() => this.store.complete({ taskId: this.task.id, workerId: this.
|
|
187
|
+
const task = await this.owned(() => this.store.complete({ taskId: this.task.id, workerId: this.ownerId, result }));
|
|
184
188
|
this.markSettled();
|
|
185
189
|
return task;
|
|
186
190
|
}
|
|
@@ -211,7 +215,7 @@ export class TaskContext {
|
|
|
211
215
|
if (this.isSettled)
|
|
212
216
|
return null;
|
|
213
217
|
const task = await this.owned(async () => {
|
|
214
|
-
const { task } = await this.store.completeIn({ taskId: this.task.id, workerId: this.
|
|
218
|
+
const { task } = await this.store.completeIn({ taskId: this.task.id, workerId: this.ownerId }, write);
|
|
215
219
|
return task;
|
|
216
220
|
});
|
|
217
221
|
this.markSettled();
|
|
@@ -229,7 +233,7 @@ export class TaskContext {
|
|
|
229
233
|
const [envelope, retryable] = asEnvelope(error, opts.retryable ?? true);
|
|
230
234
|
const task = await this.owned(() => this.store.fail({
|
|
231
235
|
taskId: this.task.id,
|
|
232
|
-
workerId: this.
|
|
236
|
+
workerId: this.ownerId,
|
|
233
237
|
error: envelope,
|
|
234
238
|
retryable,
|
|
235
239
|
delayMs: failDelayMs(this.task.attempt, retryable, this.backoffMs, this.backoffMaxMs),
|
package/dist/errors.d.ts
CHANGED
|
@@ -101,6 +101,23 @@ export declare class LostLease extends CairnQError {
|
|
|
101
101
|
export declare class ProtocolVersionMismatch extends CairnQError {
|
|
102
102
|
constructor(message: string);
|
|
103
103
|
}
|
|
104
|
+
/**
|
|
105
|
+
* This connection is not pointed at the cairnq installation the rest of the
|
|
106
|
+
* deployment is using — raised at connect, before any task is written.
|
|
107
|
+
*
|
|
108
|
+
* The schema a Postgres connection resolves to is out-of-band configuration
|
|
109
|
+
* (`search_path`, a `schema` option, an ORM's pool settings), so two processes
|
|
110
|
+
* given the same DSN can still land in different schemas. Every migration is
|
|
111
|
+
* `create table if not exists`, so the odd one out does not fail: it builds a
|
|
112
|
+
* second, empty installation and its protocol version check passes against the
|
|
113
|
+
* cairnq_meta it just created. Left undetected, an API and a worker then agree
|
|
114
|
+
* about everything except WHERE, and no task ever crosses.
|
|
115
|
+
*
|
|
116
|
+
* The Python SDK raises the same named error.
|
|
117
|
+
*/
|
|
118
|
+
export declare class SchemaMismatch extends CairnQError {
|
|
119
|
+
constructor(message: string);
|
|
120
|
+
}
|
|
104
121
|
/** A value could not be encoded for a protocol JSON column (non-finite number,
|
|
105
122
|
* BigInt, circular structure, …). Raised at the boundary — submit rejects with
|
|
106
123
|
* it, and a worker records a handler result that triggers it as a permanent
|
package/dist/errors.js
CHANGED
|
@@ -187,6 +187,26 @@ export class ProtocolVersionMismatch extends CairnQError {
|
|
|
187
187
|
this.name = "ProtocolVersionMismatch";
|
|
188
188
|
}
|
|
189
189
|
}
|
|
190
|
+
/**
|
|
191
|
+
* This connection is not pointed at the cairnq installation the rest of the
|
|
192
|
+
* deployment is using — raised at connect, before any task is written.
|
|
193
|
+
*
|
|
194
|
+
* The schema a Postgres connection resolves to is out-of-band configuration
|
|
195
|
+
* (`search_path`, a `schema` option, an ORM's pool settings), so two processes
|
|
196
|
+
* given the same DSN can still land in different schemas. Every migration is
|
|
197
|
+
* `create table if not exists`, so the odd one out does not fail: it builds a
|
|
198
|
+
* second, empty installation and its protocol version check passes against the
|
|
199
|
+
* cairnq_meta it just created. Left undetected, an API and a worker then agree
|
|
200
|
+
* about everything except WHERE, and no task ever crosses.
|
|
201
|
+
*
|
|
202
|
+
* The Python SDK raises the same named error.
|
|
203
|
+
*/
|
|
204
|
+
export class SchemaMismatch extends CairnQError {
|
|
205
|
+
constructor(message) {
|
|
206
|
+
super(message);
|
|
207
|
+
this.name = "SchemaMismatch";
|
|
208
|
+
}
|
|
209
|
+
}
|
|
190
210
|
/** A value could not be encoded for a protocol JSON column (non-finite number,
|
|
191
211
|
* BigInt, circular structure, …). Raised at the boundary — submit rejects with
|
|
192
212
|
* it, and a worker records a handler result that triggers it as a permanent
|
package/dist/index.d.ts
CHANGED
|
@@ -20,5 +20,5 @@ export type { ListInput, PurgeInput, SubmitInput, Conflict, WatchOptions, WatchS
|
|
|
20
20
|
export { DEFAULT_WATCH_POLL_MS } from "./store/base.js";
|
|
21
21
|
export type { Task, TaskRef, TaskStatus, TerminalStatus } from "./models.js";
|
|
22
22
|
export { STATUSES, isTerminal, isTerminalStatus, cancelRequested, isQueued, isRunning, isSucceeded, isFailed, isCanceled, } from "./models.js";
|
|
23
|
-
export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SerializationError, } from "./errors.js";
|
|
23
|
+
export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SchemaMismatch, SerializationError, } from "./errors.js";
|
|
24
24
|
export type { FailReason } from "./errors.js";
|
package/dist/index.js
CHANGED
|
@@ -11,4 +11,4 @@ export { createPoolExecutor } from "./store/pg-pool.js";
|
|
|
11
11
|
export { TaskStore } from "./store/base.js";
|
|
12
12
|
export { DEFAULT_WATCH_POLL_MS } from "./store/base.js";
|
|
13
13
|
export { STATUSES, isTerminal, isTerminalStatus, cancelRequested, isQueued, isRunning, isSucceeded, isFailed, isCanceled, } from "./models.js";
|
|
14
|
-
export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SerializationError, } from "./errors.js";
|
|
14
|
+
export { CairnQError, AlreadyExists, QueueFull, TaskTimeout, TaskFailed, TaskCanceled, TaskError, LostLease, EventLoopBlocked, ProtocolVersionMismatch, SchemaMismatch, SerializationError, } from "./errors.js";
|
package/dist/models.js
CHANGED
|
@@ -4,6 +4,19 @@
|
|
|
4
4
|
// conformance suite pins this set against.
|
|
5
5
|
export const STATUSES = ["queued", "running", "succeeded", "failed", "canceled"];
|
|
6
6
|
const JSON_COLUMNS = ["payload", "result", "error", "metadata"];
|
|
7
|
+
// The bigint columns. `attempt` / `max_attempts` / `priority` are int4 and
|
|
8
|
+
// `progress` is double precision, so every driver already gives those as numbers;
|
|
9
|
+
// only int8 has a wire form worth normalizing. Nullability differs per column
|
|
10
|
+
// (completed_at_ms may be null, created_at_ms may not), so the coercion has to
|
|
11
|
+
// preserve null rather than turn it into 0.
|
|
12
|
+
const MS_COLUMNS = [
|
|
13
|
+
"lease_until_ms",
|
|
14
|
+
"run_at_ms",
|
|
15
|
+
"cancel_requested_at_ms",
|
|
16
|
+
"created_at_ms",
|
|
17
|
+
"updated_at_ms",
|
|
18
|
+
"completed_at_ms",
|
|
19
|
+
];
|
|
7
20
|
// As a const tuple so TerminalStatus derives from it — the same declare-once
|
|
8
21
|
// pattern as STATUSES/TaskStatus above.
|
|
9
22
|
export const TERMINAL = ["succeeded", "failed", "canceled"];
|
|
@@ -24,6 +37,20 @@ export function rowToTask(row) {
|
|
|
24
37
|
// already-decoded object. Parse only a string — never assume one backend.
|
|
25
38
|
t[col] = typeof v === "string" ? JSON.parse(v) : (v ?? null);
|
|
26
39
|
}
|
|
40
|
+
for (const col of MS_COLUMNS) {
|
|
41
|
+
const v = row[col];
|
|
42
|
+
// Same argument, for the other column type the drivers disagree about.
|
|
43
|
+
// Postgres sends int8 down the wire as text to protect precision it cannot
|
|
44
|
+
// know is unneeded; `pg` surfaces that as a string, postgres.js does too,
|
|
45
|
+
// asyncpg decodes to int. Normalizing here rather than demanding it of every
|
|
46
|
+
// driver is what keeps an INJECTED executor honest: the alternative is
|
|
47
|
+
// asking an application to change its driver's global int8 handling to suit
|
|
48
|
+
// cairnq, which breaks that application's own bigint columns.
|
|
49
|
+
//
|
|
50
|
+
// Lossless: every one of these is an epoch-ms, and a millisecond timestamp
|
|
51
|
+
// does not reach Number.MAX_SAFE_INTEGER until the year 287396.
|
|
52
|
+
t[col] = v == null ? null : Number(v);
|
|
53
|
+
}
|
|
27
54
|
return t;
|
|
28
55
|
}
|
|
29
56
|
/** Accepts anything carrying a status — a Task or a TaskRef probe. */
|
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
* instead of opening a second one, which is what makes a task's settlement
|
|
9
9
|
* commit in the same transaction as the rows the task produced.
|
|
10
10
|
*
|
|
11
|
-
* Implementing one is small — see `createPoolExecutor`
|
|
12
|
-
* implementation over `pg
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* Implementing one is small — see `createPoolExecutor` in pg-pool.ts for the
|
|
12
|
+
* reference implementation over `pg`. An adapter passes rows through as its
|
|
13
|
+
* driver produced them: cairnq normalizes both column types the drivers disagree
|
|
14
|
+
* about (jsonb decoded or not, int8 as text or number) in `rowToTask`, so no
|
|
15
|
+
* adapter has to reconfigure its driver — and none has to change how the
|
|
16
|
+
* application's OWN columns come back in order to satisfy cairnq.
|
|
16
17
|
*/
|
|
17
18
|
/** A row as the driver hands it back: column name -> value. */
|
|
18
19
|
export type Row = Record<string, unknown>;
|
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
* instead of opening a second one, which is what makes a task's settlement
|
|
9
9
|
* commit in the same transaction as the rows the task produced.
|
|
10
10
|
*
|
|
11
|
-
* Implementing one is small — see `createPoolExecutor`
|
|
12
|
-
* implementation over `pg
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* Implementing one is small — see `createPoolExecutor` in pg-pool.ts for the
|
|
12
|
+
* reference implementation over `pg`. An adapter passes rows through as its
|
|
13
|
+
* driver produced them: cairnq normalizes both column types the drivers disagree
|
|
14
|
+
* about (jsonb decoded or not, int8 as text or number) in `rowToTask`, so no
|
|
15
|
+
* adapter has to reconfigure its driver — and none has to change how the
|
|
16
|
+
* application's OWN columns come back in order to satisfy cairnq.
|
|
16
17
|
*/
|
|
17
18
|
/**
|
|
18
19
|
* LISTEN will not work on this connection, and retrying cannot change that.
|
package/dist/store/pg-pool.js
CHANGED
|
@@ -14,11 +14,12 @@ async function loadPg() {
|
|
|
14
14
|
throw new Error("PostgresStore requires the 'pg' package — install it (e.g. `npm i pg`)");
|
|
15
15
|
}
|
|
16
16
|
const pg = (mod.default ?? mod);
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
|
|
17
|
+
// Deliberately NOT setting a global int8 type parser here. It would be the
|
|
18
|
+
// shortest fix for Postgres sending bigint as text, and it is what this file
|
|
19
|
+
// used to do — but pg's type parsers are process-global, so a library that
|
|
20
|
+
// installs one silently changes how the APPLICATION's own bigint columns come
|
|
21
|
+
// back. rowToTask normalizes instead, which costs nothing and leaves the
|
|
22
|
+
// caller's driver as they configured it.
|
|
22
23
|
pgModule = pg;
|
|
23
24
|
return pg;
|
|
24
25
|
}
|
package/dist/store/postgres.d.ts
CHANGED
|
@@ -83,6 +83,29 @@ export declare class PostgresStore extends TaskStore {
|
|
|
83
83
|
close(): Promise<void>;
|
|
84
84
|
private ensure;
|
|
85
85
|
private doConnect;
|
|
86
|
+
/**
|
|
87
|
+
* Refuse a connection pointed somewhere other than the deployment's cairnq.
|
|
88
|
+
*
|
|
89
|
+
* Two shapes, because `schema` means "the schema cairnq's tables live in" and
|
|
90
|
+
* cairnq can either arrange that (it built the connection) or only check it
|
|
91
|
+
* (the caller's executor did):
|
|
92
|
+
*
|
|
93
|
+
* - `schema` configured -> assert the connection actually resolves there. On
|
|
94
|
+
* the DSN path this is a cheap self-check; on an injected executor it is the
|
|
95
|
+
* only way to state the expectation at all.
|
|
96
|
+
* - `schema` not configured -> the dangerous case is being about to create a
|
|
97
|
+
* SECOND installation while one already exists elsewhere in this database,
|
|
98
|
+
* which is exactly what a mismatched pair of SDKs does. Joining an existing
|
|
99
|
+
* installation is fine no matter what else is around, so the check is
|
|
100
|
+
* deliberately narrow: it fires only when this schema has no cairnq and some
|
|
101
|
+
* other schema does.
|
|
102
|
+
*
|
|
103
|
+
* That narrowness is what keeps it from crying wolf. Two applications each
|
|
104
|
+
* running their own cairnq in their own schema are legitimate; the second one
|
|
105
|
+
* to be set up trips this once, and saying `schema` explicitly — which such a
|
|
106
|
+
* deployment should be doing anyway — is both the fix and the confirmation.
|
|
107
|
+
*/
|
|
108
|
+
private checkSchema;
|
|
86
109
|
private applyMigrations;
|
|
87
110
|
private readProtocolVersion;
|
|
88
111
|
protocolVersion(): Promise<number>;
|
package/dist/store/postgres.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { SchemaMismatch } from "../errors.js";
|
|
1
2
|
import { loadMigrations, loadStatements } from "../sql.js";
|
|
2
3
|
import { checkProtocolVersion, COMMENT, NAMED, statementParams, TaskStore, } from "./base.js";
|
|
3
4
|
import { ListenUnavailable } from "./pg-executor.js";
|
|
@@ -139,6 +140,9 @@ export class PostgresStore extends TaskStore {
|
|
|
139
140
|
const executor = this.provided ??
|
|
140
141
|
(await createPoolExecutor(this.dsn, { max: this.opts.max, schema: this.opts.schema }));
|
|
141
142
|
try {
|
|
143
|
+
// Before migrations, which would otherwise create the very installation
|
|
144
|
+
// this is trying to warn about.
|
|
145
|
+
await this.checkSchema(executor);
|
|
142
146
|
await this.applyMigrations(executor);
|
|
143
147
|
checkProtocolVersion(await this.readProtocolVersion(executor));
|
|
144
148
|
}
|
|
@@ -153,6 +157,58 @@ export class PostgresStore extends TaskStore {
|
|
|
153
157
|
// already wakeable. Fire-and-forget: failure just means polling.
|
|
154
158
|
this.listenerReady();
|
|
155
159
|
}
|
|
160
|
+
/**
|
|
161
|
+
* Refuse a connection pointed somewhere other than the deployment's cairnq.
|
|
162
|
+
*
|
|
163
|
+
* Two shapes, because `schema` means "the schema cairnq's tables live in" and
|
|
164
|
+
* cairnq can either arrange that (it built the connection) or only check it
|
|
165
|
+
* (the caller's executor did):
|
|
166
|
+
*
|
|
167
|
+
* - `schema` configured -> assert the connection actually resolves there. On
|
|
168
|
+
* the DSN path this is a cheap self-check; on an injected executor it is the
|
|
169
|
+
* only way to state the expectation at all.
|
|
170
|
+
* - `schema` not configured -> the dangerous case is being about to create a
|
|
171
|
+
* SECOND installation while one already exists elsewhere in this database,
|
|
172
|
+
* which is exactly what a mismatched pair of SDKs does. Joining an existing
|
|
173
|
+
* installation is fine no matter what else is around, so the check is
|
|
174
|
+
* deliberately narrow: it fires only when this schema has no cairnq and some
|
|
175
|
+
* other schema does.
|
|
176
|
+
*
|
|
177
|
+
* That narrowness is what keeps it from crying wolf. Two applications each
|
|
178
|
+
* running their own cairnq in their own schema are legitimate; the second one
|
|
179
|
+
* to be set up trips this once, and saying `schema` explicitly — which such a
|
|
180
|
+
* deployment should be doing anyway — is both the fix and the confirmation.
|
|
181
|
+
*/
|
|
182
|
+
async checkSchema(executor) {
|
|
183
|
+
// One row per installation; the statement's LEFT JOIN guarantees at least
|
|
184
|
+
// one, so current_schema is readable even where cairnq lives nowhere yet.
|
|
185
|
+
const rows = await executor.query(this.statements.installations, []);
|
|
186
|
+
const current = rows[0]?.current_schema ?? null;
|
|
187
|
+
const installations = rows
|
|
188
|
+
.map((r) => r.schema)
|
|
189
|
+
.filter((s) => s != null);
|
|
190
|
+
const wanted = this.opts.schema;
|
|
191
|
+
if (wanted != null) {
|
|
192
|
+
if (current !== wanted) {
|
|
193
|
+
throw new SchemaMismatch(`cairnq is configured for schema ${JSON.stringify(wanted)} but this ` +
|
|
194
|
+
`connection resolves to ${JSON.stringify(current)} — check the ` +
|
|
195
|
+
`connection's search_path`);
|
|
196
|
+
}
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
// A search_path naming nothing that exists: there is no "here" to compare
|
|
200
|
+
// against, and the migrations are about to fail with a clearer message.
|
|
201
|
+
if (current === null)
|
|
202
|
+
return;
|
|
203
|
+
if (installations.length === 0 || installations.includes(current))
|
|
204
|
+
return;
|
|
205
|
+
throw new SchemaMismatch(`cairnq tables already exist in schema ${installations.map((s) => JSON.stringify(s)).join(", ")} ` +
|
|
206
|
+
`of this database, but this connection resolves to ${JSON.stringify(current)}, where there are ` +
|
|
207
|
+
`none. Connecting would create a second, parallel installation that the other one can never see ` +
|
|
208
|
+
`— an API and a worker split this way agree about everything except where, and no task crosses. ` +
|
|
209
|
+
`Point this process at the same schema (\`schema\` option, or \`options=-c search_path=...\` in the ` +
|
|
210
|
+
`DSN), or pass \`schema\` explicitly to confirm a separate installation is what you meant.`);
|
|
211
|
+
}
|
|
156
212
|
async applyMigrations(executor) {
|
|
157
213
|
await executor.exec("create table if not exists cairnq_migrations " +
|
|
158
214
|
"(name text primary key, applied_at_ms bigint not null)");
|
package/dist/worker.d.ts
CHANGED
|
@@ -98,7 +98,7 @@ export declare class Worker {
|
|
|
98
98
|
private readonly queues;
|
|
99
99
|
private readonly opts;
|
|
100
100
|
private readonly handlers;
|
|
101
|
-
private readonly
|
|
101
|
+
private readonly ownId;
|
|
102
102
|
/** Payload bytes charged to running handlers — see maxInFlightBytes. */
|
|
103
103
|
private inFlightBytes;
|
|
104
104
|
/** Calls in flight, for the names that cap their own concurrency. */
|
|
@@ -139,7 +139,9 @@ export declare class Worker {
|
|
|
139
139
|
max?: number;
|
|
140
140
|
schema?: string;
|
|
141
141
|
}): Worker;
|
|
142
|
-
|
|
142
|
+
/** This worker's id — what `worker_id` on a running task points at, and what
|
|
143
|
+
* the Python SDK calls `worker_id` too. */
|
|
144
|
+
get workerId(): string;
|
|
143
145
|
task(handler: Handler): this;
|
|
144
146
|
task(name: string, handler: Handler): this;
|
|
145
147
|
task<P, R>(def: TaskDef<P, R>, handler: TypedHandler<P, R>): this;
|
package/dist/worker.js
CHANGED
|
@@ -72,7 +72,7 @@ export class Worker {
|
|
|
72
72
|
queues;
|
|
73
73
|
opts;
|
|
74
74
|
handlers = new Map();
|
|
75
|
-
|
|
75
|
+
ownId = newId("worker");
|
|
76
76
|
/** Payload bytes charged to running handlers — see maxInFlightBytes. */
|
|
77
77
|
inFlightBytes = 0;
|
|
78
78
|
/** Calls in flight, for the names that cap their own concurrency. */
|
|
@@ -142,8 +142,10 @@ export class Worker {
|
|
|
142
142
|
worker.ownsStore = true;
|
|
143
143
|
return worker;
|
|
144
144
|
}
|
|
145
|
-
|
|
146
|
-
|
|
145
|
+
/** This worker's id — what `worker_id` on a running task points at, and what
|
|
146
|
+
* the Python SDK calls `worker_id` too. */
|
|
147
|
+
get workerId() {
|
|
148
|
+
return this.ownId;
|
|
147
149
|
}
|
|
148
150
|
task(arg, second, third) {
|
|
149
151
|
// Option form: (name | def, { batch?, concurrency?, resource? }, handler).
|
|
@@ -274,7 +276,7 @@ export class Worker {
|
|
|
274
276
|
return out;
|
|
275
277
|
}
|
|
276
278
|
context(task, leaseMs) {
|
|
277
|
-
return new TaskContext(this.store, task, this.
|
|
279
|
+
return new TaskContext(this.store, task, this.ownId, leaseMs, {
|
|
278
280
|
retryBackoffMs: this.backoffMs,
|
|
279
281
|
retryBackoffMaxMs: this.backoffMaxMs,
|
|
280
282
|
});
|
|
@@ -380,7 +382,7 @@ export class Worker {
|
|
|
380
382
|
// Only what this worker can run. Queues do not partition work by task
|
|
381
383
|
// name, so another worker's tasks would otherwise be claimed here and
|
|
382
384
|
// failed for want of a handler.
|
|
383
|
-
{ queues: this.queues, workerId: this.
|
|
385
|
+
{ queues: this.queues, workerId: this.ownId, leaseMs, names }, async (claim) => {
|
|
384
386
|
const drawn = [];
|
|
385
387
|
let left = free;
|
|
386
388
|
// Resource units this poll has already drawn. resourceCalls only
|
|
@@ -612,7 +614,7 @@ export class Worker {
|
|
|
612
614
|
try {
|
|
613
615
|
// complete (not succeed): finalizes as canceled if a cancel was requested
|
|
614
616
|
// while the handler ran, else succeeded.
|
|
615
|
-
await this.store.complete({ taskId: ctx.taskId, workerId: this.
|
|
617
|
+
await this.store.complete({ taskId: ctx.taskId, workerId: this.ownId, result });
|
|
616
618
|
ctx.markSettled();
|
|
617
619
|
}
|
|
618
620
|
catch (err) {
|
|
@@ -745,7 +747,7 @@ export class Worker {
|
|
|
745
747
|
try {
|
|
746
748
|
const renewed = await this.store.heartbeatBatch({
|
|
747
749
|
taskIds: live.map((c) => c.taskId),
|
|
748
|
-
workerId: this.
|
|
750
|
+
workerId: this.ownId,
|
|
749
751
|
leaseMs,
|
|
750
752
|
});
|
|
751
753
|
for (const ctx of live) {
|
|
@@ -780,7 +782,7 @@ export class Worker {
|
|
|
780
782
|
try {
|
|
781
783
|
await this.store.fail({
|
|
782
784
|
taskId: ctx.taskId,
|
|
783
|
-
workerId: this.
|
|
785
|
+
workerId: this.ownId,
|
|
784
786
|
error: envelope,
|
|
785
787
|
retryable,
|
|
786
788
|
delayMs: failDelayMs(ctx.attempt, retryable, this.backoffMs, this.backoffMaxMs),
|
package/package.json
CHANGED
package/src/context.ts
CHANGED
|
@@ -37,7 +37,7 @@ export class TaskContext {
|
|
|
37
37
|
constructor(
|
|
38
38
|
private readonly store: TaskStore,
|
|
39
39
|
private readonly task: Task,
|
|
40
|
-
|
|
40
|
+
private readonly ownerId: string,
|
|
41
41
|
private readonly leaseMs: number,
|
|
42
42
|
opts: TaskContextOptions = {},
|
|
43
43
|
) {
|
|
@@ -48,6 +48,10 @@ export class TaskContext {
|
|
|
48
48
|
get taskId(): string {
|
|
49
49
|
return this.task.id;
|
|
50
50
|
}
|
|
51
|
+
/** The worker running this task — what `worker_id` on the row points at. */
|
|
52
|
+
get workerId(): string {
|
|
53
|
+
return this.ownerId;
|
|
54
|
+
}
|
|
51
55
|
get name(): string {
|
|
52
56
|
return this.task.name;
|
|
53
57
|
}
|
|
@@ -150,7 +154,7 @@ export class TaskContext {
|
|
|
150
154
|
return this.owned(() =>
|
|
151
155
|
this.store.progress({
|
|
152
156
|
taskId: this.task.id,
|
|
153
|
-
workerId: this.
|
|
157
|
+
workerId: this.ownerId,
|
|
154
158
|
progress: value,
|
|
155
159
|
message,
|
|
156
160
|
}),
|
|
@@ -161,7 +165,7 @@ export class TaskContext {
|
|
|
161
165
|
return this.owned(() =>
|
|
162
166
|
this.store.heartbeat({
|
|
163
167
|
taskId: this.task.id,
|
|
164
|
-
workerId: this.
|
|
168
|
+
workerId: this.ownerId,
|
|
165
169
|
leaseMs: this.leaseMs,
|
|
166
170
|
}),
|
|
167
171
|
);
|
|
@@ -196,7 +200,7 @@ export class TaskContext {
|
|
|
196
200
|
async succeed(result: unknown = null): Promise<Task | null> {
|
|
197
201
|
if (this.isSettled) return null;
|
|
198
202
|
const task = await this.owned(() =>
|
|
199
|
-
this.store.complete({ taskId: this.task.id, workerId: this.
|
|
203
|
+
this.store.complete({ taskId: this.task.id, workerId: this.ownerId, result }),
|
|
200
204
|
);
|
|
201
205
|
this.markSettled();
|
|
202
206
|
return task;
|
|
@@ -229,7 +233,7 @@ export class TaskContext {
|
|
|
229
233
|
if (this.isSettled) return null;
|
|
230
234
|
const task = await this.owned(async () => {
|
|
231
235
|
const { task } = await this.store.completeIn<PgSession, T>(
|
|
232
|
-
{ taskId: this.task.id, workerId: this.
|
|
236
|
+
{ taskId: this.task.id, workerId: this.ownerId },
|
|
233
237
|
write,
|
|
234
238
|
);
|
|
235
239
|
return task;
|
|
@@ -253,7 +257,7 @@ export class TaskContext {
|
|
|
253
257
|
const task = await this.owned(() =>
|
|
254
258
|
this.store.fail({
|
|
255
259
|
taskId: this.task.id,
|
|
256
|
-
workerId: this.
|
|
260
|
+
workerId: this.ownerId,
|
|
257
261
|
error: envelope,
|
|
258
262
|
retryable,
|
|
259
263
|
delayMs: failDelayMs(this.task.attempt, retryable, this.backoffMs, this.backoffMaxMs),
|
package/src/errors.ts
CHANGED
|
@@ -208,6 +208,27 @@ export class ProtocolVersionMismatch extends CairnQError {
|
|
|
208
208
|
}
|
|
209
209
|
}
|
|
210
210
|
|
|
211
|
+
/**
|
|
212
|
+
* This connection is not pointed at the cairnq installation the rest of the
|
|
213
|
+
* deployment is using — raised at connect, before any task is written.
|
|
214
|
+
*
|
|
215
|
+
* The schema a Postgres connection resolves to is out-of-band configuration
|
|
216
|
+
* (`search_path`, a `schema` option, an ORM's pool settings), so two processes
|
|
217
|
+
* given the same DSN can still land in different schemas. Every migration is
|
|
218
|
+
* `create table if not exists`, so the odd one out does not fail: it builds a
|
|
219
|
+
* second, empty installation and its protocol version check passes against the
|
|
220
|
+
* cairnq_meta it just created. Left undetected, an API and a worker then agree
|
|
221
|
+
* about everything except WHERE, and no task ever crosses.
|
|
222
|
+
*
|
|
223
|
+
* The Python SDK raises the same named error.
|
|
224
|
+
*/
|
|
225
|
+
export class SchemaMismatch extends CairnQError {
|
|
226
|
+
constructor(message: string) {
|
|
227
|
+
super(message);
|
|
228
|
+
this.name = "SchemaMismatch";
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
211
232
|
/** A value could not be encoded for a protocol JSON column (non-finite number,
|
|
212
233
|
* BigInt, circular structure, …). Raised at the boundary — submit rejects with
|
|
213
234
|
* it, and a worker records a handler result that triggers it as a permanent
|
package/src/index.ts
CHANGED
package/src/models.ts
CHANGED
|
@@ -39,6 +39,19 @@ export interface TaskRef {
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
const JSON_COLUMNS = ["payload", "result", "error", "metadata"] as const;
|
|
42
|
+
// The bigint columns. `attempt` / `max_attempts` / `priority` are int4 and
|
|
43
|
+
// `progress` is double precision, so every driver already gives those as numbers;
|
|
44
|
+
// only int8 has a wire form worth normalizing. Nullability differs per column
|
|
45
|
+
// (completed_at_ms may be null, created_at_ms may not), so the coercion has to
|
|
46
|
+
// preserve null rather than turn it into 0.
|
|
47
|
+
const MS_COLUMNS = [
|
|
48
|
+
"lease_until_ms",
|
|
49
|
+
"run_at_ms",
|
|
50
|
+
"cancel_requested_at_ms",
|
|
51
|
+
"created_at_ms",
|
|
52
|
+
"updated_at_ms",
|
|
53
|
+
"completed_at_ms",
|
|
54
|
+
] as const;
|
|
42
55
|
// As a const tuple so TerminalStatus derives from it — the same declare-once
|
|
43
56
|
// pattern as STATUSES/TaskStatus above.
|
|
44
57
|
export const TERMINAL = ["succeeded", "failed", "canceled"] as const;
|
|
@@ -63,6 +76,20 @@ export function rowToTask(row: Record<string, unknown>): Task {
|
|
|
63
76
|
// already-decoded object. Parse only a string — never assume one backend.
|
|
64
77
|
t[col] = typeof v === "string" ? JSON.parse(v) : (v ?? null);
|
|
65
78
|
}
|
|
79
|
+
for (const col of MS_COLUMNS) {
|
|
80
|
+
const v = row[col];
|
|
81
|
+
// Same argument, for the other column type the drivers disagree about.
|
|
82
|
+
// Postgres sends int8 down the wire as text to protect precision it cannot
|
|
83
|
+
// know is unneeded; `pg` surfaces that as a string, postgres.js does too,
|
|
84
|
+
// asyncpg decodes to int. Normalizing here rather than demanding it of every
|
|
85
|
+
// driver is what keeps an INJECTED executor honest: the alternative is
|
|
86
|
+
// asking an application to change its driver's global int8 handling to suit
|
|
87
|
+
// cairnq, which breaks that application's own bigint columns.
|
|
88
|
+
//
|
|
89
|
+
// Lossless: every one of these is an epoch-ms, and a millisecond timestamp
|
|
90
|
+
// does not reach Number.MAX_SAFE_INTEGER until the year 287396.
|
|
91
|
+
t[col] = v == null ? null : Number(v);
|
|
92
|
+
}
|
|
66
93
|
return t as unknown as Task;
|
|
67
94
|
}
|
|
68
95
|
|
package/src/store/pg-executor.ts
CHANGED
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
* instead of opening a second one, which is what makes a task's settlement
|
|
9
9
|
* commit in the same transaction as the rows the task produced.
|
|
10
10
|
*
|
|
11
|
-
* Implementing one is small — see `createPoolExecutor`
|
|
12
|
-
* implementation over `pg
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* Implementing one is small — see `createPoolExecutor` in pg-pool.ts for the
|
|
12
|
+
* reference implementation over `pg`. An adapter passes rows through as its
|
|
13
|
+
* driver produced them: cairnq normalizes both column types the drivers disagree
|
|
14
|
+
* about (jsonb decoded or not, int8 as text or number) in `rowToTask`, so no
|
|
15
|
+
* adapter has to reconfigure its driver — and none has to change how the
|
|
16
|
+
* application's OWN columns come back in order to satisfy cairnq.
|
|
16
17
|
*/
|
|
17
18
|
|
|
18
19
|
/** A row as the driver hands it back: column name -> value. */
|
package/src/store/pg-pool.ts
CHANGED
|
@@ -15,11 +15,12 @@ async function loadPg(): Promise<typeof import("pg")> {
|
|
|
15
15
|
throw new Error("PostgresStore requires the 'pg' package — install it (e.g. `npm i pg`)");
|
|
16
16
|
}
|
|
17
17
|
const pg = (mod.default ?? mod) as typeof import("pg");
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
|
|
18
|
+
// Deliberately NOT setting a global int8 type parser here. It would be the
|
|
19
|
+
// shortest fix for Postgres sending bigint as text, and it is what this file
|
|
20
|
+
// used to do — but pg's type parsers are process-global, so a library that
|
|
21
|
+
// installs one silently changes how the APPLICATION's own bigint columns come
|
|
22
|
+
// back. rowToTask normalizes instead, which costs nothing and leaves the
|
|
23
|
+
// caller's driver as they configured it.
|
|
23
24
|
pgModule = pg;
|
|
24
25
|
return pg;
|
|
25
26
|
}
|
package/src/store/postgres.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { SchemaMismatch } from "../errors.js";
|
|
1
2
|
import { loadMigrations, loadStatements } from "../sql.js";
|
|
2
3
|
import {
|
|
3
4
|
checkProtocolVersion,
|
|
@@ -163,6 +164,9 @@ export class PostgresStore extends TaskStore {
|
|
|
163
164
|
this.provided ??
|
|
164
165
|
(await createPoolExecutor(this.dsn!, { max: this.opts.max, schema: this.opts.schema }));
|
|
165
166
|
try {
|
|
167
|
+
// Before migrations, which would otherwise create the very installation
|
|
168
|
+
// this is trying to warn about.
|
|
169
|
+
await this.checkSchema(executor);
|
|
166
170
|
await this.applyMigrations(executor);
|
|
167
171
|
checkProtocolVersion(await this.readProtocolVersion(executor));
|
|
168
172
|
} catch (e) {
|
|
@@ -176,6 +180,63 @@ export class PostgresStore extends TaskStore {
|
|
|
176
180
|
this.listenerReady();
|
|
177
181
|
}
|
|
178
182
|
|
|
183
|
+
/**
|
|
184
|
+
* Refuse a connection pointed somewhere other than the deployment's cairnq.
|
|
185
|
+
*
|
|
186
|
+
* Two shapes, because `schema` means "the schema cairnq's tables live in" and
|
|
187
|
+
* cairnq can either arrange that (it built the connection) or only check it
|
|
188
|
+
* (the caller's executor did):
|
|
189
|
+
*
|
|
190
|
+
* - `schema` configured -> assert the connection actually resolves there. On
|
|
191
|
+
* the DSN path this is a cheap self-check; on an injected executor it is the
|
|
192
|
+
* only way to state the expectation at all.
|
|
193
|
+
* - `schema` not configured -> the dangerous case is being about to create a
|
|
194
|
+
* SECOND installation while one already exists elsewhere in this database,
|
|
195
|
+
* which is exactly what a mismatched pair of SDKs does. Joining an existing
|
|
196
|
+
* installation is fine no matter what else is around, so the check is
|
|
197
|
+
* deliberately narrow: it fires only when this schema has no cairnq and some
|
|
198
|
+
* other schema does.
|
|
199
|
+
*
|
|
200
|
+
* That narrowness is what keeps it from crying wolf. Two applications each
|
|
201
|
+
* running their own cairnq in their own schema are legitimate; the second one
|
|
202
|
+
* to be set up trips this once, and saying `schema` explicitly — which such a
|
|
203
|
+
* deployment should be doing anyway — is both the fix and the confirmation.
|
|
204
|
+
*/
|
|
205
|
+
private async checkSchema(executor: PgExecutor): Promise<void> {
|
|
206
|
+
// One row per installation; the statement's LEFT JOIN guarantees at least
|
|
207
|
+
// one, so current_schema is readable even where cairnq lives nowhere yet.
|
|
208
|
+
const rows = await executor.query(this.statements.installations, []);
|
|
209
|
+
const current = (rows[0]?.current_schema as string | null) ?? null;
|
|
210
|
+
const installations = rows
|
|
211
|
+
.map((r) => r.schema as string | null)
|
|
212
|
+
.filter((s): s is string => s != null);
|
|
213
|
+
const wanted = this.opts.schema;
|
|
214
|
+
|
|
215
|
+
if (wanted != null) {
|
|
216
|
+
if (current !== wanted) {
|
|
217
|
+
throw new SchemaMismatch(
|
|
218
|
+
`cairnq is configured for schema ${JSON.stringify(wanted)} but this ` +
|
|
219
|
+
`connection resolves to ${JSON.stringify(current)} — check the ` +
|
|
220
|
+
`connection's search_path`,
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
// A search_path naming nothing that exists: there is no "here" to compare
|
|
226
|
+
// against, and the migrations are about to fail with a clearer message.
|
|
227
|
+
if (current === null) return;
|
|
228
|
+
if (installations.length === 0 || installations.includes(current)) return;
|
|
229
|
+
|
|
230
|
+
throw new SchemaMismatch(
|
|
231
|
+
`cairnq tables already exist in schema ${installations.map((s) => JSON.stringify(s)).join(", ")} ` +
|
|
232
|
+
`of this database, but this connection resolves to ${JSON.stringify(current)}, where there are ` +
|
|
233
|
+
`none. Connecting would create a second, parallel installation that the other one can never see ` +
|
|
234
|
+
`— an API and a worker split this way agree about everything except where, and no task crosses. ` +
|
|
235
|
+
`Point this process at the same schema (\`schema\` option, or \`options=-c search_path=...\` in the ` +
|
|
236
|
+
`DSN), or pass \`schema\` explicitly to confirm a separate installation is what you meant.`,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
|
|
179
240
|
private async applyMigrations(executor: PgExecutor): Promise<void> {
|
|
180
241
|
await executor.exec(
|
|
181
242
|
"create table if not exists cairnq_migrations " +
|
package/src/worker.ts
CHANGED
|
@@ -222,7 +222,7 @@ function release(counts: Map<string, number>, key: string | undefined): void {
|
|
|
222
222
|
|
|
223
223
|
export class Worker {
|
|
224
224
|
private readonly handlers = new Map<string, Registration>();
|
|
225
|
-
private readonly
|
|
225
|
+
private readonly ownId = newId("worker");
|
|
226
226
|
/** Payload bytes charged to running handlers — see maxInFlightBytes. */
|
|
227
227
|
private inFlightBytes = 0;
|
|
228
228
|
/** Calls in flight, for the names that cap their own concurrency. */
|
|
@@ -305,8 +305,10 @@ export class Worker {
|
|
|
305
305
|
return worker;
|
|
306
306
|
}
|
|
307
307
|
|
|
308
|
-
|
|
309
|
-
|
|
308
|
+
/** This worker's id — what `worker_id` on a running task points at, and what
|
|
309
|
+
* the Python SDK calls `worker_id` too. */
|
|
310
|
+
get workerId(): string {
|
|
311
|
+
return this.ownId;
|
|
310
312
|
}
|
|
311
313
|
|
|
312
314
|
task(handler: Handler): this;
|
|
@@ -474,7 +476,7 @@ export class Worker {
|
|
|
474
476
|
}
|
|
475
477
|
|
|
476
478
|
private context(task: Task, leaseMs: number): TaskContext {
|
|
477
|
-
return new TaskContext(this.store, task, this.
|
|
479
|
+
return new TaskContext(this.store, task, this.ownId, leaseMs, {
|
|
478
480
|
retryBackoffMs: this.backoffMs,
|
|
479
481
|
retryBackoffMaxMs: this.backoffMaxMs,
|
|
480
482
|
});
|
|
@@ -588,7 +590,7 @@ export class Worker {
|
|
|
588
590
|
// Only what this worker can run. Queues do not partition work by task
|
|
589
591
|
// name, so another worker's tasks would otherwise be claimed here and
|
|
590
592
|
// failed for want of a handler.
|
|
591
|
-
{ queues: this.queues, workerId: this.
|
|
593
|
+
{ queues: this.queues, workerId: this.ownId, leaseMs, names },
|
|
592
594
|
async (claim) => {
|
|
593
595
|
const drawn: { src: ClaimSource; calls: [Registration | undefined, Task[]][] }[] = [];
|
|
594
596
|
let left = free;
|
|
@@ -826,7 +828,7 @@ export class Worker {
|
|
|
826
828
|
try {
|
|
827
829
|
// complete (not succeed): finalizes as canceled if a cancel was requested
|
|
828
830
|
// while the handler ran, else succeeded.
|
|
829
|
-
await this.store.complete({ taskId: ctx.taskId, workerId: this.
|
|
831
|
+
await this.store.complete({ taskId: ctx.taskId, workerId: this.ownId, result });
|
|
830
832
|
ctx.markSettled();
|
|
831
833
|
} catch (err) {
|
|
832
834
|
if (err instanceof LostLease) {
|
|
@@ -967,7 +969,7 @@ export class Worker {
|
|
|
967
969
|
try {
|
|
968
970
|
const renewed = await this.store.heartbeatBatch({
|
|
969
971
|
taskIds: live.map((c) => c.taskId),
|
|
970
|
-
workerId: this.
|
|
972
|
+
workerId: this.ownId,
|
|
971
973
|
leaseMs,
|
|
972
974
|
});
|
|
973
975
|
for (const ctx of live) {
|
|
@@ -1003,7 +1005,7 @@ export class Worker {
|
|
|
1003
1005
|
try {
|
|
1004
1006
|
await this.store.fail({
|
|
1005
1007
|
taskId: ctx.taskId,
|
|
1006
|
-
workerId: this.
|
|
1008
|
+
workerId: this.ownId,
|
|
1007
1009
|
error: envelope,
|
|
1008
1010
|
retryable,
|
|
1009
1011
|
delayMs: failDelayMs(ctx.attempt, retryable, this.backoffMs, this.backoffMaxMs),
|