@pikku/queue-nats 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/LICENSE +21 -0
- package/README.md +47 -0
- package/dist/consume.d.ts +54 -0
- package/dist/consume.js +213 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.js +42 -0
- package/dist/nats-delayed-publisher.d.ts +35 -0
- package/dist/nats-delayed-publisher.js +85 -0
- package/dist/nats-queue-service.d.ts +44 -0
- package/dist/nats-queue-service.js +84 -0
- package/dist/nats-queue-worker.d.ts +46 -0
- package/dist/nats-queue-worker.js +199 -0
- package/dist/nats-scheduler-service.d.ts +156 -0
- package/dist/nats-scheduler-service.js +369 -0
- package/dist/nats-service-factory.d.ts +104 -0
- package/dist/nats-service-factory.js +231 -0
- package/dist/utils.d.ts +122 -0
- package/dist/utils.js +154 -0
- package/package.json +33 -0
- package/run-tests.sh +56 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# @pikku/queue-nats
|
|
2
|
+
|
|
3
|
+
## 0.12.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 427c4fe: **`@pikku/queue-nats`** — NATS JetStream queue, worker and scheduler services, alongside the pg-boss and bullmq adapters. Jobs publish to a work-queue stream, each queue gets its own durable pull consumer, and `max_ack_pending` is real per-message concurrency rather than a batch size, so a slow job occupies one slot instead of holding its batch siblings. The per-job retry policy travels in headers because JetStream's `max_deliver` is per-consumer and cannot express a step marked "never retry", and `ack_wait` is always set to pg-boss's 900s parity rather than left to the server's 30s default — unset, any job running longer than 30s is redelivered while it is still running and executed twice. Cron and one-shot delays use the server's own message scheduler, so a schedule is a retained message on a reserved subject: publishing registers it, republishing replaces it, purging cancels it. That needs a server on 2.14+, and `NatsServiceFactory` refuses to start against anything older rather than silently not scheduling. `supportsResults` is `false`, deliberately — a work-queue stream deletes a message on ack, so there is nowhere for a job result or any job history to live, and per-job outcomes come from telemetry instead.
|
|
8
|
+
- Updated dependencies [1ab831e]
|
|
9
|
+
- @pikku/core@0.12.105
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021 - present Yasser Fadl and Pikku contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# @pikku/queue-nats
|
|
2
|
+
|
|
3
|
+
NATS JetStream queue, worker and scheduler services for Pikku. Backed by a
|
|
4
|
+
JetStream work-queue stream, with the schedules living in the broker rather
|
|
5
|
+
than in a database.
|
|
6
|
+
|
|
7
|
+
Requires a NATS server on 2.14 or newer — that is the release that added
|
|
8
|
+
repeating cron schedules, and `NatsServiceFactory` refuses to start against
|
|
9
|
+
anything older rather than silently not scheduling.
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install @pikku/queue-nats
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Usage
|
|
18
|
+
|
|
19
|
+
```typescript
|
|
20
|
+
import { NatsServiceFactory } from '@pikku/queue-nats'
|
|
21
|
+
|
|
22
|
+
const factory = new NatsServiceFactory({
|
|
23
|
+
servers: 'nats://localhost:4222',
|
|
24
|
+
streamName: 'pikku',
|
|
25
|
+
subjectPrefix: 'pikku',
|
|
26
|
+
})
|
|
27
|
+
await factory.init()
|
|
28
|
+
|
|
29
|
+
const queue = factory.getQueueService()
|
|
30
|
+
const workers = factory.getQueueWorkers()
|
|
31
|
+
|
|
32
|
+
await workers.registerQueues()
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Use `NatsSchedulerService` for cron wirings. For a Postgres-backed alternative
|
|
36
|
+
see `@pikku/queue-pg-boss`, and for a Redis-backed one `@pikku/queue-bullmq`.
|
|
37
|
+
|
|
38
|
+
## Job results
|
|
39
|
+
|
|
40
|
+
`supportsResults` is `false`. A work-queue stream deletes a message the moment
|
|
41
|
+
it is acked, so there is nowhere for a job result — or any job history — to
|
|
42
|
+
live. Per-job outcomes come from telemetry, not from the broker; producers that
|
|
43
|
+
need a return value should use an RPC rather than a queue.
|
|
44
|
+
|
|
45
|
+
## Docs
|
|
46
|
+
|
|
47
|
+
https://pikku.dev/docs
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { ConsumerConfig, JetStreamClient, JetStreamManager, JsMsg } from '@nats-io/jetstream';
|
|
2
|
+
import type { Logger } from '@pikku/core/services';
|
|
3
|
+
export interface ConsumeOptions {
|
|
4
|
+
js: JetStreamClient;
|
|
5
|
+
jsm: JetStreamManager;
|
|
6
|
+
streamName: string;
|
|
7
|
+
subjectPrefix: string;
|
|
8
|
+
queueName: string;
|
|
9
|
+
/** Merged over the defaults produced by `mapPikkuWorkerToNats`. */
|
|
10
|
+
config?: Partial<ConsumerConfig>;
|
|
11
|
+
/**
|
|
12
|
+
* Handles one message. It owns the message's terminal call — exactly one of
|
|
13
|
+
* `ack`/`nak`/`term` — because only the caller knows whether a failure is
|
|
14
|
+
* retryable. A handler that returns without acking leaves the message to be
|
|
15
|
+
* redelivered after `ack_wait`.
|
|
16
|
+
*/
|
|
17
|
+
handler: (msg: JsMsg) => Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Used to report a pull session that ended and had to be re-established.
|
|
20
|
+
* Optional only so existing callers and tests keep compiling — pass it in
|
|
21
|
+
* production, because a silent re-attach loop is the failure this guards.
|
|
22
|
+
*/
|
|
23
|
+
logger?: Pick<Logger, 'error' | 'info'>;
|
|
24
|
+
}
|
|
25
|
+
export interface ConsumeHandle {
|
|
26
|
+
/** Stop pulling. In-flight handlers are left to settle their own acks. */
|
|
27
|
+
stop: () => void;
|
|
28
|
+
/** Handlers currently running — messages delivered but not yet settled. */
|
|
29
|
+
readonly inFlight: number;
|
|
30
|
+
/**
|
|
31
|
+
* Stop pulling, then wait for in-flight handlers to settle their acks.
|
|
32
|
+
*
|
|
33
|
+
* Resolves `true` if everything drained, `false` if the timeout expired
|
|
34
|
+
* first. Timing out is safe rather than lossy — whatever is still unacked is
|
|
35
|
+
* redelivered after `ack_wait`. Draining just avoids paying that redelivery
|
|
36
|
+
* latency on every deploy, and avoids re-running side effects a handler had
|
|
37
|
+
* already performed before it was killed.
|
|
38
|
+
*/
|
|
39
|
+
drain: (timeoutMs: number) => Promise<boolean>;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Start a durable pull consumer and dispatch its messages to `handler`.
|
|
43
|
+
*
|
|
44
|
+
* The one thing that must never change here: `handler` is launched, not
|
|
45
|
+
* awaited. Awaiting inside the `for await` would serialise processing no matter
|
|
46
|
+
* what `max_ack_pending` says, so a single slow job would stall every message
|
|
47
|
+
* behind it — the same head-of-line blocking as a pg-boss batch handler, just
|
|
48
|
+
* moved client-side. In-flight work stays bounded anyway, because the server
|
|
49
|
+
* will not deliver more than `max_ack_pending` unacked messages at once.
|
|
50
|
+
*
|
|
51
|
+
* This exists as a shared function precisely so that invariant is written down
|
|
52
|
+
* once and every consumer inherits it, rather than being re-derived per caller.
|
|
53
|
+
*/
|
|
54
|
+
export declare const consumeQueue: (options: ConsumeOptions) => Promise<ConsumeHandle>;
|
package/dist/consume.js
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { consumerNameForQueue, subjectForQueue } from './utils.js';
|
|
2
|
+
/** Backoff bounds for re-establishing a pull session that ended under us. */
|
|
3
|
+
const REATTACH_MIN_DELAY_MS = 250;
|
|
4
|
+
const REATTACH_MAX_DELAY_MS = 30_000;
|
|
5
|
+
/**
|
|
6
|
+
* How many consecutive "can't reach it" notifications count as dead rather than
|
|
7
|
+
* transient. Missed heartbeats and a not-found consumer both occur harmlessly
|
|
8
|
+
* during a reconnect, so a single one must not tear down a healthy session.
|
|
9
|
+
*/
|
|
10
|
+
const REATTACH_NOTIFICATION_THRESHOLD = 3;
|
|
11
|
+
/**
|
|
12
|
+
* Decide whether a notification means this pull session is permanently dead,
|
|
13
|
+
* returning the reason to log if so.
|
|
14
|
+
*
|
|
15
|
+
* This exists because of a trap documented in the client's own types: for a
|
|
16
|
+
* durable pull consumer these conditions do NOT end the message iterator. On
|
|
17
|
+
* `consumer_deleted` the docs say outright that "the client will continue to
|
|
18
|
+
* attempt to pull messages" — only an *ordered* consumer recreates itself. So
|
|
19
|
+
* the session sits there issuing pulls that can never be answered, the
|
|
20
|
+
* `for await` never returns, and no error is ever thrown. Watching `status()`
|
|
21
|
+
* is the only way to find out, which is why the supervisor below cannot rely on
|
|
22
|
+
* the iterator ending on its own.
|
|
23
|
+
*/
|
|
24
|
+
const fatalNotificationReason = (notification) => {
|
|
25
|
+
switch (notification.type) {
|
|
26
|
+
case 'consumer_deleted':
|
|
27
|
+
return `consumer deleted server-side (${notification.code} ${notification.description})`;
|
|
28
|
+
case 'stream_not_found':
|
|
29
|
+
return `stream ${notification.name} not found`;
|
|
30
|
+
case 'consumer_not_found':
|
|
31
|
+
return notification.count >= REATTACH_NOTIFICATION_THRESHOLD
|
|
32
|
+
? `consumer not found on ${notification.count} consecutive pulls`
|
|
33
|
+
: null;
|
|
34
|
+
case 'heartbeats_missed':
|
|
35
|
+
return notification.count >= REATTACH_NOTIFICATION_THRESHOLD
|
|
36
|
+
? `${notification.count} consecutive heartbeats missed`
|
|
37
|
+
: null;
|
|
38
|
+
default:
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Start a durable pull consumer and dispatch its messages to `handler`.
|
|
44
|
+
*
|
|
45
|
+
* The one thing that must never change here: `handler` is launched, not
|
|
46
|
+
* awaited. Awaiting inside the `for await` would serialise processing no matter
|
|
47
|
+
* what `max_ack_pending` says, so a single slow job would stall every message
|
|
48
|
+
* behind it — the same head-of-line blocking as a pg-boss batch handler, just
|
|
49
|
+
* moved client-side. In-flight work stays bounded anyway, because the server
|
|
50
|
+
* will not deliver more than `max_ack_pending` unacked messages at once.
|
|
51
|
+
*
|
|
52
|
+
* This exists as a shared function precisely so that invariant is written down
|
|
53
|
+
* once and every consumer inherits it, rather than being re-derived per caller.
|
|
54
|
+
*/
|
|
55
|
+
export const consumeQueue = async (options) => {
|
|
56
|
+
const { js, jsm, streamName, subjectPrefix, queueName, config, handler, logger, } = options;
|
|
57
|
+
const durable = consumerNameForQueue(subjectPrefix, queueName);
|
|
58
|
+
// `add` is CONSUMER.CREATE: a no-op when the existing durable matches, but it
|
|
59
|
+
// THROWS "consumer already exists" when any setting differs. So create first,
|
|
60
|
+
// then fall back to update — otherwise the first deploy after changing a
|
|
61
|
+
// queue's batchSize or lockDuration would fail at boot, on the second deploy
|
|
62
|
+
// rather than the one that introduced the change. Same failure shape as
|
|
63
|
+
// STREAM.CREATE vs STREAM.UPDATE in nats-service-factory.
|
|
64
|
+
const consumerConfig = {
|
|
65
|
+
durable_name: durable,
|
|
66
|
+
filter_subject: subjectForQueue(subjectPrefix, queueName),
|
|
67
|
+
...config,
|
|
68
|
+
};
|
|
69
|
+
const attach = async () => {
|
|
70
|
+
try {
|
|
71
|
+
await jsm.consumers.add(streamName, consumerConfig);
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
75
|
+
if (!/already exists|already in use/i.test(message))
|
|
76
|
+
throw err;
|
|
77
|
+
await jsm.consumers.update(streamName, durable, consumerConfig);
|
|
78
|
+
}
|
|
79
|
+
const consumer = await js.consumers.get(streamName, durable);
|
|
80
|
+
return await consumer.consume();
|
|
81
|
+
};
|
|
82
|
+
let stopped = false;
|
|
83
|
+
let inFlight = 0;
|
|
84
|
+
// Bumped on every re-attach so a watcher left over from a previous session
|
|
85
|
+
// can never stop the session that replaced it.
|
|
86
|
+
let generation = 0;
|
|
87
|
+
const describe = (err) => err instanceof Error ? err.message : String(err);
|
|
88
|
+
// Turns a silently-dead session into an ended one, which is the single shape
|
|
89
|
+
// the supervisor below knows how to recover from.
|
|
90
|
+
const watchStatus = (session, sessionGeneration) => {
|
|
91
|
+
void (async () => {
|
|
92
|
+
try {
|
|
93
|
+
for await (const notification of session.status()) {
|
|
94
|
+
if (stopped || generation !== sessionGeneration)
|
|
95
|
+
return;
|
|
96
|
+
const reason = fatalNotificationReason(notification);
|
|
97
|
+
if (reason === null)
|
|
98
|
+
continue;
|
|
99
|
+
logger?.error(`NATS consumer ${durable} on ${streamName}: ${reason} — dropping the pull session so it can re-attach`);
|
|
100
|
+
session.stop();
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
// The status stream itself ending or failing is not the failure we are
|
|
106
|
+
// guarding: the message iterator reports its own end, and the supervisor
|
|
107
|
+
// acts on that. Swallowed so this never surfaces as an unhandled
|
|
108
|
+
// rejection during shutdown.
|
|
109
|
+
}
|
|
110
|
+
})();
|
|
111
|
+
};
|
|
112
|
+
// The first attach is awaited and NOT retried: a failure here is a bad stream
|
|
113
|
+
// or consumer config, and boot must fail loudly on it rather than spin.
|
|
114
|
+
let messages = await attach();
|
|
115
|
+
watchStatus(messages, generation);
|
|
116
|
+
// Not awaited: this supervisor runs for the lifetime of the consumer.
|
|
117
|
+
//
|
|
118
|
+
// The inner `for await` is a pull SESSION, not the consumer itself, and the
|
|
119
|
+
// session can die while the consumer lives on. Before this loop existed the
|
|
120
|
+
// iterator simply ran off the end: the IIFE resolved, nothing pulled ever
|
|
121
|
+
// again, and nothing said so. The durable consumer still existed server-side
|
|
122
|
+
// with its backlog growing, every health check stayed green, and the only
|
|
123
|
+
// symptom was work silently not happening. That is exactly how the
|
|
124
|
+
// orchestrator queue reached 1681 undelivered messages while the backend
|
|
125
|
+
// looked healthy — see knowledge/stability.md. So: re-attach, with backoff,
|
|
126
|
+
// and say so at error level every time.
|
|
127
|
+
//
|
|
128
|
+
// Note this loop alone is not enough, which is why `watchStatus` exists: the
|
|
129
|
+
// iterator ending is only one of the two ways a session dies, and it is the
|
|
130
|
+
// less common one. The other is the session staying open forever while the
|
|
131
|
+
// server has nothing to answer it with, and only `status()` reveals that.
|
|
132
|
+
void (async () => {
|
|
133
|
+
let backoffMs = REATTACH_MIN_DELAY_MS;
|
|
134
|
+
for (;;) {
|
|
135
|
+
try {
|
|
136
|
+
for await (const msg of messages) {
|
|
137
|
+
if (stopped)
|
|
138
|
+
break;
|
|
139
|
+
// Deliberately NOT awaited — see the note above. The counter is what
|
|
140
|
+
// lets shutdown wait for these without serialising them; `finally` is
|
|
141
|
+
// chained rather than `catch`ed so a rejecting handler still surfaces
|
|
142
|
+
// as an unhandled rejection exactly as it did before.
|
|
143
|
+
//
|
|
144
|
+
// Started through a resolved promise so that a handler which throws
|
|
145
|
+
// SYNCHRONOUSLY rejects like any other failure instead of unwinding
|
|
146
|
+
// into the catch below, which would diagnose a handler bug as a dead
|
|
147
|
+
// pull session and re-attach for no reason.
|
|
148
|
+
const settled = Promise.resolve().then(() => handler(msg));
|
|
149
|
+
inFlight++;
|
|
150
|
+
void settled.finally(() => {
|
|
151
|
+
inFlight--;
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
if (stopped)
|
|
155
|
+
return;
|
|
156
|
+
logger?.error(`NATS consumer ${durable} on ${streamName}: pull session ended unexpectedly — re-attaching`);
|
|
157
|
+
}
|
|
158
|
+
catch (err) {
|
|
159
|
+
if (stopped)
|
|
160
|
+
return;
|
|
161
|
+
logger?.error(`NATS consumer ${durable} on ${streamName}: pull session threw (${describe(err)}) — re-attaching`);
|
|
162
|
+
}
|
|
163
|
+
// Re-attach until it takes. `stop()` is checked on both sides of the wait
|
|
164
|
+
// so a shutdown during backoff exits instead of reviving the consumer.
|
|
165
|
+
for (;;) {
|
|
166
|
+
if (stopped)
|
|
167
|
+
return;
|
|
168
|
+
await new Promise((resolve) => setTimeout(resolve, backoffMs));
|
|
169
|
+
if (stopped)
|
|
170
|
+
return;
|
|
171
|
+
try {
|
|
172
|
+
messages = await attach();
|
|
173
|
+
// `stop()` only stops the session it can see, so one that arrives
|
|
174
|
+
// after shutdown began has to stop itself. Without this the loop
|
|
175
|
+
// below would enter `for await` on a live session whose only exit is
|
|
176
|
+
// a message arriving — on an idle queue, never — leaving a pull
|
|
177
|
+
// consumer on the connection after `drain()` reported it settled.
|
|
178
|
+
if (stopped) {
|
|
179
|
+
messages.stop();
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
generation++;
|
|
183
|
+
watchStatus(messages, generation);
|
|
184
|
+
logger?.info(`NATS consumer ${durable} on ${streamName}: re-attached`);
|
|
185
|
+
backoffMs = REATTACH_MIN_DELAY_MS;
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
188
|
+
catch (err) {
|
|
189
|
+
logger?.error(`NATS consumer ${durable} on ${streamName}: re-attach failed (${describe(err)}), retrying in ${backoffMs}ms`);
|
|
190
|
+
backoffMs = Math.min(backoffMs * 2, REATTACH_MAX_DELAY_MS);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
})();
|
|
195
|
+
const stop = () => {
|
|
196
|
+
stopped = true;
|
|
197
|
+
messages.stop();
|
|
198
|
+
};
|
|
199
|
+
return {
|
|
200
|
+
stop,
|
|
201
|
+
get inFlight() {
|
|
202
|
+
return inFlight;
|
|
203
|
+
},
|
|
204
|
+
drain: async (timeoutMs) => {
|
|
205
|
+
stop();
|
|
206
|
+
const deadline = Date.now() + timeoutMs;
|
|
207
|
+
while (inFlight > 0 && Date.now() < deadline) {
|
|
208
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
209
|
+
}
|
|
210
|
+
return inFlight === 0;
|
|
211
|
+
},
|
|
212
|
+
};
|
|
213
|
+
};
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module @pikku/queue-nats
|
|
3
|
+
*
|
|
4
|
+
* NATS JetStream queue service for Pikku.
|
|
5
|
+
*
|
|
6
|
+
* Mirrors `@pikku/queue-pg-boss` / `@pikku/queue-bullmq`, with one deliberate
|
|
7
|
+
* difference:
|
|
8
|
+
*
|
|
9
|
+
* - `supportsResults` is false. A work-queue stream deletes a message on ack,
|
|
10
|
+
* so there is nowhere for a job result — or any job history — to live.
|
|
11
|
+
* Success rates and per-job outcomes come from telemetry, not the broker.
|
|
12
|
+
*
|
|
13
|
+
* This package used to carry a second difference: no scheduler service, because
|
|
14
|
+
* JetStream had no delayed or scheduled message primitive, so cron and delayed
|
|
15
|
+
* work had to stay on a database-backed queue under the rule "Postgres owns
|
|
16
|
+
* time, NATS owns delivery". That stopped being true — server 2.12 added
|
|
17
|
+
* one-shot `@at` delays and 2.14 added repeating cron schedules — so NATS now
|
|
18
|
+
* owns both. Requires a server on 2.14+; `NatsServiceFactory` refuses to start
|
|
19
|
+
* against anything older rather than silently not scheduling.
|
|
20
|
+
*
|
|
21
|
+
* A database has not left the picture entirely, and shouldn't: the broker has
|
|
22
|
+
* no API to list schedules and no record of whether a fire succeeded, so a
|
|
23
|
+
* schedule registry and a last-fired ledger belong in one. NATS owns *when*,
|
|
24
|
+
* the database owns *what is registered* and *what happened*.
|
|
25
|
+
*/
|
|
26
|
+
export { NatsServiceFactory } from './nats-service-factory.js';
|
|
27
|
+
export type { NatsServiceFactoryOptions } from './nats-service-factory.js';
|
|
28
|
+
export { NatsQueueService } from './nats-queue-service.js';
|
|
29
|
+
export type { DelayedPublisher } from './nats-queue-service.js';
|
|
30
|
+
export { NatsDelayedPublisher } from './nats-delayed-publisher.js';
|
|
31
|
+
export { NatsSchedulerService, toNatsCron, parseNatsSchedule, } from './nats-scheduler-service.js';
|
|
32
|
+
export { NatsQueueWorkers, mapPikkuWorkerToNats } from './nats-queue-worker.js';
|
|
33
|
+
export { consumeQueue } from './consume.js';
|
|
34
|
+
export type { ConsumeHandle, ConsumeOptions } from './consume.js';
|
|
35
|
+
export { consumerNameForQueue, queueNameToToken, subjectForQueue, backoffDelayMs, attemptsFor, backoffFor, scheduleSubjectFor, PIKKU_USER_ID_HEADER, NATS_MSG_ID_HEADER, ATTEMPTS_HEADER, BACKOFF_TYPE_HEADER, BACKOFF_DELAY_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_NEXT_HEADER, SCHEDULER_HEADER, SCHEDULE_SUBJECT_SEGMENT, } from './utils.js';
|
|
36
|
+
export { headers as natsHeaders } from '@nats-io/nats-core';
|
|
37
|
+
export type { QueueService, PikkuWorkerConfig, PikkuJobConfig, QueueJob, JobOptions, } from '@pikku/core/queue';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module @pikku/queue-nats
|
|
3
|
+
*
|
|
4
|
+
* NATS JetStream queue service for Pikku.
|
|
5
|
+
*
|
|
6
|
+
* Mirrors `@pikku/queue-pg-boss` / `@pikku/queue-bullmq`, with one deliberate
|
|
7
|
+
* difference:
|
|
8
|
+
*
|
|
9
|
+
* - `supportsResults` is false. A work-queue stream deletes a message on ack,
|
|
10
|
+
* so there is nowhere for a job result — or any job history — to live.
|
|
11
|
+
* Success rates and per-job outcomes come from telemetry, not the broker.
|
|
12
|
+
*
|
|
13
|
+
* This package used to carry a second difference: no scheduler service, because
|
|
14
|
+
* JetStream had no delayed or scheduled message primitive, so cron and delayed
|
|
15
|
+
* work had to stay on a database-backed queue under the rule "Postgres owns
|
|
16
|
+
* time, NATS owns delivery". That stopped being true — server 2.12 added
|
|
17
|
+
* one-shot `@at` delays and 2.14 added repeating cron schedules — so NATS now
|
|
18
|
+
* owns both. Requires a server on 2.14+; `NatsServiceFactory` refuses to start
|
|
19
|
+
* against anything older rather than silently not scheduling.
|
|
20
|
+
*
|
|
21
|
+
* A database has not left the picture entirely, and shouldn't: the broker has
|
|
22
|
+
* no API to list schedules and no record of whether a fire succeeded, so a
|
|
23
|
+
* schedule registry and a last-fired ledger belong in one. NATS owns *when*,
|
|
24
|
+
* the database owns *what is registered* and *what happened*.
|
|
25
|
+
*/
|
|
26
|
+
export { NatsServiceFactory } from './nats-service-factory.js';
|
|
27
|
+
export { NatsQueueService } from './nats-queue-service.js';
|
|
28
|
+
// Backs workflow.sleep() and backoff retries, using the server's own scheduler.
|
|
29
|
+
// Wired automatically by NatsServiceFactory; exported for tests and for callers
|
|
30
|
+
// building a queue service by hand.
|
|
31
|
+
export { NatsDelayedPublisher } from './nats-delayed-publisher.js';
|
|
32
|
+
export { NatsSchedulerService, toNatsCron, parseNatsSchedule, } from './nats-scheduler-service.js';
|
|
33
|
+
export { NatsQueueWorkers, mapPikkuWorkerToNats } from './nats-queue-worker.js';
|
|
34
|
+
// For consumers that need their own dispatch loop rather than pikku's queue
|
|
35
|
+
// registry — e.g. a dispatcher whose queues are shared across tenants and must
|
|
36
|
+
// not collide with the registry's global queue names.
|
|
37
|
+
export { consumeQueue } from './consume.js';
|
|
38
|
+
export { consumerNameForQueue, queueNameToToken, subjectForQueue, backoffDelayMs, attemptsFor, backoffFor, scheduleSubjectFor, PIKKU_USER_ID_HEADER, NATS_MSG_ID_HEADER, ATTEMPTS_HEADER, BACKOFF_TYPE_HEADER, BACKOFF_DELAY_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_NEXT_HEADER, SCHEDULER_HEADER, SCHEDULE_SUBJECT_SEGMENT, } from './utils.js';
|
|
39
|
+
// Re-exported so callers that publish their own messages (a dispatcher parking
|
|
40
|
+
// a job on a DLQ, say) can build headers without taking a direct dependency on
|
|
41
|
+
// nats-core — this package is the NATS boundary.
|
|
42
|
+
export { headers as natsHeaders } from '@nats-io/nats-core';
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { JetStreamClient } from '@nats-io/jetstream';
|
|
2
|
+
import type { JobOptions } from '@pikku/core/queue';
|
|
3
|
+
import type { DelayedPublisher } from './nats-queue-service.js';
|
|
4
|
+
/**
|
|
5
|
+
* Delayed delivery using JetStream's own message scheduler (server 2.12+).
|
|
6
|
+
*
|
|
7
|
+
* Replaces the pg-boss republisher this package used to need. A message is
|
|
8
|
+
* published to a reserved schedule subject carrying `Nats-Schedule: @at <ts>`
|
|
9
|
+
* and `Nats-Schedule-Target: <the real queue subject>`; the server holds it and
|
|
10
|
+
* publishes the body verbatim onto the target when it comes due. Nothing polls,
|
|
11
|
+
* and no second store owns time.
|
|
12
|
+
*
|
|
13
|
+
* Two constraints shape the subject layout, both enforced by the server at
|
|
14
|
+
* publish time (error 10190, `message schedules target is invalid`):
|
|
15
|
+
*
|
|
16
|
+
* - The target must be a subject captured by the SAME stream as the schedule.
|
|
17
|
+
* Cross-stream scheduling does not exist; it would need a sourcing topology.
|
|
18
|
+
* - The target must not be the schedule's own subject, which would make the
|
|
19
|
+
* schedule reproduce itself forever.
|
|
20
|
+
*
|
|
21
|
+
* So schedules live inside the queue stream, under a subject prefix that no
|
|
22
|
+
* consumer filters on. That matters more than it looks: this is a WorkQueue
|
|
23
|
+
* stream, where a message is deleted the instant a consumer acks it, and a
|
|
24
|
+
* schedule that gets acked is a schedule that silently stops firing. No
|
|
25
|
+
* consumer subscribes to `<prefix>._schedule.>`, so nothing can ack one.
|
|
26
|
+
*
|
|
27
|
+
* One-shot `@at` schedules are purged by the server once they fire, so this
|
|
28
|
+
* leaves nothing behind to clean up.
|
|
29
|
+
*/
|
|
30
|
+
export declare class NatsDelayedPublisher implements DelayedPublisher {
|
|
31
|
+
private readonly js;
|
|
32
|
+
private readonly subjectPrefix;
|
|
33
|
+
constructor(js: JetStreamClient, subjectPrefix: string);
|
|
34
|
+
publishAfter<T>(queueName: string, data: T, delayMs: number, options?: JobOptions): Promise<string>;
|
|
35
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { headers } from '@nats-io/nats-core';
|
|
2
|
+
import { ATTEMPTS_HEADER, BACKOFF_DELAY_HEADER, BACKOFF_TYPE_HEADER, PIKKU_USER_ID_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, attemptsHeaderValue, scheduleSubjectFor, subjectForQueue, } from './utils.js';
|
|
3
|
+
/**
|
|
4
|
+
* Delayed delivery using JetStream's own message scheduler (server 2.12+).
|
|
5
|
+
*
|
|
6
|
+
* Replaces the pg-boss republisher this package used to need. A message is
|
|
7
|
+
* published to a reserved schedule subject carrying `Nats-Schedule: @at <ts>`
|
|
8
|
+
* and `Nats-Schedule-Target: <the real queue subject>`; the server holds it and
|
|
9
|
+
* publishes the body verbatim onto the target when it comes due. Nothing polls,
|
|
10
|
+
* and no second store owns time.
|
|
11
|
+
*
|
|
12
|
+
* Two constraints shape the subject layout, both enforced by the server at
|
|
13
|
+
* publish time (error 10190, `message schedules target is invalid`):
|
|
14
|
+
*
|
|
15
|
+
* - The target must be a subject captured by the SAME stream as the schedule.
|
|
16
|
+
* Cross-stream scheduling does not exist; it would need a sourcing topology.
|
|
17
|
+
* - The target must not be the schedule's own subject, which would make the
|
|
18
|
+
* schedule reproduce itself forever.
|
|
19
|
+
*
|
|
20
|
+
* So schedules live inside the queue stream, under a subject prefix that no
|
|
21
|
+
* consumer filters on. That matters more than it looks: this is a WorkQueue
|
|
22
|
+
* stream, where a message is deleted the instant a consumer acks it, and a
|
|
23
|
+
* schedule that gets acked is a schedule that silently stops firing. No
|
|
24
|
+
* consumer subscribes to `<prefix>._schedule.>`, so nothing can ack one.
|
|
25
|
+
*
|
|
26
|
+
* One-shot `@at` schedules are purged by the server once they fire, so this
|
|
27
|
+
* leaves nothing behind to clean up.
|
|
28
|
+
*/
|
|
29
|
+
export class NatsDelayedPublisher {
|
|
30
|
+
js;
|
|
31
|
+
subjectPrefix;
|
|
32
|
+
constructor(js, subjectPrefix) {
|
|
33
|
+
this.js = js;
|
|
34
|
+
this.subjectPrefix = subjectPrefix;
|
|
35
|
+
}
|
|
36
|
+
async publishAfter(queueName, data, delayMs, options) {
|
|
37
|
+
const hdrs = headers();
|
|
38
|
+
// RFC3339, which is what the server parses. A timestamp already in the past
|
|
39
|
+
// fires immediately rather than erroring — the same shape as pg-boss's
|
|
40
|
+
// `startAfter` with an elapsed date, so a delay that races the clock still
|
|
41
|
+
// delivers instead of being dropped.
|
|
42
|
+
hdrs.set(SCHEDULE_HEADER, `@at ${new Date(Date.now() + delayMs).toISOString()}`);
|
|
43
|
+
hdrs.set(SCHEDULE_TARGET_HEADER, subjectForQueue(this.subjectPrefix, queueName));
|
|
44
|
+
// Every non-schedule header is copied onto the produced message verbatim,
|
|
45
|
+
// so the job's retry policy survives the delay. `jobId` is deliberately not
|
|
46
|
+
// one of them, in either direction:
|
|
47
|
+
//
|
|
48
|
+
// - As `Nats-Msg-Id` it would be pointless — the server strips that header
|
|
49
|
+
// when producing, so it could never reach the stream's dedupe window.
|
|
50
|
+
// - As the schedule's subject it would be actively wrong. Republishing to
|
|
51
|
+
// a schedule subject REPLACES the schedule, so two delayed jobs sharing
|
|
52
|
+
// a jobId would silently cancel each other. The pg-boss publisher this
|
|
53
|
+
// replaces passed only `startAfter` and no singleton key, so delayed
|
|
54
|
+
// publishes have never deduped; making them dedupe now would be a
|
|
55
|
+
// behaviour change disguised as a port.
|
|
56
|
+
//
|
|
57
|
+
// Hence: no key, and every delayed publish gets its own subject.
|
|
58
|
+
const { delay: _delay, jobId: _jobId, ...rest } = options ?? {};
|
|
59
|
+
applyJobHeaders(hdrs, rest, queueName);
|
|
60
|
+
const ack = await this.js.publish(scheduleSubjectFor(this.subjectPrefix, queueName), JSON.stringify(data), { headers: hdrs });
|
|
61
|
+
return String(ack.seq);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Copy the queue-level job options that must survive the delay onto the
|
|
66
|
+
* schedule message. Kept here rather than shared with `NatsQueueService.add`
|
|
67
|
+
* because the two disagree deliberately about `jobId` — see above.
|
|
68
|
+
*/
|
|
69
|
+
const applyJobHeaders = (hdrs, options, queueName) => {
|
|
70
|
+
if (options.pikkuUserId) {
|
|
71
|
+
hdrs.set(PIKKU_USER_ID_HEADER, options.pikkuUserId);
|
|
72
|
+
}
|
|
73
|
+
if (options.attempts !== undefined) {
|
|
74
|
+
hdrs.set(ATTEMPTS_HEADER, attemptsHeaderValue(options.attempts, queueName));
|
|
75
|
+
}
|
|
76
|
+
if (options.backoff !== undefined) {
|
|
77
|
+
const backoff = typeof options.backoff === 'string'
|
|
78
|
+
? { type: options.backoff }
|
|
79
|
+
: options.backoff;
|
|
80
|
+
hdrs.set(BACKOFF_TYPE_HEADER, backoff.type);
|
|
81
|
+
if (backoff.delay !== undefined) {
|
|
82
|
+
hdrs.set(BACKOFF_DELAY_HEADER, String(backoff.delay));
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { JetStreamClient } from '@nats-io/jetstream';
|
|
2
|
+
import type { JobOptions, QueueJob, QueueService } from '@pikku/core/queue';
|
|
3
|
+
/**
|
|
4
|
+
* Delayed delivery. Implemented by `NatsDelayedPublisher` against the server's
|
|
5
|
+
* own message scheduler (2.12+); it used to be implemented against a store that
|
|
6
|
+
* owned time, because JetStream had no such primitive.
|
|
7
|
+
*/
|
|
8
|
+
export interface DelayedPublisher {
|
|
9
|
+
publishAfter<T>(queueName: string, data: T, delayMs: number, options?: JobOptions): Promise<string>;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* JetStream queue service for publishing jobs.
|
|
13
|
+
*
|
|
14
|
+
* Deliberately narrower than the pg-boss and bullmq services in two ways, both
|
|
15
|
+
* forced by the broker rather than by choice — see `supportsResults` and
|
|
16
|
+
* `getJob` below.
|
|
17
|
+
*/
|
|
18
|
+
export declare class NatsQueueService implements QueueService {
|
|
19
|
+
private readonly js;
|
|
20
|
+
private readonly subjectPrefix;
|
|
21
|
+
/**
|
|
22
|
+
* A work-queue stream deletes a message the moment it is acked, so there is
|
|
23
|
+
* nowhere for a result to live and nothing to read it back from. Producers
|
|
24
|
+
* that need a return value should use an RPC, not a queue.
|
|
25
|
+
*/
|
|
26
|
+
readonly supportsResults = false;
|
|
27
|
+
private delayedPublisher?;
|
|
28
|
+
constructor(js: JetStreamClient, subjectPrefix: string, delayedPublisher?: DelayedPublisher);
|
|
29
|
+
/**
|
|
30
|
+
* Attach the delayed publisher after construction. Needed because the pg-boss
|
|
31
|
+
* publisher republishes *through* this service, so the two reference each
|
|
32
|
+
* other and one of them has to be wired second.
|
|
33
|
+
*/
|
|
34
|
+
setDelayedPublisher(publisher: DelayedPublisher): void;
|
|
35
|
+
add<T>(queueName: string, data: T, options?: JobOptions): Promise<string>;
|
|
36
|
+
/**
|
|
37
|
+
* Not supported. A message in a work-queue stream is only reachable through a
|
|
38
|
+
* consumer, and only while it is being delivered — there is no lookup by id,
|
|
39
|
+
* and an acked message no longer exists. Returning null (rather than throwing)
|
|
40
|
+
* keeps callers that poll for optional status working; callers that genuinely
|
|
41
|
+
* need per-job state should read it from telemetry.
|
|
42
|
+
*/
|
|
43
|
+
getJob<T, R>(_queueName: string, _jobId: string): Promise<QueueJob<T, R> | null>;
|
|
44
|
+
}
|