@pikku/queue-nats 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/LICENSE +21 -0
- package/README.md +47 -0
- package/dist/consume.d.ts +54 -0
- package/dist/consume.js +213 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.js +42 -0
- package/dist/nats-delayed-publisher.d.ts +35 -0
- package/dist/nats-delayed-publisher.js +85 -0
- package/dist/nats-queue-service.d.ts +44 -0
- package/dist/nats-queue-service.js +84 -0
- package/dist/nats-queue-worker.d.ts +46 -0
- package/dist/nats-queue-worker.js +199 -0
- package/dist/nats-scheduler-service.d.ts +156 -0
- package/dist/nats-scheduler-service.js +369 -0
- package/dist/nats-service-factory.d.ts +104 -0
- package/dist/nats-service-factory.js +231 -0
- package/dist/utils.d.ts +122 -0
- package/dist/utils.js +154 -0
- package/package.json +33 -0
- package/run-tests.sh +56 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { jetstream, jetstreamManager, RetentionPolicy, } from '@nats-io/jetstream';
|
|
2
|
+
import { connect } from '@nats-io/transport-node';
|
|
3
|
+
import { NatsDelayedPublisher } from './nats-delayed-publisher.js';
|
|
4
|
+
import { NatsQueueService } from './nats-queue-service.js';
|
|
5
|
+
import { NatsSchedulerService } from './nats-scheduler-service.js';
|
|
6
|
+
import { NatsQueueWorkers } from './nats-queue-worker.js';
|
|
7
|
+
/** Server version that first shipped repeating (cron / `@every`) message schedules. */
|
|
8
|
+
const MIN_SCHEDULER_VERSION = [2, 14, 0];
|
|
9
|
+
/**
|
|
10
|
+
* Refuse to start against a server too old to schedule messages.
|
|
11
|
+
*
|
|
12
|
+
* Without this the failure is late, rare and cryptic: the stream config quietly
|
|
13
|
+
* drops `allow_msg_schedules`, every publish keeps working, and the first thing
|
|
14
|
+
* anyone notices is that a cron never fired — or a publish is rejected with
|
|
15
|
+
* `10188 message schedules is disabled`, minutes to hours after boot, on a code
|
|
16
|
+
* path nobody was watching. Since this package no longer carries a pg-boss
|
|
17
|
+
* fallback, an old server means scheduling is simply absent, so it is worth
|
|
18
|
+
* failing at connect time and naming the actual requirement.
|
|
19
|
+
*/
|
|
20
|
+
const assertSchedulerSupport = (version) => {
|
|
21
|
+
if (!version)
|
|
22
|
+
return;
|
|
23
|
+
const parts = version.split('.').map((n) => Number.parseInt(n, 10));
|
|
24
|
+
if (parts.length < 3 || parts.some(Number.isNaN))
|
|
25
|
+
return;
|
|
26
|
+
const [major, minor, patch] = parts;
|
|
27
|
+
const [reqMajor, reqMinor, reqPatch] = MIN_SCHEDULER_VERSION;
|
|
28
|
+
const ok = major > reqMajor ||
|
|
29
|
+
(major === reqMajor &&
|
|
30
|
+
(minor > reqMinor || (minor === reqMinor && patch >= reqPatch)));
|
|
31
|
+
if (ok)
|
|
32
|
+
return;
|
|
33
|
+
throw new Error(`NATS server ${version} is too old: message schedules (cron and delayed publish) need ` +
|
|
34
|
+
`${MIN_SCHEDULER_VERSION.join('.')}+. Upgrading is one-way — the stream-state file format ` +
|
|
35
|
+
`changed in 2.11 and again in 2.12, and there is no documented path back to 2.10 — so back ` +
|
|
36
|
+
`up the JetStream store directory before upgrading the server.`);
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Factory for the JetStream queue services.
|
|
40
|
+
*
|
|
41
|
+
* Mirrors PgBossServiceFactory, including `getSchedulerService`. That was not
|
|
42
|
+
* always true: this package originally had no scheduler at all, because
|
|
43
|
+
* JetStream had no delayed or scheduled message primitive and cron therefore
|
|
44
|
+
* stayed on pg-boss. Server 2.12 added one-shot `@at` delays and 2.14 added
|
|
45
|
+
* repeating cron schedules, so the split no longer buys anything and the
|
|
46
|
+
* pg-boss dependency is gone.
|
|
47
|
+
*/
|
|
48
|
+
export class NatsServiceFactory {
|
|
49
|
+
options;
|
|
50
|
+
connection;
|
|
51
|
+
js;
|
|
52
|
+
jsm;
|
|
53
|
+
queueService;
|
|
54
|
+
queueWorkers;
|
|
55
|
+
schedulerService;
|
|
56
|
+
initialized = false;
|
|
57
|
+
/** Set by `close()`, so the close watcher can tell a shutdown from a fault. */
|
|
58
|
+
closing = false;
|
|
59
|
+
constructor(options) {
|
|
60
|
+
this.options = options;
|
|
61
|
+
}
|
|
62
|
+
async init() {
|
|
63
|
+
if (this.initialized)
|
|
64
|
+
return;
|
|
65
|
+
this.connection = await connect({
|
|
66
|
+
servers: this.options.servers,
|
|
67
|
+
user: this.options.user,
|
|
68
|
+
pass: this.options.pass,
|
|
69
|
+
token: this.options.token,
|
|
70
|
+
name: this.options.name,
|
|
71
|
+
// RECONNECT FOREVER. The client default is 10 attempts, 2s apart, so about
|
|
72
|
+
// twenty seconds of NATS being unreachable CLOSES the connection for good —
|
|
73
|
+
// and `init()` runs once at boot, so nothing ever reopens it. The process
|
|
74
|
+
// then looks healthy (HTTP fine, database fine, reads fine) while every
|
|
75
|
+
// workflow dispatch throws `ClosedConnectionError` until someone restarts
|
|
76
|
+
// the backend. That is the shape of the intermittent "project stuck in
|
|
77
|
+
// creating" wedge: the row is written, the workflow that would build it is
|
|
78
|
+
// never enqueued, and nothing surfaces because the failure is in a queue
|
|
79
|
+
// publish rather than the request.
|
|
80
|
+
//
|
|
81
|
+
// A NATS restart routinely exceeds that budget, so the default turns
|
|
82
|
+
// ordinary maintenance into a silent outage that outlives it. There is no
|
|
83
|
+
// upside to giving up: a queue producer with nowhere to publish has no
|
|
84
|
+
// useful degraded mode, and retrying forever costs one socket attempt every
|
|
85
|
+
// few seconds.
|
|
86
|
+
maxReconnectAttempts: -1,
|
|
87
|
+
reconnectTimeWait: 2_000,
|
|
88
|
+
// Spread the herd. Every backend process reconnects off the same outage, and
|
|
89
|
+
// without jitter they retry in lockstep and hit the server together at the
|
|
90
|
+
// moment it is least able to take them.
|
|
91
|
+
reconnectJitter: 1_000,
|
|
92
|
+
// Boot must not race the queue either: without this, a backend starting
|
|
93
|
+
// while NATS is still coming up fails `init()` outright rather than waiting
|
|
94
|
+
// for it, which is the same outage one layer earlier.
|
|
95
|
+
waitOnFirstConnect: true,
|
|
96
|
+
});
|
|
97
|
+
void this.watchForUnexpectedClose(this.connection);
|
|
98
|
+
assertSchedulerSupport(this.connection.info?.version);
|
|
99
|
+
const jsm = await jetstreamManager(this.connection);
|
|
100
|
+
const js = jetstream(this.connection);
|
|
101
|
+
this.js = js;
|
|
102
|
+
this.jsm = jsm;
|
|
103
|
+
// WorkQueue retention: a message is removed from the stream the instant it
|
|
104
|
+
// is acked. That is what makes this a queue rather than a log, and it is
|
|
105
|
+
// also why there is no history here — a completed job leaves no trace in
|
|
106
|
+
// NATS, so success/failure rates must come from telemetry instead.
|
|
107
|
+
//
|
|
108
|
+
// WorkQueue also requires consumer filter subjects not to overlap, which is
|
|
109
|
+
// satisfied by construction: one consumer per queue, filtered to exactly
|
|
110
|
+
// that queue's subject.
|
|
111
|
+
const streamConfig = {
|
|
112
|
+
name: this.options.streamName,
|
|
113
|
+
subjects: [`${this.options.subjectPrefix}.>`],
|
|
114
|
+
retention: RetentionPolicy.Workqueue,
|
|
115
|
+
num_replicas: this.options.replicas ?? 1,
|
|
116
|
+
// Lets this stream hold schedule messages, which is how delayed publishes
|
|
117
|
+
// and cron work — see NatsDelayedPublisher. Schedules must live in the
|
|
118
|
+
// same stream as the subject they target, so this goes on the queue
|
|
119
|
+
// stream itself rather than a stream beside it.
|
|
120
|
+
//
|
|
121
|
+
// Safe to turn on for an existing stream (it applies without recreating),
|
|
122
|
+
// but note it is one-way: the server has no way to disable it again. It
|
|
123
|
+
// also implicitly enables AllowRollup and clears DenyPurge, because
|
|
124
|
+
// schedules are stored as rollup-subject messages.
|
|
125
|
+
allow_msg_schedules: true,
|
|
126
|
+
...(this.options.duplicateWindowMs !== undefined
|
|
127
|
+
? { duplicate_window: this.options.duplicateWindowMs * 1_000_000 }
|
|
128
|
+
: {}),
|
|
129
|
+
};
|
|
130
|
+
// `add` is STREAM.CREATE, which errors if the stream exists with a different
|
|
131
|
+
// config — so the first deploy that changes duplicate_window or replicas
|
|
132
|
+
// would fail at boot rather than converge. Try create, then update. Update
|
|
133
|
+
// is the right call for a stream that exists: it applies the new config in
|
|
134
|
+
// place without touching the messages already in it.
|
|
135
|
+
try {
|
|
136
|
+
await jsm.streams.add(streamConfig);
|
|
137
|
+
}
|
|
138
|
+
catch (err) {
|
|
139
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
140
|
+
// Only "already exists" is recoverable this way; anything else (no
|
|
141
|
+
// JetStream enabled, bad subject, insufficient replicas for the cluster
|
|
142
|
+
// size) must surface, not be papered over by a second failing call.
|
|
143
|
+
if (!/already in use|already exists/i.test(message))
|
|
144
|
+
throw err;
|
|
145
|
+
await jsm.streams.update(this.options.streamName, streamConfig);
|
|
146
|
+
}
|
|
147
|
+
// Wired at construction rather than via setDelayedPublisher: the pg-boss
|
|
148
|
+
// publisher had to be attached late because it republished *through* this
|
|
149
|
+
// service, so the two referenced each other. The native one only needs the
|
|
150
|
+
// JetStream client, so the cycle is gone.
|
|
151
|
+
this.queueService = new NatsQueueService(js, this.options.subjectPrefix, new NatsDelayedPublisher(js, this.options.subjectPrefix));
|
|
152
|
+
this.queueWorkers = new NatsQueueWorkers(js, jsm, this.options.streamName, this.options.subjectPrefix, this.options.defaultConsumerConfig);
|
|
153
|
+
this.schedulerService = new NatsSchedulerService(js, jsm, this.options.streamName, this.options.subjectPrefix, this.options.schedulerTimezone);
|
|
154
|
+
this.initialized = true;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Surfaces a connection that closed without being asked to — see
|
|
158
|
+
* `onConnectionLost`. Detached on purpose: `closed()` resolves only at the end
|
|
159
|
+
* of the connection's life, so awaiting it would never let `init()` return.
|
|
160
|
+
*/
|
|
161
|
+
async watchForUnexpectedClose(connection) {
|
|
162
|
+
const err = await connection.closed();
|
|
163
|
+
if (this.closing)
|
|
164
|
+
return;
|
|
165
|
+
const reason = err instanceof Error ? err : undefined;
|
|
166
|
+
console.error(`NATS connection closed unexpectedly and will not reopen — this process can no longer publish jobs or run cron. ${reason?.message ?? ''}`);
|
|
167
|
+
this.options.onConnectionLost?.(reason);
|
|
168
|
+
}
|
|
169
|
+
getQueueService() {
|
|
170
|
+
if (!this.queueService) {
|
|
171
|
+
throw new Error('NatsServiceFactory.init() must be awaited before use');
|
|
172
|
+
}
|
|
173
|
+
return this.queueService;
|
|
174
|
+
}
|
|
175
|
+
getSchedulerService() {
|
|
176
|
+
if (!this.schedulerService) {
|
|
177
|
+
throw new Error('NatsServiceFactory.init() must be awaited before use');
|
|
178
|
+
}
|
|
179
|
+
return this.schedulerService;
|
|
180
|
+
}
|
|
181
|
+
getQueueWorkers() {
|
|
182
|
+
if (!this.queueWorkers) {
|
|
183
|
+
throw new Error('NatsServiceFactory.init() must be awaited before use');
|
|
184
|
+
}
|
|
185
|
+
return this.queueWorkers;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Raw JetStream handles, for callers that run their own consumers instead of
|
|
189
|
+
* going through pikku's queue registry — see `consumeQueue`.
|
|
190
|
+
*/
|
|
191
|
+
getJetStream() {
|
|
192
|
+
if (!this.js || !this.jsm) {
|
|
193
|
+
throw new Error('NatsServiceFactory.init() must be awaited before use');
|
|
194
|
+
}
|
|
195
|
+
return { js: this.js, jsm: this.jsm };
|
|
196
|
+
}
|
|
197
|
+
getConnection() {
|
|
198
|
+
if (!this.connection) {
|
|
199
|
+
throw new Error('NatsServiceFactory.init() must be awaited before use');
|
|
200
|
+
}
|
|
201
|
+
return this.connection;
|
|
202
|
+
}
|
|
203
|
+
/** Drain the queue workers, if any are running. Returns the queues that still
|
|
204
|
+
* had jobs in flight. Safe before `init()`. */
|
|
205
|
+
async drainWorkers(timeoutMs) {
|
|
206
|
+
return (await this.queueWorkers?.drain(timeoutMs)) ?? [];
|
|
207
|
+
}
|
|
208
|
+
async close() {
|
|
209
|
+
// Guarded on the connection rather than on `initialized`, because the two
|
|
210
|
+
// diverge exactly when it matters: if `init()` fails after `connect()`
|
|
211
|
+
// resolved — an old server rejected by `assertSchedulerSupport` is the
|
|
212
|
+
// likely one — `initialized` is still false while a live connection is
|
|
213
|
+
// already reconnecting forever, and an `initialized` guard would leak it
|
|
214
|
+
// and keep the process alive.
|
|
215
|
+
if (!this.connection)
|
|
216
|
+
return;
|
|
217
|
+
this.closing = true;
|
|
218
|
+
await this.schedulerService?.stop();
|
|
219
|
+
await this.queueWorkers?.stop();
|
|
220
|
+
// `drain` rather than `close`: it flushes pending acks and lets in-flight
|
|
221
|
+
// publishes land instead of dropping them, which is the difference between
|
|
222
|
+
// a clean deploy and a batch of messages redelivered after ack_wait.
|
|
223
|
+
await this.connection.drain();
|
|
224
|
+
this.connection = undefined;
|
|
225
|
+
this.initialized = false;
|
|
226
|
+
}
|
|
227
|
+
/** Alias for close() — used by stopSingletonServices. */
|
|
228
|
+
async stop() {
|
|
229
|
+
await this.close();
|
|
230
|
+
}
|
|
231
|
+
}
|
package/dist/utils.d.ts
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import type { JsMsg } from '@nats-io/jetstream';
|
|
2
|
+
import type { QueueJob, QueueJobStatus } from '@pikku/core/queue';
|
|
3
|
+
/** Header carrying the producer's pikku user id through to the worker. */
|
|
4
|
+
export declare const PIKKU_USER_ID_HEADER = "Pikku-User-Id";
|
|
5
|
+
/** Header JetStream itself reads for publish-side dedupe within the stream's window. */
|
|
6
|
+
export declare const NATS_MSG_ID_HEADER = "Nats-Msg-Id";
|
|
7
|
+
/**
|
|
8
|
+
* Per-job retry policy, carried in headers.
|
|
9
|
+
*
|
|
10
|
+
* pg-boss and bullmq store `attempts`/`backoff` per job; JetStream only has
|
|
11
|
+
* per-consumer `max_deliver` and a plain `nak`. Since the workflow engine sets
|
|
12
|
+
* these per step — and treats `attempts: 1` as "never retry this" — a
|
|
13
|
+
* per-consumer setting cannot express it. So the policy rides with the message
|
|
14
|
+
* and the worker enforces it. Without this a step marked `retries: 0` would be
|
|
15
|
+
* redelivered forever, because an unset `max_deliver` means unlimited.
|
|
16
|
+
*/
|
|
17
|
+
export declare const ATTEMPTS_HEADER = "Pikku-Attempts";
|
|
18
|
+
export declare const BACKOFF_TYPE_HEADER = "Pikku-Backoff-Type";
|
|
19
|
+
export declare const BACKOFF_DELAY_HEADER = "Pikku-Backoff-Delay";
|
|
20
|
+
/**
|
|
21
|
+
* Header value for a job's retry limit, rejecting anything the worker could not
|
|
22
|
+
* enforce.
|
|
23
|
+
*
|
|
24
|
+
* `JobOptions.attempts` is typed as a plain number, and an unenforceable value
|
|
25
|
+
* fails in the worst direction: `attemptsFor` reads a zero, a negative or a NaN
|
|
26
|
+
* back as "no limit set", and the worker then naks forever. A fraction is worse
|
|
27
|
+
* than useless — it survives as a number and silently means `ceil(attempts)`.
|
|
28
|
+
* Both are caller mistakes, so they are refused at enqueue rather than turned
|
|
29
|
+
* into an infinite redelivery loop nobody asked for.
|
|
30
|
+
*/
|
|
31
|
+
export declare const attemptsHeaderValue: (attempts: number, queueName: string) => string;
|
|
32
|
+
/** Default base delay when a job asks for backoff without naming one. */
|
|
33
|
+
export declare const DEFAULT_BACKOFF_DELAY_MS = 1000;
|
|
34
|
+
/**
|
|
35
|
+
* Delay before redelivering a message on its `deliveryCount`-th failure.
|
|
36
|
+
* Exponential mirrors bullmq/pg-boss: base * 2^(n-1), capped so a long-lived
|
|
37
|
+
* retry cannot drift past any sane ack_wait/visibility reasoning.
|
|
38
|
+
*/
|
|
39
|
+
export declare const backoffDelayMs: (deliveryCount: number, type: string | undefined, baseMs?: number) => number;
|
|
40
|
+
/**
|
|
41
|
+
* Pikku queue names are free-form strings; NATS subject tokens are not. `.` is
|
|
42
|
+
* the subject separator and `*`/`>` are wildcards, so a queue name containing
|
|
43
|
+
* any of them would silently widen a consumer's filter and start stealing other
|
|
44
|
+
* queues' messages. Replace them rather than rejecting, and keep the mapping
|
|
45
|
+
* total so the same queue name always resolves to the same subject.
|
|
46
|
+
*
|
|
47
|
+
* Not injective, and knowingly so: `a.b` and `a_b` collapse to the same token,
|
|
48
|
+
* so two queues named that way would share one subject and one durable
|
|
49
|
+
* consumer. An escaping scheme would fix it, at the cost of changing the
|
|
50
|
+
* subject and durable name of every queue that already exists — which strands
|
|
51
|
+
* the backlog on the old names. Not worth it for a collision that needs two
|
|
52
|
+
* queue names differing only in a separator.
|
|
53
|
+
*/
|
|
54
|
+
export declare const queueNameToToken: (queueName: string) => string;
|
|
55
|
+
/** Subject a job for `queueName` is published to. */
|
|
56
|
+
export declare const subjectForQueue: (prefix: string, queueName: string) => string;
|
|
57
|
+
/**
|
|
58
|
+
* Headers driving JetStream's message scheduler (server 2.12+ for `@at`, 2.14+
|
|
59
|
+
* for repeating schedules). The server strips every `Nats-Schedule*` header
|
|
60
|
+
* from the message it produces, so these never reach a worker.
|
|
61
|
+
*/
|
|
62
|
+
export declare const SCHEDULE_HEADER = "Nats-Schedule";
|
|
63
|
+
export declare const SCHEDULE_TARGET_HEADER = "Nats-Schedule-Target";
|
|
64
|
+
export declare const SCHEDULE_TIMEZONE_HEADER = "Nats-Schedule-Time-Zone";
|
|
65
|
+
/** Set by the server on produced messages: the subject holding the schedule. */
|
|
66
|
+
export declare const SCHEDULER_HEADER = "Nats-Scheduler";
|
|
67
|
+
/** Set by the server on produced messages; also the client's cancel verb (`purge`). */
|
|
68
|
+
export declare const SCHEDULE_NEXT_HEADER = "Nats-Schedule-Next";
|
|
69
|
+
/**
|
|
70
|
+
* Subject segment reserved for schedule messages.
|
|
71
|
+
*
|
|
72
|
+
* A schedule must live in the same stream as the subject it targets, so these
|
|
73
|
+
* sit inside the queue stream rather than beside it. The segment leads with `_`
|
|
74
|
+
* so it cannot collide with a queue token: `queueNameToToken` never introduces
|
|
75
|
+
* a leading underscore, and a queue literally named `_schedule` would collide
|
|
76
|
+
* with itself long before it reached here.
|
|
77
|
+
*
|
|
78
|
+
* Nothing consumes this subject, and that is load-bearing on a WorkQueue
|
|
79
|
+
* stream: a consumer that acked a schedule message would delete it, and a
|
|
80
|
+
* deleted schedule stops firing with no error and no advisory.
|
|
81
|
+
*/
|
|
82
|
+
export declare const SCHEDULE_SUBJECT_SEGMENT = "_schedule";
|
|
83
|
+
/**
|
|
84
|
+
* Subject holding the schedule for a queue.
|
|
85
|
+
*
|
|
86
|
+
* Every schedule needs its own subject, because the last message on a subject
|
|
87
|
+
* *is* the schedule — republishing to the same subject replaces it. For
|
|
88
|
+
* repeating schedules that is exactly the update semantics we want, so `key`
|
|
89
|
+
* identifies the schedule. For one-shot delayed publishes there is no such
|
|
90
|
+
* identity, and reusing a subject would make two concurrent delayed jobs
|
|
91
|
+
* silently overwrite each other, so an absent key gets a unique suffix.
|
|
92
|
+
*/
|
|
93
|
+
export declare const scheduleSubjectFor: (prefix: string, queueName: string, key?: string) => string;
|
|
94
|
+
/**
|
|
95
|
+
* Durable consumer name for a queue. NATS rejects `.` in durable names, so both
|
|
96
|
+
* halves have to be tokenised — the prefix as much as the queue name, since a
|
|
97
|
+
* hierarchical prefix like `app.dispatch` is perfectly valid as a subject
|
|
98
|
+
* and would otherwise produce a name the server refuses.
|
|
99
|
+
*/
|
|
100
|
+
export declare const consumerNameForQueue: (prefix: string, queueName: string) => string;
|
|
101
|
+
/**
|
|
102
|
+
* A JetStream message has no terminal states to report: once it is acked it is
|
|
103
|
+
* removed from a work-queue stream, and once it is termed it is gone too. So a
|
|
104
|
+
* message we are holding is always mid-delivery — 'active' is the only status
|
|
105
|
+
* that can be observed from the message itself. Historical outcomes have to come
|
|
106
|
+
* from telemetry, not from the broker.
|
|
107
|
+
*/
|
|
108
|
+
export declare const jsMsgStatus: () => QueueJobStatus;
|
|
109
|
+
/** The job's own retry limit, if the producer set one. */
|
|
110
|
+
export declare const attemptsFor: (msg: JsMsg) => number | undefined;
|
|
111
|
+
/** Redelivery delay for this message's next attempt, per its backoff policy. */
|
|
112
|
+
export declare const backoffFor: (msg: JsMsg) => number;
|
|
113
|
+
export declare const jsMsgMetadata: (msg: JsMsg) => {
|
|
114
|
+
attemptsMade: number;
|
|
115
|
+
createdAt: Date;
|
|
116
|
+
processedAt: Date;
|
|
117
|
+
};
|
|
118
|
+
/**
|
|
119
|
+
* Wrap a JetStream message as a pikku QueueJob. `waitForCompletion` is absent
|
|
120
|
+
* deliberately — see NatsQueueService.supportsResults.
|
|
121
|
+
*/
|
|
122
|
+
export declare const mapJsMsgToQueueJob: <T, R>(queueName: string, msg: JsMsg) => QueueJob<T, R>;
|
package/dist/utils.js
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/** Header carrying the producer's pikku user id through to the worker. */
|
|
2
|
+
export const PIKKU_USER_ID_HEADER = 'Pikku-User-Id';
|
|
3
|
+
/** Header JetStream itself reads for publish-side dedupe within the stream's window. */
|
|
4
|
+
export const NATS_MSG_ID_HEADER = 'Nats-Msg-Id';
|
|
5
|
+
/**
|
|
6
|
+
* Per-job retry policy, carried in headers.
|
|
7
|
+
*
|
|
8
|
+
* pg-boss and bullmq store `attempts`/`backoff` per job; JetStream only has
|
|
9
|
+
* per-consumer `max_deliver` and a plain `nak`. Since the workflow engine sets
|
|
10
|
+
* these per step — and treats `attempts: 1` as "never retry this" — a
|
|
11
|
+
* per-consumer setting cannot express it. So the policy rides with the message
|
|
12
|
+
* and the worker enforces it. Without this a step marked `retries: 0` would be
|
|
13
|
+
* redelivered forever, because an unset `max_deliver` means unlimited.
|
|
14
|
+
*/
|
|
15
|
+
export const ATTEMPTS_HEADER = 'Pikku-Attempts';
|
|
16
|
+
export const BACKOFF_TYPE_HEADER = 'Pikku-Backoff-Type';
|
|
17
|
+
export const BACKOFF_DELAY_HEADER = 'Pikku-Backoff-Delay';
|
|
18
|
+
/**
|
|
19
|
+
* Header value for a job's retry limit, rejecting anything the worker could not
|
|
20
|
+
* enforce.
|
|
21
|
+
*
|
|
22
|
+
* `JobOptions.attempts` is typed as a plain number, and an unenforceable value
|
|
23
|
+
* fails in the worst direction: `attemptsFor` reads a zero, a negative or a NaN
|
|
24
|
+
* back as "no limit set", and the worker then naks forever. A fraction is worse
|
|
25
|
+
* than useless — it survives as a number and silently means `ceil(attempts)`.
|
|
26
|
+
* Both are caller mistakes, so they are refused at enqueue rather than turned
|
|
27
|
+
* into an infinite redelivery loop nobody asked for.
|
|
28
|
+
*/
|
|
29
|
+
export const attemptsHeaderValue = (attempts, queueName) => {
|
|
30
|
+
if (!Number.isSafeInteger(attempts) || attempts < 1) {
|
|
31
|
+
throw new Error(`Queue "${queueName}" was given attempts=${attempts}, which is not a positive integer. An unenforceable limit reads back as "no limit" and the job would be retried forever; pass attempts >= 1 or omit it.`);
|
|
32
|
+
}
|
|
33
|
+
return String(attempts);
|
|
34
|
+
};
|
|
35
|
+
/** Default base delay when a job asks for backoff without naming one. */
|
|
36
|
+
export const DEFAULT_BACKOFF_DELAY_MS = 1_000;
|
|
37
|
+
/**
|
|
38
|
+
* Delay before redelivering a message on its `deliveryCount`-th failure.
|
|
39
|
+
* Exponential mirrors bullmq/pg-boss: base * 2^(n-1), capped so a long-lived
|
|
40
|
+
* retry cannot drift past any sane ack_wait/visibility reasoning.
|
|
41
|
+
*/
|
|
42
|
+
export const backoffDelayMs = (deliveryCount, type, baseMs = DEFAULT_BACKOFF_DELAY_MS) => {
|
|
43
|
+
if (!type)
|
|
44
|
+
return 0;
|
|
45
|
+
if (type === 'fixed')
|
|
46
|
+
return baseMs;
|
|
47
|
+
// 'exponential' and anything unrecognised — growing is the safe default,
|
|
48
|
+
// since the failure mode of guessing wrong is retrying too slowly, not too fast.
|
|
49
|
+
const capMs = 60 * 60_000;
|
|
50
|
+
return Math.min(baseMs * 2 ** Math.max(0, deliveryCount - 1), capMs);
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* Pikku queue names are free-form strings; NATS subject tokens are not. `.` is
|
|
54
|
+
* the subject separator and `*`/`>` are wildcards, so a queue name containing
|
|
55
|
+
* any of them would silently widen a consumer's filter and start stealing other
|
|
56
|
+
* queues' messages. Replace them rather than rejecting, and keep the mapping
|
|
57
|
+
* total so the same queue name always resolves to the same subject.
|
|
58
|
+
*
|
|
59
|
+
* Not injective, and knowingly so: `a.b` and `a_b` collapse to the same token,
|
|
60
|
+
* so two queues named that way would share one subject and one durable
|
|
61
|
+
* consumer. An escaping scheme would fix it, at the cost of changing the
|
|
62
|
+
* subject and durable name of every queue that already exists — which strands
|
|
63
|
+
* the backlog on the old names. Not worth it for a collision that needs two
|
|
64
|
+
* queue names differing only in a separator.
|
|
65
|
+
*/
|
|
66
|
+
export const queueNameToToken = (queueName) => queueName.replace(/[.*>\s]/g, '_');
|
|
67
|
+
/** Subject a job for `queueName` is published to. */
|
|
68
|
+
export const subjectForQueue = (prefix, queueName) => `${prefix}.${queueNameToToken(queueName)}`;
|
|
69
|
+
/**
|
|
70
|
+
* Headers driving JetStream's message scheduler (server 2.12+ for `@at`, 2.14+
|
|
71
|
+
* for repeating schedules). The server strips every `Nats-Schedule*` header
|
|
72
|
+
* from the message it produces, so these never reach a worker.
|
|
73
|
+
*/
|
|
74
|
+
export const SCHEDULE_HEADER = 'Nats-Schedule';
|
|
75
|
+
export const SCHEDULE_TARGET_HEADER = 'Nats-Schedule-Target';
|
|
76
|
+
export const SCHEDULE_TIMEZONE_HEADER = 'Nats-Schedule-Time-Zone';
|
|
77
|
+
/** Set by the server on produced messages: the subject holding the schedule. */
|
|
78
|
+
export const SCHEDULER_HEADER = 'Nats-Scheduler';
|
|
79
|
+
/** Set by the server on produced messages; also the client's cancel verb (`purge`). */
|
|
80
|
+
export const SCHEDULE_NEXT_HEADER = 'Nats-Schedule-Next';
|
|
81
|
+
/**
|
|
82
|
+
* Subject segment reserved for schedule messages.
|
|
83
|
+
*
|
|
84
|
+
* A schedule must live in the same stream as the subject it targets, so these
|
|
85
|
+
* sit inside the queue stream rather than beside it. The segment leads with `_`
|
|
86
|
+
* so it cannot collide with a queue token: `queueNameToToken` never introduces
|
|
87
|
+
* a leading underscore, and a queue literally named `_schedule` would collide
|
|
88
|
+
* with itself long before it reached here.
|
|
89
|
+
*
|
|
90
|
+
* Nothing consumes this subject, and that is load-bearing on a WorkQueue
|
|
91
|
+
* stream: a consumer that acked a schedule message would delete it, and a
|
|
92
|
+
* deleted schedule stops firing with no error and no advisory.
|
|
93
|
+
*/
|
|
94
|
+
export const SCHEDULE_SUBJECT_SEGMENT = '_schedule';
|
|
95
|
+
/**
|
|
96
|
+
* Subject holding the schedule for a queue.
|
|
97
|
+
*
|
|
98
|
+
* Every schedule needs its own subject, because the last message on a subject
|
|
99
|
+
* *is* the schedule — republishing to the same subject replaces it. For
|
|
100
|
+
* repeating schedules that is exactly the update semantics we want, so `key`
|
|
101
|
+
* identifies the schedule. For one-shot delayed publishes there is no such
|
|
102
|
+
* identity, and reusing a subject would make two concurrent delayed jobs
|
|
103
|
+
* silently overwrite each other, so an absent key gets a unique suffix.
|
|
104
|
+
*/
|
|
105
|
+
export const scheduleSubjectFor = (prefix, queueName, key) => `${prefix}.${SCHEDULE_SUBJECT_SEGMENT}.${queueNameToToken(queueName)}.${key
|
|
106
|
+
? queueNameToToken(key)
|
|
107
|
+
: `${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`}`;
|
|
108
|
+
/**
|
|
109
|
+
* Durable consumer name for a queue. NATS rejects `.` in durable names, so both
|
|
110
|
+
* halves have to be tokenised — the prefix as much as the queue name, since a
|
|
111
|
+
* hierarchical prefix like `app.dispatch` is perfectly valid as a subject
|
|
112
|
+
* and would otherwise produce a name the server refuses.
|
|
113
|
+
*/
|
|
114
|
+
export const consumerNameForQueue = (prefix, queueName) => `${queueNameToToken(prefix)}_${queueNameToToken(queueName)}`;
|
|
115
|
+
/**
|
|
116
|
+
* A JetStream message has no terminal states to report: once it is acked it is
|
|
117
|
+
* removed from a work-queue stream, and once it is termed it is gone too. So a
|
|
118
|
+
* message we are holding is always mid-delivery — 'active' is the only status
|
|
119
|
+
* that can be observed from the message itself. Historical outcomes have to come
|
|
120
|
+
* from telemetry, not from the broker.
|
|
121
|
+
*/
|
|
122
|
+
export const jsMsgStatus = () => 'active';
|
|
123
|
+
/** The job's own retry limit, if the producer set one. */
|
|
124
|
+
export const attemptsFor = (msg) => {
|
|
125
|
+
const raw = msg.headers?.get(ATTEMPTS_HEADER);
|
|
126
|
+
if (!raw)
|
|
127
|
+
return undefined;
|
|
128
|
+
const n = Number(raw);
|
|
129
|
+
return Number.isFinite(n) && n > 0 ? n : undefined;
|
|
130
|
+
};
|
|
131
|
+
/** Redelivery delay for this message's next attempt, per its backoff policy. */
|
|
132
|
+
export const backoffFor = (msg) => {
|
|
133
|
+
const type = msg.headers?.get(BACKOFF_TYPE_HEADER) || undefined;
|
|
134
|
+
const rawBase = msg.headers?.get(BACKOFF_DELAY_HEADER);
|
|
135
|
+
const base = rawBase ? Number(rawBase) : undefined;
|
|
136
|
+
return backoffDelayMs(msg.info.deliveryCount, type, Number.isFinite(base) && base > 0 ? base : DEFAULT_BACKOFF_DELAY_MS);
|
|
137
|
+
};
|
|
138
|
+
export const jsMsgMetadata = (msg) => ({
|
|
139
|
+
attemptsMade: msg.info.deliveryCount,
|
|
140
|
+
createdAt: new Date(Number(msg.info.timestampNanos) / 1_000_000),
|
|
141
|
+
processedAt: new Date(),
|
|
142
|
+
});
|
|
143
|
+
/**
|
|
144
|
+
* Wrap a JetStream message as a pikku QueueJob. `waitForCompletion` is absent
|
|
145
|
+
* deliberately — see NatsQueueService.supportsResults.
|
|
146
|
+
*/
|
|
147
|
+
export const mapJsMsgToQueueJob = (queueName, msg) => ({
|
|
148
|
+
id: String(msg.seq),
|
|
149
|
+
queueName,
|
|
150
|
+
status: jsMsgStatus,
|
|
151
|
+
data: msg.json(),
|
|
152
|
+
metadata: () => jsMsgMetadata(msg),
|
|
153
|
+
pikkuUserId: msg.headers?.get(PIKKU_USER_ID_HEADER) || undefined,
|
|
154
|
+
});
|
package/package.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@pikku/queue-nats",
|
|
3
|
+
"version": "0.12.1",
|
|
4
|
+
"description": "NATS JetStream queue, worker and scheduler services for Pikku",
|
|
5
|
+
"author": "yasser.fadl@gmail.com",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"module": "dist/index.js",
|
|
8
|
+
"main": "dist/index.js",
|
|
9
|
+
"type": "module",
|
|
10
|
+
"scripts": {
|
|
11
|
+
"tsc": "tsc",
|
|
12
|
+
"test": "bash run-tests.sh",
|
|
13
|
+
"test:coverage": "bash run-tests.sh --coverage",
|
|
14
|
+
"ncu": "npx npm-check-updates",
|
|
15
|
+
"build": "tsc -b",
|
|
16
|
+
"release": "npm run build && npm test",
|
|
17
|
+
"prepublishOnly": "yarn build"
|
|
18
|
+
},
|
|
19
|
+
"peerDependencies": {
|
|
20
|
+
"@pikku/core": "^0.12.105"
|
|
21
|
+
},
|
|
22
|
+
"dependencies": {
|
|
23
|
+
"@nats-io/jetstream": "^3.4.0",
|
|
24
|
+
"@nats-io/nats-core": "^3.4.0",
|
|
25
|
+
"@nats-io/transport-node": "^3.4.0"
|
|
26
|
+
},
|
|
27
|
+
"devDependencies": {
|
|
28
|
+
"typescript": "^6.0.3"
|
|
29
|
+
},
|
|
30
|
+
"exports": {
|
|
31
|
+
".": "./dist/index.js"
|
|
32
|
+
}
|
|
33
|
+
}
|
package/run-tests.sh
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
4
|
+
cd "$script_dir"
|
|
5
|
+
|
|
6
|
+
# Enable nullglob to handle cases where no files match the pattern
|
|
7
|
+
shopt -s nullglob
|
|
8
|
+
|
|
9
|
+
# Initialize variables for options
|
|
10
|
+
watch_mode=false
|
|
11
|
+
coverage_mode=false
|
|
12
|
+
|
|
13
|
+
# Parse command-line options
|
|
14
|
+
while [[ $# -gt 0 ]]; do
|
|
15
|
+
case $1 in
|
|
16
|
+
--watch)
|
|
17
|
+
watch_mode=true
|
|
18
|
+
shift
|
|
19
|
+
;;
|
|
20
|
+
--coverage)
|
|
21
|
+
coverage_mode=true
|
|
22
|
+
shift
|
|
23
|
+
;;
|
|
24
|
+
*)
|
|
25
|
+
echo "Unknown option: $1"
|
|
26
|
+
exit 1
|
|
27
|
+
;;
|
|
28
|
+
esac
|
|
29
|
+
done
|
|
30
|
+
|
|
31
|
+
# Define the pattern to match your test files
|
|
32
|
+
pattern="src/*.test.ts"
|
|
33
|
+
|
|
34
|
+
# Expand the pattern into an array of files
|
|
35
|
+
files=($(find src -type f -name "*.test.ts"))
|
|
36
|
+
|
|
37
|
+
# Check if any files matched the pattern
|
|
38
|
+
if [ ${#files[@]} -eq 0 ]; then
|
|
39
|
+
echo "No test files found matching pattern: $pattern"
|
|
40
|
+
exit 0
|
|
41
|
+
fi
|
|
42
|
+
|
|
43
|
+
# Construct the node command
|
|
44
|
+
node_cmd=(node --import tsx --test)
|
|
45
|
+
|
|
46
|
+
# Append options based on flags
|
|
47
|
+
if [ "$watch_mode" = true ]; then
|
|
48
|
+
node_cmd+=(--watch)
|
|
49
|
+
fi
|
|
50
|
+
|
|
51
|
+
if [ "$coverage_mode" = true ]; then
|
|
52
|
+
node_cmd+=(--test-coverage-include="src/**/*.{ts,js}" --test-coverage-exclude="**/dist/**" --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info)
|
|
53
|
+
fi
|
|
54
|
+
|
|
55
|
+
# Execute the node command with the expanded list of files
|
|
56
|
+
"${node_cmd[@]}" "${files[@]}"
|