@pikku/queue-nats 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md ADDED
@@ -0,0 +1,9 @@
1
+ # @pikku/queue-nats
2
+
3
+ ## 0.12.1
4
+
5
+ ### Patch Changes
6
+
7
+ - 427c4fe: **`@pikku/queue-nats`** — NATS JetStream queue, worker and scheduler services, alongside the pg-boss and bullmq adapters. Jobs publish to a work-queue stream, each queue gets its own durable pull consumer, and `max_ack_pending` is real per-message concurrency rather than a batch size, so a slow job occupies one slot instead of holding its batch siblings. The per-job retry policy travels in headers because JetStream's `max_deliver` is per-consumer and cannot express a step marked "never retry", and `ack_wait` is always set to pg-boss's 900s parity rather than left to the server's 30s default — unset, any job running longer than 30s is redelivered while it is still running and executed twice. Cron and one-shot delays use the server's own message scheduler, so a schedule is a retained message on a reserved subject: publishing registers it, republishing replaces it, purging cancels it. That needs a server on 2.14+, and `NatsServiceFactory` refuses to start against anything older rather than silently not scheduling. `supportsResults` is `false`, deliberately — a work-queue stream deletes a message on ack, so there is nowhere for a job result or any job history to live, and per-job outcomes come from telemetry instead.
8
+ - Updated dependencies [1ab831e]
9
+ - @pikku/core@0.12.105
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2021 - present Yasser Fadl and Pikku contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,47 @@
1
+ # @pikku/queue-nats
2
+
3
+ NATS JetStream queue, worker and scheduler services for Pikku. Backed by a
4
+ JetStream work-queue stream, with the schedules living in the broker rather
5
+ than in a database.
6
+
7
+ Requires a NATS server on 2.14 or newer — that is the release that added
8
+ repeating cron schedules, and `NatsServiceFactory` refuses to start against
9
+ anything older rather than silently not scheduling.
10
+
11
+ ## Install
12
+
13
+ ```bash
14
+ npm install @pikku/queue-nats
15
+ ```
16
+
17
+ ## Usage
18
+
19
+ ```typescript
20
+ import { NatsServiceFactory } from '@pikku/queue-nats'
21
+
22
+ const factory = new NatsServiceFactory({
23
+ servers: 'nats://localhost:4222',
24
+ streamName: 'pikku',
25
+ subjectPrefix: 'pikku',
26
+ })
27
+ await factory.init()
28
+
29
+ const queue = factory.getQueueService()
30
+ const workers = factory.getQueueWorkers()
31
+
32
+ await workers.registerQueues()
33
+ ```
34
+
35
+ Use `NatsSchedulerService` for cron wirings. For a Postgres-backed alternative
36
+ see `@pikku/queue-pg-boss`, and for a Redis-backed one `@pikku/queue-bullmq`.
37
+
38
+ ## Job results
39
+
40
+ `supportsResults` is `false`. A work-queue stream deletes a message the moment
41
+ it is acked, so there is nowhere for a job result — or any job history — to
42
+ live. Per-job outcomes come from telemetry, not from the broker; producers that
43
+ need a return value should use an RPC rather than a queue.
44
+
45
+ ## Docs
46
+
47
+ https://pikku.dev/docs
@@ -0,0 +1,54 @@
1
+ import type { ConsumerConfig, JetStreamClient, JetStreamManager, JsMsg } from '@nats-io/jetstream';
2
+ import type { Logger } from '@pikku/core/services';
3
+ export interface ConsumeOptions {
4
+ js: JetStreamClient;
5
+ jsm: JetStreamManager;
6
+ streamName: string;
7
+ subjectPrefix: string;
8
+ queueName: string;
9
+ /** Merged over the defaults produced by `mapPikkuWorkerToNats`. */
10
+ config?: Partial<ConsumerConfig>;
11
+ /**
12
+ * Handles one message. It owns the message's terminal call — exactly one of
13
+ * `ack`/`nak`/`term` — because only the caller knows whether a failure is
14
+ * retryable. A handler that returns without acking leaves the message to be
15
+ * redelivered after `ack_wait`.
16
+ */
17
+ handler: (msg: JsMsg) => Promise<void>;
18
+ /**
19
+ * Used to report a pull session that ended and had to be re-established.
20
+ * Optional only so existing callers and tests keep compiling — pass it in
21
+ * production, because a silent re-attach loop is the failure this guards.
22
+ */
23
+ logger?: Pick<Logger, 'error' | 'info'>;
24
+ }
25
+ export interface ConsumeHandle {
26
+ /** Stop pulling. In-flight handlers are left to settle their own acks. */
27
+ stop: () => void;
28
+ /** Handlers currently running — messages delivered but not yet settled. */
29
+ readonly inFlight: number;
30
+ /**
31
+ * Stop pulling, then wait for in-flight handlers to settle their acks.
32
+ *
33
+ * Resolves `true` if everything drained, `false` if the timeout expired
34
+ * first. Timing out is safe rather than lossy — whatever is still unacked is
35
+ * redelivered after `ack_wait`. Draining just avoids paying that redelivery
36
+ * latency on every deploy, and avoids re-running side effects a handler had
37
+ * already performed before it was killed.
38
+ */
39
+ drain: (timeoutMs: number) => Promise<boolean>;
40
+ }
41
+ /**
42
+ * Start a durable pull consumer and dispatch its messages to `handler`.
43
+ *
44
+ * The one thing that must never change here: `handler` is launched, not
45
+ * awaited. Awaiting inside the `for await` would serialise processing no matter
46
+ * what `max_ack_pending` says, so a single slow job would stall every message
47
+ * behind it — the same head-of-line blocking as a pg-boss batch handler, just
48
+ * moved client-side. In-flight work stays bounded anyway, because the server
49
+ * will not deliver more than `max_ack_pending` unacked messages at once.
50
+ *
51
+ * This exists as a shared function precisely so that invariant is written down
52
+ * once and every consumer inherits it, rather than being re-derived per caller.
53
+ */
54
+ export declare const consumeQueue: (options: ConsumeOptions) => Promise<ConsumeHandle>;
@@ -0,0 +1,213 @@
1
+ import { consumerNameForQueue, subjectForQueue } from './utils.js';
2
+ /** Backoff bounds for re-establishing a pull session that ended under us. */
3
+ const REATTACH_MIN_DELAY_MS = 250;
4
+ const REATTACH_MAX_DELAY_MS = 30_000;
5
+ /**
6
+ * How many consecutive "can't reach it" notifications count as dead rather than
7
+ * transient. Missed heartbeats and a not-found consumer both occur harmlessly
8
+ * during a reconnect, so a single one must not tear down a healthy session.
9
+ */
10
+ const REATTACH_NOTIFICATION_THRESHOLD = 3;
11
+ /**
12
+ * Decide whether a notification means this pull session is permanently dead,
13
+ * returning the reason to log if so.
14
+ *
15
+ * This exists because of a trap documented in the client's own types: for a
16
+ * durable pull consumer these conditions do NOT end the message iterator. On
17
+ * `consumer_deleted` the docs say outright that "the client will continue to
18
+ * attempt to pull messages" — only an *ordered* consumer recreates itself. So
19
+ * the session sits there issuing pulls that can never be answered, the
20
+ * `for await` never returns, and no error is ever thrown. Watching `status()`
21
+ * is the only way to find out, which is why the supervisor below cannot rely on
22
+ * the iterator ending on its own.
23
+ */
24
+ const fatalNotificationReason = (notification) => {
25
+ switch (notification.type) {
26
+ case 'consumer_deleted':
27
+ return `consumer deleted server-side (${notification.code} ${notification.description})`;
28
+ case 'stream_not_found':
29
+ return `stream ${notification.name} not found`;
30
+ case 'consumer_not_found':
31
+ return notification.count >= REATTACH_NOTIFICATION_THRESHOLD
32
+ ? `consumer not found on ${notification.count} consecutive pulls`
33
+ : null;
34
+ case 'heartbeats_missed':
35
+ return notification.count >= REATTACH_NOTIFICATION_THRESHOLD
36
+ ? `${notification.count} consecutive heartbeats missed`
37
+ : null;
38
+ default:
39
+ return null;
40
+ }
41
+ };
42
+ /**
43
+ * Start a durable pull consumer and dispatch its messages to `handler`.
44
+ *
45
+ * The one thing that must never change here: `handler` is launched, not
46
+ * awaited. Awaiting inside the `for await` would serialise processing no matter
47
+ * what `max_ack_pending` says, so a single slow job would stall every message
48
+ * behind it — the same head-of-line blocking as a pg-boss batch handler, just
49
+ * moved client-side. In-flight work stays bounded anyway, because the server
50
+ * will not deliver more than `max_ack_pending` unacked messages at once.
51
+ *
52
+ * This exists as a shared function precisely so that invariant is written down
53
+ * once and every consumer inherits it, rather than being re-derived per caller.
54
+ */
55
+ export const consumeQueue = async (options) => {
56
+ const { js, jsm, streamName, subjectPrefix, queueName, config, handler, logger, } = options;
57
+ const durable = consumerNameForQueue(subjectPrefix, queueName);
58
+ // `add` is CONSUMER.CREATE: a no-op when the existing durable matches, but it
59
+ // THROWS "consumer already exists" when any setting differs. So create first,
60
+ // then fall back to update — otherwise the first deploy after changing a
61
+ // queue's batchSize or lockDuration would fail at boot, on the second deploy
62
+ // rather than the one that introduced the change. Same failure shape as
63
+ // STREAM.CREATE vs STREAM.UPDATE in nats-service-factory.
64
+ const consumerConfig = {
65
+ durable_name: durable,
66
+ filter_subject: subjectForQueue(subjectPrefix, queueName),
67
+ ...config,
68
+ };
69
+ const attach = async () => {
70
+ try {
71
+ await jsm.consumers.add(streamName, consumerConfig);
72
+ }
73
+ catch (err) {
74
+ const message = err instanceof Error ? err.message : String(err);
75
+ if (!/already exists|already in use/i.test(message))
76
+ throw err;
77
+ await jsm.consumers.update(streamName, durable, consumerConfig);
78
+ }
79
+ const consumer = await js.consumers.get(streamName, durable);
80
+ return await consumer.consume();
81
+ };
82
+ let stopped = false;
83
+ let inFlight = 0;
84
+ // Bumped on every re-attach so a watcher left over from a previous session
85
+ // can never stop the session that replaced it.
86
+ let generation = 0;
87
+ const describe = (err) => err instanceof Error ? err.message : String(err);
88
+ // Turns a silently-dead session into an ended one, which is the single shape
89
+ // the supervisor below knows how to recover from.
90
+ const watchStatus = (session, sessionGeneration) => {
91
+ void (async () => {
92
+ try {
93
+ for await (const notification of session.status()) {
94
+ if (stopped || generation !== sessionGeneration)
95
+ return;
96
+ const reason = fatalNotificationReason(notification);
97
+ if (reason === null)
98
+ continue;
99
+ logger?.error(`NATS consumer ${durable} on ${streamName}: ${reason} — dropping the pull session so it can re-attach`);
100
+ session.stop();
101
+ return;
102
+ }
103
+ }
104
+ catch {
105
+ // The status stream itself ending or failing is not the failure we are
106
+ // guarding: the message iterator reports its own end, and the supervisor
107
+ // acts on that. Swallowed so this never surfaces as an unhandled
108
+ // rejection during shutdown.
109
+ }
110
+ })();
111
+ };
112
+ // The first attach is awaited and NOT retried: a failure here is a bad stream
113
+ // or consumer config, and boot must fail loudly on it rather than spin.
114
+ let messages = await attach();
115
+ watchStatus(messages, generation);
116
+ // Not awaited: this supervisor runs for the lifetime of the consumer.
117
+ //
118
+ // The inner `for await` is a pull SESSION, not the consumer itself, and the
119
+ // session can die while the consumer lives on. Before this loop existed the
120
+ // iterator simply ran off the end: the IIFE resolved, nothing pulled ever
121
+ // again, and nothing said so. The durable consumer still existed server-side
122
+ // with its backlog growing, every health check stayed green, and the only
123
+ // symptom was work silently not happening. That is exactly how the
124
+ // orchestrator queue reached 1681 undelivered messages while the backend
125
+ // looked healthy — see knowledge/stability.md. So: re-attach, with backoff,
126
+ // and say so at error level every time.
127
+ //
128
+ // Note this loop alone is not enough, which is why `watchStatus` exists: the
129
+ // iterator ending is only one of the two ways a session dies, and it is the
130
+ // less common one. The other is the session staying open forever while the
131
+ // server has nothing to answer it with, and only `status()` reveals that.
132
+ void (async () => {
133
+ let backoffMs = REATTACH_MIN_DELAY_MS;
134
+ for (;;) {
135
+ try {
136
+ for await (const msg of messages) {
137
+ if (stopped)
138
+ break;
139
+ // Deliberately NOT awaited — see the note above. The counter is what
140
+ // lets shutdown wait for these without serialising them; `finally` is
141
+ // chained rather than `catch`ed so a rejecting handler still surfaces
142
+ // as an unhandled rejection exactly as it did before.
143
+ //
144
+ // Started through a resolved promise so that a handler which throws
145
+ // SYNCHRONOUSLY rejects like any other failure instead of unwinding
146
+ // into the catch below, which would diagnose a handler bug as a dead
147
+ // pull session and re-attach for no reason.
148
+ const settled = Promise.resolve().then(() => handler(msg));
149
+ inFlight++;
150
+ void settled.finally(() => {
151
+ inFlight--;
152
+ });
153
+ }
154
+ if (stopped)
155
+ return;
156
+ logger?.error(`NATS consumer ${durable} on ${streamName}: pull session ended unexpectedly — re-attaching`);
157
+ }
158
+ catch (err) {
159
+ if (stopped)
160
+ return;
161
+ logger?.error(`NATS consumer ${durable} on ${streamName}: pull session threw (${describe(err)}) — re-attaching`);
162
+ }
163
+ // Re-attach until it takes. `stop()` is checked on both sides of the wait
164
+ // so a shutdown during backoff exits instead of reviving the consumer.
165
+ for (;;) {
166
+ if (stopped)
167
+ return;
168
+ await new Promise((resolve) => setTimeout(resolve, backoffMs));
169
+ if (stopped)
170
+ return;
171
+ try {
172
+ messages = await attach();
173
+ // `stop()` only stops the session it can see, so one that arrives
174
+ // after shutdown began has to stop itself. Without this the loop
175
+ // below would enter `for await` on a live session whose only exit is
176
+ // a message arriving — on an idle queue, never — leaving a pull
177
+ // consumer on the connection after `drain()` reported it settled.
178
+ if (stopped) {
179
+ messages.stop();
180
+ return;
181
+ }
182
+ generation++;
183
+ watchStatus(messages, generation);
184
+ logger?.info(`NATS consumer ${durable} on ${streamName}: re-attached`);
185
+ backoffMs = REATTACH_MIN_DELAY_MS;
186
+ break;
187
+ }
188
+ catch (err) {
189
+ logger?.error(`NATS consumer ${durable} on ${streamName}: re-attach failed (${describe(err)}), retrying in ${backoffMs}ms`);
190
+ backoffMs = Math.min(backoffMs * 2, REATTACH_MAX_DELAY_MS);
191
+ }
192
+ }
193
+ }
194
+ })();
195
+ const stop = () => {
196
+ stopped = true;
197
+ messages.stop();
198
+ };
199
+ return {
200
+ stop,
201
+ get inFlight() {
202
+ return inFlight;
203
+ },
204
+ drain: async (timeoutMs) => {
205
+ stop();
206
+ const deadline = Date.now() + timeoutMs;
207
+ while (inFlight > 0 && Date.now() < deadline) {
208
+ await new Promise((resolve) => setTimeout(resolve, 25));
209
+ }
210
+ return inFlight === 0;
211
+ },
212
+ };
213
+ };
@@ -0,0 +1,37 @@
1
+ /**
2
+ * @module @pikku/queue-nats
3
+ *
4
+ * NATS JetStream queue service for Pikku.
5
+ *
6
+ * Mirrors `@pikku/queue-pg-boss` / `@pikku/queue-bullmq`, with one deliberate
7
+ * difference:
8
+ *
9
+ * - `supportsResults` is false. A work-queue stream deletes a message on ack,
10
+ * so there is nowhere for a job result — or any job history — to live.
11
+ * Success rates and per-job outcomes come from telemetry, not the broker.
12
+ *
13
+ * This package used to carry a second difference: no scheduler service, because
14
+ * JetStream had no delayed or scheduled message primitive, so cron and delayed
15
+ * work had to stay on a database-backed queue under the rule "Postgres owns
16
+ * time, NATS owns delivery". That stopped being true — server 2.12 added
17
+ * one-shot `@at` delays and 2.14 added repeating cron schedules — so NATS now
18
+ * owns both. Requires a server on 2.14+; `NatsServiceFactory` refuses to start
19
+ * against anything older rather than silently not scheduling.
20
+ *
21
+ * A database has not left the picture entirely, and shouldn't: the broker has
22
+ * no API to list schedules and no record of whether a fire succeeded, so a
23
+ * schedule registry and a last-fired ledger belong in one. NATS owns *when*,
24
+ * the database owns *what is registered* and *what happened*.
25
+ */
26
+ export { NatsServiceFactory } from './nats-service-factory.js';
27
+ export type { NatsServiceFactoryOptions } from './nats-service-factory.js';
28
+ export { NatsQueueService } from './nats-queue-service.js';
29
+ export type { DelayedPublisher } from './nats-queue-service.js';
30
+ export { NatsDelayedPublisher } from './nats-delayed-publisher.js';
31
+ export { NatsSchedulerService, toNatsCron, parseNatsSchedule, } from './nats-scheduler-service.js';
32
+ export { NatsQueueWorkers, mapPikkuWorkerToNats } from './nats-queue-worker.js';
33
+ export { consumeQueue } from './consume.js';
34
+ export type { ConsumeHandle, ConsumeOptions } from './consume.js';
35
+ export { consumerNameForQueue, queueNameToToken, subjectForQueue, backoffDelayMs, attemptsFor, backoffFor, scheduleSubjectFor, PIKKU_USER_ID_HEADER, NATS_MSG_ID_HEADER, ATTEMPTS_HEADER, BACKOFF_TYPE_HEADER, BACKOFF_DELAY_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_NEXT_HEADER, SCHEDULER_HEADER, SCHEDULE_SUBJECT_SEGMENT, } from './utils.js';
36
+ export { headers as natsHeaders } from '@nats-io/nats-core';
37
+ export type { QueueService, PikkuWorkerConfig, PikkuJobConfig, QueueJob, JobOptions, } from '@pikku/core/queue';
package/dist/index.js ADDED
@@ -0,0 +1,42 @@
1
+ /**
2
+ * @module @pikku/queue-nats
3
+ *
4
+ * NATS JetStream queue service for Pikku.
5
+ *
6
+ * Mirrors `@pikku/queue-pg-boss` / `@pikku/queue-bullmq`, with one deliberate
7
+ * difference:
8
+ *
9
+ * - `supportsResults` is false. A work-queue stream deletes a message on ack,
10
+ * so there is nowhere for a job result — or any job history — to live.
11
+ * Success rates and per-job outcomes come from telemetry, not the broker.
12
+ *
13
+ * This package used to carry a second difference: no scheduler service, because
14
+ * JetStream had no delayed or scheduled message primitive, so cron and delayed
15
+ * work had to stay on a database-backed queue under the rule "Postgres owns
16
+ * time, NATS owns delivery". That stopped being true — server 2.12 added
17
+ * one-shot `@at` delays and 2.14 added repeating cron schedules — so NATS now
18
+ * owns both. Requires a server on 2.14+; `NatsServiceFactory` refuses to start
19
+ * against anything older rather than silently not scheduling.
20
+ *
21
+ * A database has not left the picture entirely, and shouldn't: the broker has
22
+ * no API to list schedules and no record of whether a fire succeeded, so a
23
+ * schedule registry and a last-fired ledger belong in one. NATS owns *when*,
24
+ * the database owns *what is registered* and *what happened*.
25
+ */
26
+ export { NatsServiceFactory } from './nats-service-factory.js';
27
+ export { NatsQueueService } from './nats-queue-service.js';
28
+ // Backs workflow.sleep() and backoff retries, using the server's own scheduler.
29
+ // Wired automatically by NatsServiceFactory; exported for tests and for callers
30
+ // building a queue service by hand.
31
+ export { NatsDelayedPublisher } from './nats-delayed-publisher.js';
32
+ export { NatsSchedulerService, toNatsCron, parseNatsSchedule, } from './nats-scheduler-service.js';
33
+ export { NatsQueueWorkers, mapPikkuWorkerToNats } from './nats-queue-worker.js';
34
+ // For consumers that need their own dispatch loop rather than pikku's queue
35
+ // registry — e.g. a dispatcher whose queues are shared across tenants and must
36
+ // not collide with the registry's global queue names.
37
+ export { consumeQueue } from './consume.js';
38
+ export { consumerNameForQueue, queueNameToToken, subjectForQueue, backoffDelayMs, attemptsFor, backoffFor, scheduleSubjectFor, PIKKU_USER_ID_HEADER, NATS_MSG_ID_HEADER, ATTEMPTS_HEADER, BACKOFF_TYPE_HEADER, BACKOFF_DELAY_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_NEXT_HEADER, SCHEDULER_HEADER, SCHEDULE_SUBJECT_SEGMENT, } from './utils.js';
39
+ // Re-exported so callers that publish their own messages (a dispatcher parking
40
+ // a job on a DLQ, say) can build headers without taking a direct dependency on
41
+ // nats-core — this package is the NATS boundary.
42
+ export { headers as natsHeaders } from '@nats-io/nats-core';
@@ -0,0 +1,35 @@
1
+ import type { JetStreamClient } from '@nats-io/jetstream';
2
+ import type { JobOptions } from '@pikku/core/queue';
3
+ import type { DelayedPublisher } from './nats-queue-service.js';
4
+ /**
5
+ * Delayed delivery using JetStream's own message scheduler (server 2.12+).
6
+ *
7
+ * Replaces the pg-boss republisher this package used to need. A message is
8
+ * published to a reserved schedule subject carrying `Nats-Schedule: @at <ts>`
9
+ * and `Nats-Schedule-Target: <the real queue subject>`; the server holds it and
10
+ * publishes the body verbatim onto the target when it comes due. Nothing polls,
11
+ * and no second store owns time.
12
+ *
13
+ * Two constraints shape the subject layout, both enforced by the server at
14
+ * publish time (error 10190, `message schedules target is invalid`):
15
+ *
16
+ * - The target must be a subject captured by the SAME stream as the schedule.
17
+ * Cross-stream scheduling does not exist; it would need a sourcing topology.
18
+ * - The target must not be the schedule's own subject, which would make the
19
+ * schedule reproduce itself forever.
20
+ *
21
+ * So schedules live inside the queue stream, under a subject prefix that no
22
+ * consumer filters on. That matters more than it looks: this is a WorkQueue
23
+ * stream, where a message is deleted the instant a consumer acks it, and a
24
+ * schedule that gets acked is a schedule that silently stops firing. No
25
+ * consumer subscribes to `<prefix>._schedule.>`, so nothing can ack one.
26
+ *
27
+ * One-shot `@at` schedules are purged by the server once they fire, so this
28
+ * leaves nothing behind to clean up.
29
+ */
30
+ export declare class NatsDelayedPublisher implements DelayedPublisher {
31
+ private readonly js;
32
+ private readonly subjectPrefix;
33
+ constructor(js: JetStreamClient, subjectPrefix: string);
34
+ publishAfter<T>(queueName: string, data: T, delayMs: number, options?: JobOptions): Promise<string>;
35
+ }
@@ -0,0 +1,85 @@
1
+ import { headers } from '@nats-io/nats-core';
2
+ import { ATTEMPTS_HEADER, BACKOFF_DELAY_HEADER, BACKOFF_TYPE_HEADER, PIKKU_USER_ID_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, attemptsHeaderValue, scheduleSubjectFor, subjectForQueue, } from './utils.js';
3
+ /**
4
+ * Delayed delivery using JetStream's own message scheduler (server 2.12+).
5
+ *
6
+ * Replaces the pg-boss republisher this package used to need. A message is
7
+ * published to a reserved schedule subject carrying `Nats-Schedule: @at <ts>`
8
+ * and `Nats-Schedule-Target: <the real queue subject>`; the server holds it and
9
+ * publishes the body verbatim onto the target when it comes due. Nothing polls,
10
+ * and no second store owns time.
11
+ *
12
+ * Two constraints shape the subject layout, both enforced by the server at
13
+ * publish time (error 10190, `message schedules target is invalid`):
14
+ *
15
+ * - The target must be a subject captured by the SAME stream as the schedule.
16
+ * Cross-stream scheduling does not exist; it would need a sourcing topology.
17
+ * - The target must not be the schedule's own subject, which would make the
18
+ * schedule reproduce itself forever.
19
+ *
20
+ * So schedules live inside the queue stream, under a subject prefix that no
21
+ * consumer filters on. That matters more than it looks: this is a WorkQueue
22
+ * stream, where a message is deleted the instant a consumer acks it, and a
23
+ * schedule that gets acked is a schedule that silently stops firing. No
24
+ * consumer subscribes to `<prefix>._schedule.>`, so nothing can ack one.
25
+ *
26
+ * One-shot `@at` schedules are purged by the server once they fire, so this
27
+ * leaves nothing behind to clean up.
28
+ */
29
+ export class NatsDelayedPublisher {
30
+ js;
31
+ subjectPrefix;
32
+ constructor(js, subjectPrefix) {
33
+ this.js = js;
34
+ this.subjectPrefix = subjectPrefix;
35
+ }
36
+ async publishAfter(queueName, data, delayMs, options) {
37
+ const hdrs = headers();
38
+ // RFC3339, which is what the server parses. A timestamp already in the past
39
+ // fires immediately rather than erroring — the same shape as pg-boss's
40
+ // `startAfter` with an elapsed date, so a delay that races the clock still
41
+ // delivers instead of being dropped.
42
+ hdrs.set(SCHEDULE_HEADER, `@at ${new Date(Date.now() + delayMs).toISOString()}`);
43
+ hdrs.set(SCHEDULE_TARGET_HEADER, subjectForQueue(this.subjectPrefix, queueName));
44
+ // Every non-schedule header is copied onto the produced message verbatim,
45
+ // so the job's retry policy survives the delay. `jobId` is deliberately not
46
+ // one of them, in either direction:
47
+ //
48
+ // - As `Nats-Msg-Id` it would be pointless — the server strips that header
49
+ // when producing, so it could never reach the stream's dedupe window.
50
+ // - As the schedule's subject it would be actively wrong. Republishing to
51
+ // a schedule subject REPLACES the schedule, so two delayed jobs sharing
52
+ // a jobId would silently cancel each other. The pg-boss publisher this
53
+ // replaces passed only `startAfter` and no singleton key, so delayed
54
+ // publishes have never deduped; making them dedupe now would be a
55
+ // behaviour change disguised as a port.
56
+ //
57
+ // Hence: no key, and every delayed publish gets its own subject.
58
+ const { delay: _delay, jobId: _jobId, ...rest } = options ?? {};
59
+ applyJobHeaders(hdrs, rest, queueName);
60
+ const ack = await this.js.publish(scheduleSubjectFor(this.subjectPrefix, queueName), JSON.stringify(data), { headers: hdrs });
61
+ return String(ack.seq);
62
+ }
63
+ }
64
+ /**
65
+ * Copy the queue-level job options that must survive the delay onto the
66
+ * schedule message. Kept here rather than shared with `NatsQueueService.add`
67
+ * because the two disagree deliberately about `jobId` — see above.
68
+ */
69
+ const applyJobHeaders = (hdrs, options, queueName) => {
70
+ if (options.pikkuUserId) {
71
+ hdrs.set(PIKKU_USER_ID_HEADER, options.pikkuUserId);
72
+ }
73
+ if (options.attempts !== undefined) {
74
+ hdrs.set(ATTEMPTS_HEADER, attemptsHeaderValue(options.attempts, queueName));
75
+ }
76
+ if (options.backoff !== undefined) {
77
+ const backoff = typeof options.backoff === 'string'
78
+ ? { type: options.backoff }
79
+ : options.backoff;
80
+ hdrs.set(BACKOFF_TYPE_HEADER, backoff.type);
81
+ if (backoff.delay !== undefined) {
82
+ hdrs.set(BACKOFF_DELAY_HEADER, String(backoff.delay));
83
+ }
84
+ }
85
+ };
@@ -0,0 +1,44 @@
1
+ import type { JetStreamClient } from '@nats-io/jetstream';
2
+ import type { JobOptions, QueueJob, QueueService } from '@pikku/core/queue';
3
+ /**
4
+ * Delayed delivery. Implemented by `NatsDelayedPublisher` against the server's
5
+ * own message scheduler (2.12+); it used to be implemented against a store that
6
+ * owned time, because JetStream had no such primitive.
7
+ */
8
+ export interface DelayedPublisher {
9
+ publishAfter<T>(queueName: string, data: T, delayMs: number, options?: JobOptions): Promise<string>;
10
+ }
11
+ /**
12
+ * JetStream queue service for publishing jobs.
13
+ *
14
+ * Deliberately narrower than the pg-boss and bullmq services in two ways, both
15
+ * forced by the broker rather than by choice — see `supportsResults` and
16
+ * `getJob` below.
17
+ */
18
+ export declare class NatsQueueService implements QueueService {
19
+ private readonly js;
20
+ private readonly subjectPrefix;
21
+ /**
22
+ * A work-queue stream deletes a message the moment it is acked, so there is
23
+ * nowhere for a result to live and nothing to read it back from. Producers
24
+ * that need a return value should use an RPC, not a queue.
25
+ */
26
+ readonly supportsResults = false;
27
+ private delayedPublisher?;
28
+ constructor(js: JetStreamClient, subjectPrefix: string, delayedPublisher?: DelayedPublisher);
29
+ /**
30
+ * Attach the delayed publisher after construction. Needed because the pg-boss
31
+ * publisher republishes *through* this service, so the two reference each
32
+ * other and one of them has to be wired second.
33
+ */
34
+ setDelayedPublisher(publisher: DelayedPublisher): void;
35
+ add<T>(queueName: string, data: T, options?: JobOptions): Promise<string>;
36
+ /**
37
+ * Not supported. A message in a work-queue stream is only reachable through a
38
+ * consumer, and only while it is being delivered — there is no lookup by id,
39
+ * and an acked message no longer exists. Returning null (rather than throwing)
40
+ * keeps callers that poll for optional status working; callers that genuinely
41
+ * need per-job state should read it from telemetry.
42
+ */
43
+ getJob<T, R>(_queueName: string, _jobId: string): Promise<QueueJob<T, R> | null>;
44
+ }