@pikku/queue-nats 0.12.3 → 0.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/nats-delayed-publisher.js +4 -1
- package/dist/nats-queue-service.js +4 -1
- package/dist/nats-queue-worker.js +5 -3
- package/dist/nats-scheduler-service.d.ts +1 -1
- package/dist/nats-scheduler-service.js +8 -6
- package/dist/utils.d.ts +1 -0
- package/dist/utils.js +2 -0
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -32,6 +32,6 @@ export { NatsSchedulerService, toNatsCron, parseNatsSchedule, } from './nats-sch
|
|
|
32
32
|
export { NatsQueueWorkers, mapPikkuWorkerToNats } from './nats-queue-worker.js';
|
|
33
33
|
export { consumeQueue } from './consume.js';
|
|
34
34
|
export type { ConsumeHandle, ConsumeOptions } from './consume.js';
|
|
35
|
-
export { consumerNameForQueue, queueNameToToken, subjectForQueue, backoffDelayMs, attemptsFor, backoffFor, scheduleSubjectFor, PIKKU_USER_ID_HEADER, NATS_MSG_ID_HEADER, ATTEMPTS_HEADER, BACKOFF_TYPE_HEADER, BACKOFF_DELAY_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_NEXT_HEADER, SCHEDULER_HEADER, SCHEDULE_SUBJECT_SEGMENT, } from './utils.js';
|
|
35
|
+
export { consumerNameForQueue, queueNameToToken, subjectForQueue, backoffDelayMs, attemptsFor, backoffFor, scheduleSubjectFor, PIKKU_USER_ID_HEADER, TRACEPARENT_HEADER, NATS_MSG_ID_HEADER, ATTEMPTS_HEADER, BACKOFF_TYPE_HEADER, BACKOFF_DELAY_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_NEXT_HEADER, SCHEDULER_HEADER, SCHEDULE_SUBJECT_SEGMENT, } from './utils.js';
|
|
36
36
|
export { headers as natsHeaders } from '@nats-io/nats-core';
|
|
37
37
|
export type { QueueService, PikkuWorkerConfig, PikkuJobConfig, QueueJob, JobOptions, } from '@pikku/core/queue';
|
package/dist/index.js
CHANGED
|
@@ -35,7 +35,7 @@ export { NatsQueueWorkers, mapPikkuWorkerToNats } from './nats-queue-worker.js';
|
|
|
35
35
|
// registry — e.g. a dispatcher whose queues are shared across tenants and must
|
|
36
36
|
// not collide with the registry's global queue names.
|
|
37
37
|
export { consumeQueue } from './consume.js';
|
|
38
|
-
export { consumerNameForQueue, queueNameToToken, subjectForQueue, backoffDelayMs, attemptsFor, backoffFor, scheduleSubjectFor, PIKKU_USER_ID_HEADER, NATS_MSG_ID_HEADER, ATTEMPTS_HEADER, BACKOFF_TYPE_HEADER, BACKOFF_DELAY_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_NEXT_HEADER, SCHEDULER_HEADER, SCHEDULE_SUBJECT_SEGMENT, } from './utils.js';
|
|
38
|
+
export { consumerNameForQueue, queueNameToToken, subjectForQueue, backoffDelayMs, attemptsFor, backoffFor, scheduleSubjectFor, PIKKU_USER_ID_HEADER, TRACEPARENT_HEADER, NATS_MSG_ID_HEADER, ATTEMPTS_HEADER, BACKOFF_TYPE_HEADER, BACKOFF_DELAY_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_NEXT_HEADER, SCHEDULER_HEADER, SCHEDULE_SUBJECT_SEGMENT, } from './utils.js';
|
|
39
39
|
// Re-exported so callers that publish their own messages (a dispatcher parking
|
|
40
40
|
// a job on a DLQ, say) can build headers without taking a direct dependency on
|
|
41
41
|
// nats-core — this package is the NATS boundary.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { headers } from '@nats-io/nats-core';
|
|
2
|
-
import { ATTEMPTS_HEADER, BACKOFF_DELAY_HEADER, BACKOFF_TYPE_HEADER, PIKKU_USER_ID_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, attemptsHeaderValue, scheduleSubjectFor, subjectForQueue, } from './utils.js';
|
|
2
|
+
import { ATTEMPTS_HEADER, BACKOFF_DELAY_HEADER, BACKOFF_TYPE_HEADER, PIKKU_USER_ID_HEADER, TRACEPARENT_HEADER, SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, attemptsHeaderValue, scheduleSubjectFor, subjectForQueue, } from './utils.js';
|
|
3
3
|
/**
|
|
4
4
|
* Delayed delivery using JetStream's own message scheduler (server 2.12+).
|
|
5
5
|
*
|
|
@@ -70,6 +70,9 @@ const applyJobHeaders = (hdrs, options, queueName) => {
|
|
|
70
70
|
if (options.pikkuUserId) {
|
|
71
71
|
hdrs.set(PIKKU_USER_ID_HEADER, options.pikkuUserId);
|
|
72
72
|
}
|
|
73
|
+
if (options.traceparent) {
|
|
74
|
+
hdrs.set(TRACEPARENT_HEADER, options.traceparent);
|
|
75
|
+
}
|
|
73
76
|
if (options.attempts !== undefined) {
|
|
74
77
|
hdrs.set(ATTEMPTS_HEADER, attemptsHeaderValue(options.attempts, queueName));
|
|
75
78
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { headers } from '@nats-io/nats-core';
|
|
2
|
-
import { ATTEMPTS_HEADER, BACKOFF_DELAY_HEADER, BACKOFF_TYPE_HEADER, NATS_MSG_ID_HEADER, PIKKU_USER_ID_HEADER, attemptsHeaderValue, subjectForQueue, } from './utils.js';
|
|
2
|
+
import { ATTEMPTS_HEADER, BACKOFF_DELAY_HEADER, BACKOFF_TYPE_HEADER, NATS_MSG_ID_HEADER, PIKKU_USER_ID_HEADER, TRACEPARENT_HEADER, attemptsHeaderValue, subjectForQueue, } from './utils.js';
|
|
3
3
|
/**
|
|
4
4
|
* JetStream queue service for publishing jobs.
|
|
5
5
|
*
|
|
@@ -35,6 +35,9 @@ export class NatsQueueService {
|
|
|
35
35
|
if (options?.pikkuUserId) {
|
|
36
36
|
hdrs.set(PIKKU_USER_ID_HEADER, options.pikkuUserId);
|
|
37
37
|
}
|
|
38
|
+
if (options?.traceparent) {
|
|
39
|
+
hdrs.set(TRACEPARENT_HEADER, options.traceparent);
|
|
40
|
+
}
|
|
38
41
|
// JetStream dedupes on this header within the stream's duplicate window,
|
|
39
42
|
// which is the closest equivalent to pg-boss's singleton `jobId`.
|
|
40
43
|
if (options?.jobId) {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { AckPolicy, DeliverPolicy } from '@nats-io/jetstream';
|
|
2
2
|
import { QueueJobDiscardedError, QueueJobFailedError, registerQueueWorkers, runQueueJob, } from '@pikku/core/queue';
|
|
3
3
|
import { pikkuState } from '@pikku/core/state';
|
|
4
|
-
import {
|
|
4
|
+
import { parseTraceparent, tracedLogger } from '@pikku/core/trace';
|
|
5
|
+
import { attemptsFor, backoffFor, mapJsMsgToQueueJob, TRACEPARENT_HEADER, } from './utils.js';
|
|
5
6
|
import { consumeQueue } from './consume.js';
|
|
6
7
|
const MS_TO_NS = 1_000_000;
|
|
7
8
|
/**
|
|
@@ -154,14 +155,15 @@ export class NatsQueueWorkers {
|
|
|
154
155
|
// to mean "never retry"), while max_deliver is per-consumer — and an unset
|
|
155
156
|
// max_deliver means unlimited, so without this a step marked no-retry
|
|
156
157
|
// would be redelivered forever.
|
|
158
|
+
const failed = tracedLogger(logger, parseTraceparent(msg.headers?.get(TRACEPARENT_HEADER))?.traceId);
|
|
157
159
|
const attempts = attemptsFor(msg);
|
|
158
160
|
if (attempts !== undefined && msg.info.deliveryCount >= attempts) {
|
|
159
|
-
|
|
161
|
+
failed.error(`NATS message ${msg.seq} on ${queueName} failed after ${msg.info.deliveryCount}/${attempts} attempts, giving up: ${message}`);
|
|
160
162
|
msg.term();
|
|
161
163
|
return;
|
|
162
164
|
}
|
|
163
165
|
const delayMs = backoffFor(msg);
|
|
164
|
-
|
|
166
|
+
failed.error(`NATS message ${msg.seq} on ${queueName} failed (attempt ${msg.info.deliveryCount}${attempts ? `/${attempts}` : ''}), retrying in ${delayMs}ms: ${message}`);
|
|
165
167
|
// nak(delay) is what makes backoff real. A bare nak() redelivers as fast
|
|
166
168
|
// as the server can, which turns an exponential-backoff policy into a hot
|
|
167
169
|
// loop against whatever is already failing.
|
|
@@ -86,7 +86,7 @@ export declare class NatsSchedulerService extends SchedulerService {
|
|
|
86
86
|
* *is* the handle. `@at` schedules are purged by the server once they fire,
|
|
87
87
|
* so a fired task simply stops being findable.
|
|
88
88
|
*/
|
|
89
|
-
scheduleRPC(delay: number | string, rpcName: string, data?: any, session?: CoreUserSession): Promise<string>;
|
|
89
|
+
scheduleRPC(delay: number | string, rpcName: string, data?: any, session?: CoreUserSession, traceparent?: string): Promise<string>;
|
|
90
90
|
/**
|
|
91
91
|
* Cancel a scheduled task. Idempotent — purging a subject that holds no
|
|
92
92
|
* message is not an error, so cancelling a task that already fired returns
|
|
@@ -5,11 +5,11 @@ import { parseDurationString } from '@pikku/core/utils';
|
|
|
5
5
|
import { pikkuState } from '@pikku/core/state';
|
|
6
6
|
import { getScheduledTasks, runScheduledTask } from '@pikku/core/scheduler';
|
|
7
7
|
import { consumeQueue } from './consume.js';
|
|
8
|
-
import { SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_SUBJECT_SEGMENT, queueNameToToken, scheduleSubjectFor, subjectForQueue, } from './utils.js';
|
|
8
|
+
import { SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHEDULE_SUBJECT_SEGMENT, TRACEPARENT_HEADER, queueNameToToken, scheduleSubjectFor, subjectForQueue, } from './utils.js';
|
|
9
9
|
/**
|
|
10
10
|
* Queue recurring cron fires land on.
|
|
11
11
|
*
|
|
12
|
-
* Deliberately separate from `pikku-
|
|
12
|
+
* Deliberately separate from `pikku-mesh-rpc` (where one-off
|
|
13
13
|
* `scheduleRPC` calls land): that queue is a registered pikku worker whose
|
|
14
14
|
* consumer is owned by the queue registry, and a recurring task is not an RPC
|
|
15
15
|
* invocation — it goes through `runScheduledTask`, which resolves the task's
|
|
@@ -18,7 +18,7 @@ import { SCHEDULE_HEADER, SCHEDULE_TARGET_HEADER, SCHEDULE_TIMEZONE_HEADER, SCHE
|
|
|
18
18
|
*/
|
|
19
19
|
const RECURRING_TASK_QUEUE = 'pikku-recurring-scheduled-task';
|
|
20
20
|
/** Queue a one-off `scheduleRPC` fires onto — a registered pikku worker. */
|
|
21
|
-
const
|
|
21
|
+
const MESH_RPC_QUEUE = 'pikku-mesh-rpc';
|
|
22
22
|
/** Concurrent in-flight scheduled tasks. */
|
|
23
23
|
const RECURRING_MAX_ACK_PENDING = 5;
|
|
24
24
|
/** Ack window for a recurring fire. Generous: recurring tasks are batch
|
|
@@ -123,14 +123,16 @@ export class NatsSchedulerService extends SchedulerService {
|
|
|
123
123
|
* *is* the handle. `@at` schedules are purged by the server once they fire,
|
|
124
124
|
* so a fired task simply stops being findable.
|
|
125
125
|
*/
|
|
126
|
-
async scheduleRPC(delay, rpcName, data, session) {
|
|
126
|
+
async scheduleRPC(delay, rpcName, data, session, traceparent) {
|
|
127
127
|
const delayMs = typeof delay === 'string' ? parseDurationString(delay) : delay;
|
|
128
|
-
const subject = scheduleSubjectFor(this.subjectPrefix,
|
|
128
|
+
const subject = scheduleSubjectFor(this.subjectPrefix, MESH_RPC_QUEUE);
|
|
129
129
|
const hdrs = headers();
|
|
130
130
|
// A timestamp already in the past fires immediately rather than erroring,
|
|
131
131
|
// which matches pg-boss's `startAfter` with an elapsed date.
|
|
132
132
|
hdrs.set(SCHEDULE_HEADER, `@at ${new Date(Date.now() + delayMs).toISOString()}`);
|
|
133
|
-
hdrs.set(SCHEDULE_TARGET_HEADER, subjectForQueue(this.subjectPrefix,
|
|
133
|
+
hdrs.set(SCHEDULE_TARGET_HEADER, subjectForQueue(this.subjectPrefix, MESH_RPC_QUEUE));
|
|
134
|
+
if (traceparent)
|
|
135
|
+
hdrs.set(TRACEPARENT_HEADER, traceparent);
|
|
134
136
|
await this.js.publish(subject, JSON.stringify({ rpcName, data, session }), { headers: hdrs });
|
|
135
137
|
return subject;
|
|
136
138
|
}
|
package/dist/utils.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { JsMsg } from '@nats-io/jetstream';
|
|
|
2
2
|
import type { QueueJob, QueueJobStatus } from '@pikku/core/queue';
|
|
3
3
|
/** Header carrying the producer's pikku user id through to the worker. */
|
|
4
4
|
export declare const PIKKU_USER_ID_HEADER = "Pikku-User-Id";
|
|
5
|
+
export declare const TRACEPARENT_HEADER = "traceparent";
|
|
5
6
|
/** Header JetStream itself reads for publish-side dedupe within the stream's window. */
|
|
6
7
|
export declare const NATS_MSG_ID_HEADER = "Nats-Msg-Id";
|
|
7
8
|
/**
|
package/dist/utils.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/** Header carrying the producer's pikku user id through to the worker. */
|
|
2
2
|
export const PIKKU_USER_ID_HEADER = 'Pikku-User-Id';
|
|
3
|
+
export const TRACEPARENT_HEADER = 'traceparent';
|
|
3
4
|
/** Header JetStream itself reads for publish-side dedupe within the stream's window. */
|
|
4
5
|
export const NATS_MSG_ID_HEADER = 'Nats-Msg-Id';
|
|
5
6
|
/**
|
|
@@ -151,4 +152,5 @@ export const mapJsMsgToQueueJob = (queueName, msg) => ({
|
|
|
151
152
|
data: msg.json(),
|
|
152
153
|
metadata: () => jsMsgMetadata(msg),
|
|
153
154
|
pikkuUserId: msg.headers?.get(PIKKU_USER_ID_HEADER) || undefined,
|
|
155
|
+
traceparent: msg.headers?.get(TRACEPARENT_HEADER) || undefined,
|
|
154
156
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/queue-nats",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.5",
|
|
4
4
|
"repository": {
|
|
5
5
|
"type": "git",
|
|
6
6
|
"url": "git+https://github.com/pikkujs/pikku.git",
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
"clean": "rm -rf dist tsconfig.tsbuildinfo"
|
|
24
24
|
},
|
|
25
25
|
"peerDependencies": {
|
|
26
|
-
"@pikku/core": "^0.12.
|
|
26
|
+
"@pikku/core": "^0.12.151"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
29
|
"@nats-io/jetstream": "^3.4.0",
|