@zizq-labs/zizq 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -38
- package/dist/batch-config.d.ts +64 -0
- package/dist/batch-config.js +76 -0
- package/dist/client.js +4 -0
- package/dist/enqueue.js +35 -1
- package/dist/handler.js +14 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +2 -0
- package/dist/payload-hasher.d.ts +50 -0
- package/dist/payload-hasher.js +348 -0
- package/dist/resources.d.ts +23 -1
- package/dist/resources.js +16 -0
- package/dist/test-setup.d.ts +1 -0
- package/dist/test-setup.js +30 -0
- package/dist/types.d.ts +50 -1
- package/dist/unique-key.d.ts +5 -13
- package/dist/unique-key.js +18 -75
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@ This is the official Zizq client library for Node.js, written in TypeScript.
|
|
|
12
12
|
## Features
|
|
13
13
|
|
|
14
14
|
* Concurrent async-based worker
|
|
15
|
-
*
|
|
15
|
+
* Simple handler functions, or composable `Router`
|
|
16
16
|
* Enqueue and process jobs from one language to another
|
|
17
17
|
* Arbitrary named queues
|
|
18
18
|
* Granular job priorities
|
|
@@ -21,7 +21,8 @@ This is the official Zizq client library for Node.js, written in TypeScript.
|
|
|
21
21
|
* Configurable job retention policies
|
|
22
22
|
* Recurring jobs (cron)
|
|
23
23
|
* Job introspection and management APIs, with support for `jq` query filters
|
|
24
|
-
* Unique jobs
|
|
24
|
+
* Unique jobs (de-duplicated)
|
|
25
|
+
* Batched jobs (folded/merged)
|
|
25
26
|
|
|
26
27
|
## Installation
|
|
27
28
|
|
|
@@ -106,7 +107,7 @@ await client.enqueueBulk([
|
|
|
106
107
|
|
|
107
108
|
### Defining handlers
|
|
108
109
|
|
|
109
|
-
A handler is
|
|
110
|
+
A handler is any `async` function that accepts a `job` and either resolves
|
|
110
111
|
(the worker acks it as successful) or throws (the worker reports a failure
|
|
111
112
|
and the server retries per the backoff policy). The simplest version is a
|
|
112
113
|
`switch` on `job.type`:
|
|
@@ -124,38 +125,9 @@ async function handler(job) {
|
|
|
124
125
|
}
|
|
125
126
|
```
|
|
126
127
|
|
|
127
|
-
For a more composable style, the client ships `
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
(queue, priority, backoff, retention, uniqueness), so enqueuing later only
|
|
131
|
-
needs the payload:
|
|
132
|
-
|
|
133
|
-
```ts
|
|
134
|
-
import { buildHandler } from "@zizq-labs/zizq";
|
|
135
|
-
|
|
136
|
-
async function sendEmail(payload, job) {
|
|
137
|
-
// ...
|
|
138
|
-
}
|
|
139
|
-
sendEmail.zizqOptions = { queue: "emails", priority: 100 };
|
|
140
|
-
|
|
141
|
-
async function generateReport(payload) {
|
|
142
|
-
// ...
|
|
143
|
-
}
|
|
144
|
-
generateReport.zizqOptions = { queue: "reports" };
|
|
145
|
-
|
|
146
|
-
const handler = buildHandler([sendEmail, generateReport]);
|
|
147
|
-
|
|
148
|
-
// Job functions can be enqueued directly — the queue and other defaults
|
|
149
|
-
// come from the function's zizqOptions.
|
|
150
|
-
await client.enqueue({
|
|
151
|
-
type: sendEmail,
|
|
152
|
-
payload: { userId: 42, template: "welcome" },
|
|
153
|
-
});
|
|
154
|
-
```
|
|
155
|
-
|
|
156
|
-
For cross-language workflows or when you want explicit `type -> handler`
|
|
157
|
-
registration with optional fallback, use `Router`. Routes match by job
|
|
158
|
-
type (a string the producer agrees on with the consumer), and an
|
|
128
|
+
For a more composable style, the client ships a `Router` that implements
|
|
129
|
+
handler registration for named job types with optional fallback. Routes match
|
|
130
|
+
by job type (a string the producer agrees on with the consumer), and an
|
|
159
131
|
optional `fallback` catches anything unmatched:
|
|
160
132
|
|
|
161
133
|
```ts
|
|
@@ -220,8 +192,6 @@ replacing, and removing entries as the definition changes. Cron requires
|
|
|
220
192
|
a Pro license on the server.
|
|
221
193
|
|
|
222
194
|
```ts
|
|
223
|
-
import { sendDailyDigest } from "./handlers";
|
|
224
|
-
|
|
225
195
|
await client.cron("maintenance").register({
|
|
226
196
|
timezone: "Europe/London",
|
|
227
197
|
entries: [
|
|
@@ -235,7 +205,7 @@ await client.cron("maintenance").register({
|
|
|
235
205
|
{
|
|
236
206
|
name: "daily_digest",
|
|
237
207
|
expression: "0 9 * * *",
|
|
238
|
-
type:
|
|
208
|
+
type: "send_daily_digest",
|
|
239
209
|
payload: {},
|
|
240
210
|
},
|
|
241
211
|
],
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { EnqueueInput } from "./types.ts";
|
|
2
|
+
/** Options for the third argument of {@link batchConfig}. */
|
|
3
|
+
export interface BatchConfigOptions {
|
|
4
|
+
/**
|
|
5
|
+
* Append `| unique` to the fold expression to deduplicate entries
|
|
6
|
+
* within the batch. `unique` in jq also sorts, so it subsumes
|
|
7
|
+
* `sorted:` when both are set.
|
|
8
|
+
*/
|
|
9
|
+
dedup?: boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Append `| sort` to the fold expression to sort entries within
|
|
12
|
+
* the batch. Ignored when `dedup:` is also set.
|
|
13
|
+
*/
|
|
14
|
+
sorted?: boolean;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Build a batched-job configuration for a specific target path.
|
|
18
|
+
*
|
|
19
|
+
* Returns the `{key, when, fold}` shape expected by
|
|
20
|
+
* `client.enqueue({batch: ...})`, with `key` as a function that
|
|
21
|
+
* derives a stable batch key from the enqueue input at call time
|
|
22
|
+
* (hashing everything *except* the batch target path).
|
|
23
|
+
*
|
|
24
|
+
* @param limit Maximum combined length of the batched value at
|
|
25
|
+
* `path` before the current batch is sealed and a new one starts.
|
|
26
|
+
* @param path jq path to the batch target within the payload.
|
|
27
|
+
* Defaults to `.` (the whole payload is the batch target, assumed
|
|
28
|
+
* to be an array).
|
|
29
|
+
* @param options `dedup: true` deduplicates via `| unique`;
|
|
30
|
+
* `sorted: true` sorts via `| sort`. `dedup:` subsumes `sorted:`.
|
|
31
|
+
*
|
|
32
|
+
* @example Whole-payload batch (payload is an array)
|
|
33
|
+
* ```ts
|
|
34
|
+
* await client.enqueue({
|
|
35
|
+
* type: "audit.events",
|
|
36
|
+
* queue: "audit",
|
|
37
|
+
* payload: [{ actor: "u1", action: "login" }],
|
|
38
|
+
* batch: batchConfig(1000),
|
|
39
|
+
* });
|
|
40
|
+
* ```
|
|
41
|
+
*
|
|
42
|
+
* @example Batch a specific field
|
|
43
|
+
* ```ts
|
|
44
|
+
* await client.enqueue({
|
|
45
|
+
* type: "push",
|
|
46
|
+
* queue: "push",
|
|
47
|
+
* payload: { deviceIds: [id], platform: "apple" },
|
|
48
|
+
* batch: batchConfig(100, ".deviceIds"),
|
|
49
|
+
* });
|
|
50
|
+
* ```
|
|
51
|
+
*
|
|
52
|
+
* @example Custom key (spread + override)
|
|
53
|
+
* ```ts
|
|
54
|
+
* batch: {
|
|
55
|
+
* ...batchConfig(100, ".deviceIds"),
|
|
56
|
+
* key: (input) => `push:tenant-${input.payload.tenantId}`,
|
|
57
|
+
* }
|
|
58
|
+
* ```
|
|
59
|
+
*/
|
|
60
|
+
export declare function batchConfig(limit: number, path?: string, options?: BatchConfigOptions): {
|
|
61
|
+
key: (input: EnqueueInput) => string;
|
|
62
|
+
when: string;
|
|
63
|
+
fold: string;
|
|
64
|
+
};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
|
|
2
|
+
// Licensed under the MIT License. See LICENSE file for details.
|
|
3
|
+
/**
|
|
4
|
+
* Helper for building batched-job configuration objects. See
|
|
5
|
+
* {@link batchConfig}.
|
|
6
|
+
*
|
|
7
|
+
* @module
|
|
8
|
+
*/
|
|
9
|
+
import { payloadHasher } from "./payload-hasher.js";
|
|
10
|
+
/**
|
|
11
|
+
* Build a batched-job configuration for a specific target path.
|
|
12
|
+
*
|
|
13
|
+
* Returns the `{key, when, fold}` shape expected by
|
|
14
|
+
* `client.enqueue({batch: ...})`, with `key` as a function that
|
|
15
|
+
* derives a stable batch key from the enqueue input at call time
|
|
16
|
+
* (hashing everything *except* the batch target path).
|
|
17
|
+
*
|
|
18
|
+
* @param limit Maximum combined length of the batched value at
|
|
19
|
+
* `path` before the current batch is sealed and a new one starts.
|
|
20
|
+
* @param path jq path to the batch target within the payload.
|
|
21
|
+
* Defaults to `.` (the whole payload is the batch target, assumed
|
|
22
|
+
* to be an array).
|
|
23
|
+
* @param options `dedup: true` deduplicates via `| unique`;
|
|
24
|
+
* `sorted: true` sorts via `| sort`. `dedup:` subsumes `sorted:`.
|
|
25
|
+
*
|
|
26
|
+
* @example Whole-payload batch (payload is an array)
|
|
27
|
+
* ```ts
|
|
28
|
+
* await client.enqueue({
|
|
29
|
+
* type: "audit.events",
|
|
30
|
+
* queue: "audit",
|
|
31
|
+
* payload: [{ actor: "u1", action: "login" }],
|
|
32
|
+
* batch: batchConfig(1000),
|
|
33
|
+
* });
|
|
34
|
+
* ```
|
|
35
|
+
*
|
|
36
|
+
* @example Batch a specific field
|
|
37
|
+
* ```ts
|
|
38
|
+
* await client.enqueue({
|
|
39
|
+
* type: "push",
|
|
40
|
+
* queue: "push",
|
|
41
|
+
* payload: { deviceIds: [id], platform: "apple" },
|
|
42
|
+
* batch: batchConfig(100, ".deviceIds"),
|
|
43
|
+
* });
|
|
44
|
+
* ```
|
|
45
|
+
*
|
|
46
|
+
* @example Custom key (spread + override)
|
|
47
|
+
* ```ts
|
|
48
|
+
* batch: {
|
|
49
|
+
* ...batchConfig(100, ".deviceIds"),
|
|
50
|
+
* key: (input) => `push:tenant-${input.payload.tenantId}`,
|
|
51
|
+
* }
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
export function batchConfig(limit, path = ".", options = {}) {
|
|
55
|
+
// `payloadHasher` validates `path` via its jq-path parser and
|
|
56
|
+
// throws on invalid syntax at construction time — surfacing typos
|
|
57
|
+
// before any enqueue happens.
|
|
58
|
+
const key = payloadHasher({ except: [path] });
|
|
59
|
+
// Pipe-form access (`$var | <path>`) is the uniform shape that
|
|
60
|
+
// works for every path we accept, including the root `.` case
|
|
61
|
+
// where `$existing<path>` would otherwise be a syntax error.
|
|
62
|
+
const when = `(($existing | ${path}) + ($new | ${path})) | length <= ${limit}`;
|
|
63
|
+
let fold;
|
|
64
|
+
if (options.dedup) {
|
|
65
|
+
fold =
|
|
66
|
+
`$existing | ${path} = ((${path}) + ($new | ${path}) | unique)`;
|
|
67
|
+
}
|
|
68
|
+
else if (options.sorted) {
|
|
69
|
+
fold =
|
|
70
|
+
`$existing | ${path} = ((${path}) + ($new | ${path}) | sort)`;
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
fold = `$existing | ${path} += ($new | ${path})`;
|
|
74
|
+
}
|
|
75
|
+
return { key, when, fold };
|
|
76
|
+
}
|
package/dist/client.js
CHANGED
|
@@ -1154,6 +1154,7 @@ function enqueueToApi(opts) {
|
|
|
1154
1154
|
retention: opts.retention && retentionToApi(opts.retention),
|
|
1155
1155
|
unique_key: opts.uniqueKey,
|
|
1156
1156
|
unique_while: opts.uniqueWhile,
|
|
1157
|
+
batch: opts.batch,
|
|
1157
1158
|
});
|
|
1158
1159
|
}
|
|
1159
1160
|
/**
|
|
@@ -1216,6 +1217,8 @@ function jobFromApi(raw) {
|
|
|
1216
1217
|
uniqueKey: r.unique_key,
|
|
1217
1218
|
uniqueWhile: r.unique_while,
|
|
1218
1219
|
duplicate: r.duplicate,
|
|
1220
|
+
folded: r.folded,
|
|
1221
|
+
batch: r.batch,
|
|
1219
1222
|
});
|
|
1220
1223
|
}
|
|
1221
1224
|
/** Convert an API-format backoff to camelCase. */
|
|
@@ -1308,6 +1311,7 @@ function cronJobToApi(job) {
|
|
|
1308
1311
|
retention: job.retention && retentionToApi(job.retention),
|
|
1309
1312
|
unique_key: job.uniqueKey,
|
|
1310
1313
|
unique_while: job.uniqueWhile,
|
|
1314
|
+
batch: job.batch,
|
|
1311
1315
|
});
|
|
1312
1316
|
}
|
|
1313
1317
|
/** Convert a CronEntryInput to the server's API format. */
|
package/dist/enqueue.js
CHANGED
|
@@ -37,6 +37,16 @@ function computeUniqueKey(jobFn, payload) {
|
|
|
37
37
|
return uniqueKey(jobFn, payload);
|
|
38
38
|
return uniqueKey;
|
|
39
39
|
}
|
|
40
|
+
// Resolve a value that may be either a literal or a function derived
|
|
41
|
+
// from the enqueue input. Used for both `uniqueKey` and `batch.key`
|
|
42
|
+
// on `EnqueueInput` so `payloadHasher(...)` and other userland
|
|
43
|
+
// factories can produce a function that receives the input at call
|
|
44
|
+
// time.
|
|
45
|
+
function resolveKey(value, input) {
|
|
46
|
+
if (value === undefined)
|
|
47
|
+
return undefined;
|
|
48
|
+
return typeof value === "function" ? value(input) : value;
|
|
49
|
+
}
|
|
40
50
|
/**
|
|
41
51
|
* Resolve an EnqueueInput into a low-level EnqueueOptions.
|
|
42
52
|
*
|
|
@@ -54,6 +64,7 @@ export function resolveInput(input) {
|
|
|
54
64
|
if (!jobType) {
|
|
55
65
|
throw new Error("Job function must have a name or zizqOptions.type");
|
|
56
66
|
}
|
|
67
|
+
warnJobFunctionDeprecated(input.type, jobType);
|
|
57
68
|
}
|
|
58
69
|
else {
|
|
59
70
|
jobType = input.type;
|
|
@@ -62,11 +73,18 @@ export function resolveInput(input) {
|
|
|
62
73
|
if (!queue) {
|
|
63
74
|
throw new Error(`No queue specified for job type "${jobType}"`);
|
|
64
75
|
}
|
|
65
|
-
const uniqueKey = input.uniqueKey ??
|
|
76
|
+
const uniqueKey = resolveKey(input.uniqueKey, input) ??
|
|
66
77
|
(typeof input.type === "function"
|
|
67
78
|
? computeUniqueKey(input.type, input.payload)
|
|
68
79
|
: undefined);
|
|
69
80
|
const uniqueWhile = input.uniqueWhile ?? defaults?.uniqueWhile;
|
|
81
|
+
const batch = input.batch
|
|
82
|
+
? {
|
|
83
|
+
key: resolveKey(input.batch.key, input),
|
|
84
|
+
when: input.batch.when,
|
|
85
|
+
fold: input.batch.fold,
|
|
86
|
+
}
|
|
87
|
+
: undefined;
|
|
70
88
|
const opts = {
|
|
71
89
|
type: jobType,
|
|
72
90
|
queue,
|
|
@@ -78,6 +96,7 @@ export function resolveInput(input) {
|
|
|
78
96
|
retention: input.retention,
|
|
79
97
|
uniqueKey,
|
|
80
98
|
uniqueWhile: uniqueKey ? uniqueWhile : undefined,
|
|
99
|
+
batch,
|
|
81
100
|
};
|
|
82
101
|
// Apply the transform hook (if any). Transforms can change the options based
|
|
83
102
|
// on the payload or other factors, and can mutate `opts` in place, or return
|
|
@@ -88,3 +107,18 @@ export function resolveInput(input) {
|
|
|
88
107
|
}
|
|
89
108
|
return opts;
|
|
90
109
|
}
|
|
110
|
+
// --- Deprecation warnings ---
|
|
111
|
+
// One warning per unique job function, ever. WeakSet so job functions
|
|
112
|
+
// can still be GC'd even after we've warned about them.
|
|
113
|
+
const warnedFunctions = new WeakSet();
|
|
114
|
+
function warnJobFunctionDeprecated(fn, jobType) {
|
|
115
|
+
if (warnedFunctions.has(fn))
|
|
116
|
+
return;
|
|
117
|
+
warnedFunctions.add(fn);
|
|
118
|
+
process.emitWarning("Job functions are deprecated and will removed in v1.0. " +
|
|
119
|
+
`Migrate ${jobType} to ` +
|
|
120
|
+
`client.enqueue({type: '${jobType}', queue: '...', payload: ...}).`, {
|
|
121
|
+
type: "DeprecationWarning",
|
|
122
|
+
code: "ZIZQ_JOB_FUNCTIONS_DEPRECATED",
|
|
123
|
+
});
|
|
124
|
+
}
|
package/dist/handler.js
CHANGED
|
@@ -25,6 +25,7 @@ import { Router } from "./router.js";
|
|
|
25
25
|
* functions resolve to the same type.
|
|
26
26
|
*/
|
|
27
27
|
export function buildHandler(jobs) {
|
|
28
|
+
warnBuildHandlerDeprecated();
|
|
28
29
|
const seen = new Set();
|
|
29
30
|
const router = new Router();
|
|
30
31
|
for (const fn of jobs) {
|
|
@@ -40,3 +41,16 @@ export function buildHandler(jobs) {
|
|
|
40
41
|
}
|
|
41
42
|
return router.build();
|
|
42
43
|
}
|
|
44
|
+
// --- Deprecation warnings ---
|
|
45
|
+
let warnedBuildHandler = false;
|
|
46
|
+
function warnBuildHandlerDeprecated() {
|
|
47
|
+
if (warnedBuildHandler)
|
|
48
|
+
return;
|
|
49
|
+
warnedBuildHandler = true;
|
|
50
|
+
process.emitWarning("buildHandler and job functions are deprecated and will be removed in " +
|
|
51
|
+
"v1.0. Use Router directly (client.enqueue({type: '...', ...}) + a " +
|
|
52
|
+
"Router-based worker) instead.", {
|
|
53
|
+
type: "DeprecationWarning",
|
|
54
|
+
code: "ZIZQ_JOB_FUNCTIONS_DEPRECATED",
|
|
55
|
+
});
|
|
56
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -14,3 +14,7 @@ export { enqueue, enqueueBulk } from "./enqueue.ts";
|
|
|
14
14
|
export { CronHandle, CronEntryHandle } from "./cron.ts";
|
|
15
15
|
export type { CronEntryDefinition, RegisterCronOptions } from "./cron.ts";
|
|
16
16
|
export { uniqueKey } from "./unique-key.ts";
|
|
17
|
+
export { payloadHasher } from "./payload-hasher.ts";
|
|
18
|
+
export type { PayloadHasherOptions } from "./payload-hasher.ts";
|
|
19
|
+
export { batchConfig } from "./batch-config.ts";
|
|
20
|
+
export type { BatchConfigOptions } from "./batch-config.ts";
|
package/dist/index.js
CHANGED
|
@@ -8,3 +8,5 @@ export { Lazy, ErrorQuery, JobQuery } from "./query.js";
|
|
|
8
8
|
export { enqueue, enqueueBulk } from "./enqueue.js";
|
|
9
9
|
export { CronHandle, CronEntryHandle } from "./cron.js";
|
|
10
10
|
export { uniqueKey } from "./unique-key.js";
|
|
11
|
+
export { payloadHasher } from "./payload-hasher.js";
|
|
12
|
+
export { batchConfig } from "./batch-config.js";
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { EnqueueInput } from "./types.ts";
|
|
2
|
+
/** Options for {@link payloadHasher}. */
|
|
3
|
+
export interface PayloadHasherOptions {
|
|
4
|
+
/**
|
|
5
|
+
* jq paths whose values participate in the hash. When multiple paths
|
|
6
|
+
* are supplied their sub-values are hashed together, keyed by path
|
|
7
|
+
* so ordering doesn't matter. Defaults to `['.']` (whole payload).
|
|
8
|
+
*
|
|
9
|
+
* Accepts a single path or an array. Cannot be combined with `except`.
|
|
10
|
+
*/
|
|
11
|
+
only?: string | string[];
|
|
12
|
+
/**
|
|
13
|
+
* jq paths whose values are excluded from the hash. The payload is
|
|
14
|
+
* deep-cloned and each path is deleted before hashing. Missing paths
|
|
15
|
+
* are silently ignored.
|
|
16
|
+
*
|
|
17
|
+
* Accepts a single path or an array. Cannot be combined with `only`.
|
|
18
|
+
*/
|
|
19
|
+
except?: string | string[];
|
|
20
|
+
/**
|
|
21
|
+
* When `true` (default), the returned key is prefixed with the job
|
|
22
|
+
* type and a `:`. When `false`, only the raw hex digest is returned.
|
|
23
|
+
*/
|
|
24
|
+
prefix?: boolean;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Build a function that hashes some or all of an enqueue's payload
|
|
28
|
+
* into a stable key string.
|
|
29
|
+
*
|
|
30
|
+
* The returned function takes the whole `EnqueueInput` so it can read
|
|
31
|
+
* both the payload (for hashing) and the type (for the optional
|
|
32
|
+
* prefix). Suitable for assignment to `uniqueKey` or `batch.key` on
|
|
33
|
+
* `client.enqueue({...})` inputs.
|
|
34
|
+
*
|
|
35
|
+
* @example Full-payload hash with type prefix
|
|
36
|
+
* ```ts
|
|
37
|
+
* uniqueKey: payloadHasher()
|
|
38
|
+
* // => "sendEmail:<sha256>"
|
|
39
|
+
* ```
|
|
40
|
+
*
|
|
41
|
+
* @example Batch by tenant only
|
|
42
|
+
* ```ts
|
|
43
|
+
* batch: {
|
|
44
|
+
* key: payloadHasher({ except: '.notifications' }),
|
|
45
|
+
* when: "...",
|
|
46
|
+
* fold: "...",
|
|
47
|
+
* }
|
|
48
|
+
* ```
|
|
49
|
+
*/
|
|
50
|
+
export declare function payloadHasher(options?: PayloadHasherOptions): (input: EnqueueInput) => string;
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
// Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
|
|
2
|
+
// Licensed under the MIT License. See LICENSE file for details.
|
|
3
|
+
/**
|
|
4
|
+
* Builds a function that derives a stable key from an enqueue input's
|
|
5
|
+
* payload. Used for both batched-job keys (via `batchConfig`) and as
|
|
6
|
+
* a first-class option for `uniqueKey`.
|
|
7
|
+
*
|
|
8
|
+
* @module
|
|
9
|
+
*/
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
/**
|
|
12
|
+
* Build a function that hashes some or all of an enqueue's payload
|
|
13
|
+
* into a stable key string.
|
|
14
|
+
*
|
|
15
|
+
* The returned function takes the whole `EnqueueInput` so it can read
|
|
16
|
+
* both the payload (for hashing) and the type (for the optional
|
|
17
|
+
* prefix). Suitable for assignment to `uniqueKey` or `batch.key` on
|
|
18
|
+
* `client.enqueue({...})` inputs.
|
|
19
|
+
*
|
|
20
|
+
* @example Full-payload hash with type prefix
|
|
21
|
+
* ```ts
|
|
22
|
+
* uniqueKey: payloadHasher()
|
|
23
|
+
* // => "sendEmail:<sha256>"
|
|
24
|
+
* ```
|
|
25
|
+
*
|
|
26
|
+
* @example Batch by tenant only
|
|
27
|
+
* ```ts
|
|
28
|
+
* batch: {
|
|
29
|
+
* key: payloadHasher({ except: '.notifications' }),
|
|
30
|
+
* when: "...",
|
|
31
|
+
* fold: "...",
|
|
32
|
+
* }
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
export function payloadHasher(options = {}) {
|
|
36
|
+
const prefix = options.prefix ?? true;
|
|
37
|
+
const only = options.only !== undefined ? asArray(options.only) : undefined;
|
|
38
|
+
const except = options.except !== undefined ? asArray(options.except) : undefined;
|
|
39
|
+
if (only && except) {
|
|
40
|
+
throw new Error("payloadHasher: `only` and `except` cannot be combined");
|
|
41
|
+
}
|
|
42
|
+
// Parse paths once at construction so the returned function has no
|
|
43
|
+
// parse overhead per call.
|
|
44
|
+
const onlyPaths = only?.map(parsePath);
|
|
45
|
+
const exceptPaths = except?.map(parsePath);
|
|
46
|
+
return (input) => {
|
|
47
|
+
const value = deriveHashable(input.payload, onlyPaths, exceptPaths);
|
|
48
|
+
const hash = createHash("sha256");
|
|
49
|
+
hashInto(hash, value);
|
|
50
|
+
const digest = hash.digest("hex");
|
|
51
|
+
if (!prefix)
|
|
52
|
+
return digest;
|
|
53
|
+
return `${typeName(input)}:${digest}`;
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
// --- Internal helpers ---
|
|
57
|
+
function asArray(v) {
|
|
58
|
+
return Array.isArray(v) ? v : [v];
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Return the value that should participate in the hash, given the
|
|
62
|
+
* caller's `only`/`except` config.
|
|
63
|
+
*/
|
|
64
|
+
function deriveHashable(payload, onlyPaths, exceptPaths) {
|
|
65
|
+
// Normalise via JSON round-trip so exotic values (Dates, undefined,
|
|
66
|
+
// functions, etc.) become the plain JSON representation the server
|
|
67
|
+
// would see. Also detects circular references early.
|
|
68
|
+
const normalised = JSON.parse(JSON.stringify(payload));
|
|
69
|
+
if (onlyPaths) {
|
|
70
|
+
return pickPaths(normalised, onlyPaths);
|
|
71
|
+
}
|
|
72
|
+
if (exceptPaths) {
|
|
73
|
+
let out = normalised;
|
|
74
|
+
for (const steps of exceptPaths) {
|
|
75
|
+
out = deletePath(out, steps);
|
|
76
|
+
}
|
|
77
|
+
return out;
|
|
78
|
+
}
|
|
79
|
+
return normalised;
|
|
80
|
+
}
|
|
81
|
+
/** Sentinel returned by `walk` to distinguish "missing" from "value is null". */
|
|
82
|
+
const MISSING = Symbol("missing");
|
|
83
|
+
/**
|
|
84
|
+
* Reconstruct a subset of the payload containing only the requested
|
|
85
|
+
* paths, preserving the original nesting structure. Missing paths are
|
|
86
|
+
* silently skipped so hashes remain stable across payloads that omit
|
|
87
|
+
* optional fields.
|
|
88
|
+
*
|
|
89
|
+
* `only: ['.a', '.b']` on `{a: 1, b: 2, c: 3}` returns `{a: 1, b: 2}`.
|
|
90
|
+
* `only: ['.user.id']` on `{user: {id: 42, name: 'x'}}` returns
|
|
91
|
+
* `{user: {id: 42}}`.
|
|
92
|
+
*/
|
|
93
|
+
function pickPaths(source, paths) {
|
|
94
|
+
let target = undefined;
|
|
95
|
+
for (const steps of paths) {
|
|
96
|
+
if (steps.length === 0)
|
|
97
|
+
return source; // '.' short-circuits to full payload.
|
|
98
|
+
const value = walk(source, steps);
|
|
99
|
+
if (value === MISSING)
|
|
100
|
+
continue;
|
|
101
|
+
target = setPath(target, steps, value);
|
|
102
|
+
}
|
|
103
|
+
// Never-matched: fall back to an empty object so the hash is total
|
|
104
|
+
// and matches what an empty pick would produce.
|
|
105
|
+
return target ?? {};
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Write `value` into `target` at `steps`, creating intermediate
|
|
109
|
+
* objects/arrays as needed. Returns the (possibly newly-created)
|
|
110
|
+
* root. The choice between object/array at each level is driven by
|
|
111
|
+
* whether the *next* step is a key or an index.
|
|
112
|
+
*/
|
|
113
|
+
function setPath(target, steps, value) {
|
|
114
|
+
const first = steps[0];
|
|
115
|
+
if (target === undefined) {
|
|
116
|
+
target = first.kind === "key" ? {} : [];
|
|
117
|
+
}
|
|
118
|
+
let cur = target;
|
|
119
|
+
for (let i = 0; i < steps.length - 1; i++) {
|
|
120
|
+
const step = steps[i];
|
|
121
|
+
const next = steps[i + 1];
|
|
122
|
+
const childInit = next.kind === "key" ? {} : [];
|
|
123
|
+
if (step.kind === "key") {
|
|
124
|
+
if (cur[step.name] === undefined)
|
|
125
|
+
cur[step.name] = childInit;
|
|
126
|
+
cur = cur[step.name];
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
if (cur[step.index] === undefined)
|
|
130
|
+
cur[step.index] = childInit;
|
|
131
|
+
cur = cur[step.index];
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
const last = steps[steps.length - 1];
|
|
135
|
+
if (last.kind === "key")
|
|
136
|
+
cur[last.name] = value;
|
|
137
|
+
else
|
|
138
|
+
cur[last.index] = value;
|
|
139
|
+
return target;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Stream a JSON-compatible value into a crypto hash as canonical JSON:
|
|
143
|
+
* object keys sorted, arrays in order, primitives emitted via `JSON.stringify`.
|
|
144
|
+
*
|
|
145
|
+
* The resulting byte stream is unambiguous because strings are quoted,
|
|
146
|
+
* `null`/`true`/`false` are fixed tokens, and commas separate items within
|
|
147
|
+
* containers (so `[1,2]` and `[12]` hash differently).
|
|
148
|
+
*
|
|
149
|
+
* The input must already be normalised JSON data, as from
|
|
150
|
+
* `JSON.parse(JSON.stringify(x))`.
|
|
151
|
+
*/
|
|
152
|
+
function hashInto(hash, value) {
|
|
153
|
+
// Bare number, string, boolean or null. Use JSON repr.
|
|
154
|
+
if (value === null || typeof value !== "object") {
|
|
155
|
+
hash.update(JSON.stringify(value));
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
// Arrays hash in the original order, with "[" and "]" markers.
|
|
159
|
+
// We don't worry about the trailing comma; we just need a stable digest.
|
|
160
|
+
if (Array.isArray(value)) {
|
|
161
|
+
hash.update("[");
|
|
162
|
+
for (const item of value) {
|
|
163
|
+
hashInto(hash, item);
|
|
164
|
+
hash.update(",");
|
|
165
|
+
}
|
|
166
|
+
hash.update("]");
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
// Objects hash in key-sorted order, with "{" and "}" markers.
|
|
170
|
+
// We don't worry about the trailing comma; we just need a stable digest.
|
|
171
|
+
hash.update("{");
|
|
172
|
+
const obj = value;
|
|
173
|
+
const keys = Object.keys(obj).sort();
|
|
174
|
+
for (const key of keys) {
|
|
175
|
+
hash.update(JSON.stringify(key));
|
|
176
|
+
hash.update(":");
|
|
177
|
+
hashInto(hash, obj[key]);
|
|
178
|
+
hash.update(",");
|
|
179
|
+
}
|
|
180
|
+
hash.update("}");
|
|
181
|
+
}
|
|
182
|
+
function typeName(input) {
|
|
183
|
+
if (typeof input.type === "function") {
|
|
184
|
+
const fn = input.type;
|
|
185
|
+
const name = fn.zizqOptions?.type ?? fn.name;
|
|
186
|
+
if (!name) {
|
|
187
|
+
throw new Error("payloadHasher: job function must have a name or zizqOptions.type");
|
|
188
|
+
}
|
|
189
|
+
return name;
|
|
190
|
+
}
|
|
191
|
+
return input.type;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Parse a jq-compatible dotted path into an ordered list of steps.
|
|
195
|
+
*
|
|
196
|
+
* Accepted forms:
|
|
197
|
+
* - `.` — root (no steps)
|
|
198
|
+
* - `.foo` — object key
|
|
199
|
+
* - `.foo.bar` — nested keys
|
|
200
|
+
* - `.foo[0]` — nth array element
|
|
201
|
+
* - `.[0]` — root array index
|
|
202
|
+
* - `.["dotted.key"]` — quoted key (escape hatch for keys with dots)
|
|
203
|
+
*/
|
|
204
|
+
function parsePath(path) {
|
|
205
|
+
if (path === ".")
|
|
206
|
+
return [];
|
|
207
|
+
if (path === "" || path[0] !== ".") {
|
|
208
|
+
throw new Error(`payloadHasher: path must start with '.', got "${path}"`);
|
|
209
|
+
}
|
|
210
|
+
const steps = [];
|
|
211
|
+
let i = 1;
|
|
212
|
+
const n = path.length;
|
|
213
|
+
while (i < n) {
|
|
214
|
+
if (path[i] === "[") {
|
|
215
|
+
// `[N]` or `["quoted key"]`
|
|
216
|
+
i += 1;
|
|
217
|
+
if (path[i] === '"') {
|
|
218
|
+
// Quoted key. Consume until the matching unescaped ".
|
|
219
|
+
i += 1;
|
|
220
|
+
let name = "";
|
|
221
|
+
while (i < n && path[i] !== '"') {
|
|
222
|
+
if (path[i] === "\\" && i + 1 < n) {
|
|
223
|
+
name += path[i + 1];
|
|
224
|
+
i += 2;
|
|
225
|
+
}
|
|
226
|
+
else {
|
|
227
|
+
name += path[i];
|
|
228
|
+
i += 1;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (path[i] !== '"') {
|
|
232
|
+
throw new Error(`payloadHasher: unterminated quoted key in "${path}"`);
|
|
233
|
+
}
|
|
234
|
+
i += 1; // consume "
|
|
235
|
+
if (path[i] !== "]") {
|
|
236
|
+
throw new Error(`payloadHasher: expected ']' in "${path}"`);
|
|
237
|
+
}
|
|
238
|
+
i += 1;
|
|
239
|
+
steps.push({ kind: "key", name });
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
// Numeric index.
|
|
243
|
+
let digits = "";
|
|
244
|
+
while (i < n && path[i] >= "0" && path[i] <= "9") {
|
|
245
|
+
digits += path[i];
|
|
246
|
+
i += 1;
|
|
247
|
+
}
|
|
248
|
+
if (digits.length === 0 || path[i] !== "]") {
|
|
249
|
+
throw new Error(`payloadHasher: invalid array index in "${path}"`);
|
|
250
|
+
}
|
|
251
|
+
i += 1;
|
|
252
|
+
steps.push({ kind: "index", index: parseInt(digits, 10) });
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
else if (path[i] === ".") {
|
|
256
|
+
i += 1;
|
|
257
|
+
// Follow-on dot before a name or bracket.
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
else if (isNameChar(path[i], true)) {
|
|
261
|
+
let name = "";
|
|
262
|
+
while (i < n && isNameChar(path[i], false)) {
|
|
263
|
+
name += path[i];
|
|
264
|
+
i += 1;
|
|
265
|
+
}
|
|
266
|
+
steps.push({ kind: "key", name });
|
|
267
|
+
}
|
|
268
|
+
else {
|
|
269
|
+
throw new Error(`payloadHasher: unexpected character '${path[i]}' in "${path}"`);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return steps;
|
|
273
|
+
}
|
|
274
|
+
function isNameChar(c, first) {
|
|
275
|
+
if ((c >= "A" && c <= "Z") || (c >= "a" && c <= "z") || c === "_")
|
|
276
|
+
return true;
|
|
277
|
+
if (!first && c >= "0" && c <= "9")
|
|
278
|
+
return true;
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Follow `steps` and return the sub-value. Returns the `MISSING`
|
|
283
|
+
* sentinel when any hop is absent (as opposed to a legitimate `null`
|
|
284
|
+
* at the terminal step). Callers use the sentinel to distinguish
|
|
285
|
+
* "silently skip this path" from "hash a null here".
|
|
286
|
+
*/
|
|
287
|
+
function walk(value, steps) {
|
|
288
|
+
let cur = value;
|
|
289
|
+
for (const step of steps) {
|
|
290
|
+
if (cur == null)
|
|
291
|
+
return MISSING;
|
|
292
|
+
if (step.kind === "key") {
|
|
293
|
+
if (typeof cur !== "object" || Array.isArray(cur))
|
|
294
|
+
return MISSING;
|
|
295
|
+
const obj = cur;
|
|
296
|
+
if (!(step.name in obj))
|
|
297
|
+
return MISSING;
|
|
298
|
+
cur = obj[step.name];
|
|
299
|
+
}
|
|
300
|
+
else {
|
|
301
|
+
if (!Array.isArray(cur) || step.index >= cur.length)
|
|
302
|
+
return MISSING;
|
|
303
|
+
cur = cur[step.index];
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return cur;
|
|
307
|
+
}
|
|
308
|
+
/** Return a copy of `value` with the value at `steps` removed. */
|
|
309
|
+
function deletePath(value, steps) {
|
|
310
|
+
if (steps.length === 0) {
|
|
311
|
+
// `except: ['.']` — the entire payload is excluded. Nothing left
|
|
312
|
+
// to hash; return null so we still produce a stable digest
|
|
313
|
+
// (every call collapses to the same value).
|
|
314
|
+
return null;
|
|
315
|
+
}
|
|
316
|
+
// Walk to the parent of the final step, cloning as we go.
|
|
317
|
+
const cloned = structuredClone(value);
|
|
318
|
+
let cur = cloned;
|
|
319
|
+
for (let i = 0; i < steps.length - 1; i++) {
|
|
320
|
+
const step = steps[i];
|
|
321
|
+
if (cur == null)
|
|
322
|
+
return cloned; // silent no-op
|
|
323
|
+
if (step.kind === "key") {
|
|
324
|
+
if (typeof cur !== "object" || Array.isArray(cur))
|
|
325
|
+
return cloned;
|
|
326
|
+
cur = cur[step.name];
|
|
327
|
+
}
|
|
328
|
+
else {
|
|
329
|
+
if (!Array.isArray(cur))
|
|
330
|
+
return cloned;
|
|
331
|
+
cur = cur[step.index];
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (cur == null || typeof cur !== "object")
|
|
335
|
+
return cloned;
|
|
336
|
+
const last = steps[steps.length - 1];
|
|
337
|
+
if (last.kind === "key") {
|
|
338
|
+
if (Array.isArray(cur))
|
|
339
|
+
return cloned;
|
|
340
|
+
delete cur[last.name];
|
|
341
|
+
}
|
|
342
|
+
else {
|
|
343
|
+
if (Array.isArray(cur)) {
|
|
344
|
+
cur.splice(last.index, 1);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return cloned;
|
|
348
|
+
}
|
package/dist/resources.d.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* @module
|
|
8
8
|
*/
|
|
9
9
|
import type { Client } from "./client.ts";
|
|
10
|
-
import type { JobStatus, UniqueScope, BackoffConfig, RetentionConfig, EnqueueOptions, FailureOptions, UpdateJobOptions } from "./types.ts";
|
|
10
|
+
import type { JobStatus, UniqueScope, BackoffConfig, BatchConfig, RetentionConfig, EnqueueOptions, FailureOptions, UpdateJobOptions } from "./types.ts";
|
|
11
11
|
import { ErrorQuery, type ErrorQueryOptions } from "./query.ts";
|
|
12
12
|
/** Raw job data shape (camelCase, as returned by the API translation layer). */
|
|
13
13
|
export interface JobData {
|
|
@@ -51,6 +51,16 @@ export interface JobData {
|
|
|
51
51
|
uniqueWhile?: UniqueScope;
|
|
52
52
|
/** True if this job was returned as a duplicate (enqueue responses only). */
|
|
53
53
|
duplicate?: boolean;
|
|
54
|
+
/**
|
|
55
|
+
* True if this enqueue was folded into an existing pending batched
|
|
56
|
+
* job (enqueue responses only).
|
|
57
|
+
*/
|
|
58
|
+
folded?: boolean;
|
|
59
|
+
/**
|
|
60
|
+
* Batching configuration stored on this job. Present only for
|
|
61
|
+
* batched jobs.
|
|
62
|
+
*/
|
|
63
|
+
batch?: BatchConfig;
|
|
54
64
|
}
|
|
55
65
|
/**
|
|
56
66
|
* A job returned by the Zizq server.
|
|
@@ -108,6 +118,18 @@ export declare class Job {
|
|
|
108
118
|
readonly uniqueWhile?: UniqueScope;
|
|
109
119
|
/** True if this job was returned as a duplicate (enqueue responses only). */
|
|
110
120
|
readonly duplicate?: boolean;
|
|
121
|
+
/**
|
|
122
|
+
* True if this enqueue was folded into an existing pending batched
|
|
123
|
+
* job (enqueue responses only).
|
|
124
|
+
*/
|
|
125
|
+
readonly folded?: boolean;
|
|
126
|
+
/**
|
|
127
|
+
* Batching configuration stored on this job. Present only for
|
|
128
|
+
* batched jobs. The first enqueue's `when`/`fold` are the ones
|
|
129
|
+
* that apply for the life of the batch, so this is the source of
|
|
130
|
+
* truth for what's being evaluated on subsequent folds.
|
|
131
|
+
*/
|
|
132
|
+
readonly batch?: BatchConfig;
|
|
111
133
|
/** @internal */
|
|
112
134
|
private client;
|
|
113
135
|
/** @internal */
|
package/dist/resources.js
CHANGED
|
@@ -57,6 +57,18 @@ export class Job {
|
|
|
57
57
|
uniqueWhile;
|
|
58
58
|
/** True if this job was returned as a duplicate (enqueue responses only). */
|
|
59
59
|
duplicate;
|
|
60
|
+
/**
|
|
61
|
+
* True if this enqueue was folded into an existing pending batched
|
|
62
|
+
* job (enqueue responses only).
|
|
63
|
+
*/
|
|
64
|
+
folded;
|
|
65
|
+
/**
|
|
66
|
+
* Batching configuration stored on this job. Present only for
|
|
67
|
+
* batched jobs. The first enqueue's `when`/`fold` are the ones
|
|
68
|
+
* that apply for the life of the batch, so this is the source of
|
|
69
|
+
* truth for what's being evaluated on subsequent folds.
|
|
70
|
+
*/
|
|
71
|
+
batch;
|
|
60
72
|
/** @internal */
|
|
61
73
|
client;
|
|
62
74
|
/** @internal */
|
|
@@ -80,6 +92,8 @@ export class Job {
|
|
|
80
92
|
this.uniqueKey = data.uniqueKey;
|
|
81
93
|
this.uniqueWhile = data.uniqueWhile;
|
|
82
94
|
this.duplicate = data.duplicate;
|
|
95
|
+
this.folded = data.folded;
|
|
96
|
+
this.batch = data.batch;
|
|
83
97
|
}
|
|
84
98
|
/**
|
|
85
99
|
* Return a lazy async iterator over error records for this job.
|
|
@@ -147,6 +161,8 @@ export class Job {
|
|
|
147
161
|
uniqueKey: this.uniqueKey,
|
|
148
162
|
uniqueWhile: this.uniqueWhile,
|
|
149
163
|
duplicate: this.duplicate,
|
|
164
|
+
folded: this.folded,
|
|
165
|
+
batch: this.batch,
|
|
150
166
|
};
|
|
151
167
|
}
|
|
152
168
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
|
|
2
|
+
// Licensed under the MIT License. See LICENSE file for details.
|
|
3
|
+
/**
|
|
4
|
+
* Loaded before every test via `node --import`.
|
|
5
|
+
*
|
|
6
|
+
* Suppresses the `ZIZQ_JOB_FUNCTIONS_DEPRECATED` warning during test
|
|
7
|
+
* runs — the tests that exercise the deprecated code paths still need
|
|
8
|
+
* to run, and we don't want their output cluttered with the same
|
|
9
|
+
* migration hint they're validating. Any other warning (Node's own
|
|
10
|
+
* experimental notices, real deprecations from dependencies) flows
|
|
11
|
+
* through unchanged.
|
|
12
|
+
*
|
|
13
|
+
* @module
|
|
14
|
+
*/
|
|
15
|
+
const SUPPRESSED_CODES = new Set(["ZIZQ_JOB_FUNCTIONS_DEPRECATED"]);
|
|
16
|
+
const originalListeners = process.listeners("warning");
|
|
17
|
+
process.removeAllListeners("warning");
|
|
18
|
+
process.on("warning", (warning) => {
|
|
19
|
+
if (warning instanceof Error &&
|
|
20
|
+
"code" in warning &&
|
|
21
|
+
typeof warning.code === "string" &&
|
|
22
|
+
SUPPRESSED_CODES.has(warning.code)) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
// Re-dispatch to whatever the default handler would have done.
|
|
26
|
+
for (const listener of originalListeners) {
|
|
27
|
+
listener(warning);
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
export {};
|
package/dist/types.d.ts
CHANGED
|
@@ -13,6 +13,29 @@ export type JobStatus = "ready" | "in_flight" | "scheduled" | "completed" | "dea
|
|
|
13
13
|
export type SortDirection = "asc" | "desc";
|
|
14
14
|
/** Uniqueness scope for deduplication. */
|
|
15
15
|
export type UniqueScope = "queued" | "active" | "exists";
|
|
16
|
+
/**
|
|
17
|
+
* Batching configuration for folded jobs.
|
|
18
|
+
*
|
|
19
|
+
* When present on an enqueue, subsequent enqueues that share `key` may
|
|
20
|
+
* be folded into the same pending job's payload. `when` (jq predicate)
|
|
21
|
+
* decides whether to fold; `fold` (jq reducer) produces the merged
|
|
22
|
+
* payload. Both run with `$existing` bound to the current pending
|
|
23
|
+
* payload and `$new` bound to the incoming one.
|
|
24
|
+
*
|
|
25
|
+
* Requires a pro license on the server.
|
|
26
|
+
*/
|
|
27
|
+
export interface BatchConfig {
|
|
28
|
+
/** Identifies the batch. Only one pending batch per key at a time. */
|
|
29
|
+
key: string;
|
|
30
|
+
/**
|
|
31
|
+
* jq predicate deciding whether to fold. Truthy → merge into the
|
|
32
|
+
* existing batch. Falsy → seal the existing batch and start a fresh
|
|
33
|
+
* one from this enqueue.
|
|
34
|
+
*/
|
|
35
|
+
when: string;
|
|
36
|
+
/** jq expression producing the merged payload when `when` returns true. */
|
|
37
|
+
fold: string;
|
|
38
|
+
}
|
|
16
39
|
/** Serialisation format for client-server communication. */
|
|
17
40
|
export type Format = "json" | "msgpack";
|
|
18
41
|
/**
|
|
@@ -99,6 +122,12 @@ export interface EnqueueOptions {
|
|
|
99
122
|
* Uniqueness scope. Only valid when `uniqueKey` is set.
|
|
100
123
|
*/
|
|
101
124
|
uniqueWhile?: UniqueScope;
|
|
125
|
+
/**
|
|
126
|
+
* Batching configuration for folded jobs.
|
|
127
|
+
*
|
|
128
|
+
* Requires a pro license on the server.
|
|
129
|
+
*/
|
|
130
|
+
batch?: BatchConfig;
|
|
102
131
|
}
|
|
103
132
|
/** Options for reporting a job failure. */
|
|
104
133
|
export interface FailureOptions {
|
|
@@ -366,8 +395,28 @@ export interface EnqueueInput {
|
|
|
366
395
|
*
|
|
367
396
|
* The key is global across all queues and job types. Prefix with the job
|
|
368
397
|
* type to make it unique per job type.
|
|
398
|
+
*
|
|
399
|
+
* Accepts either a literal string or a function that derives the key
|
|
400
|
+
* from this enqueue input at call time. The function form is what
|
|
401
|
+
* `payloadHasher(...)` returns.
|
|
369
402
|
*/
|
|
370
|
-
uniqueKey?: string;
|
|
403
|
+
uniqueKey?: string | ((input: EnqueueInput) => string);
|
|
371
404
|
/** Uniqueness scope. Only valid when `uniqueKey` is set. */
|
|
372
405
|
uniqueWhile?: UniqueScope;
|
|
406
|
+
/**
|
|
407
|
+
* Batching configuration for folded jobs.
|
|
408
|
+
*
|
|
409
|
+
* `key` can be either a literal string or a function that derives the
|
|
410
|
+
* key from this enqueue input at call time. `batchConfig(...)`
|
|
411
|
+
* returns a value shaped like this with `key` as a function; users
|
|
412
|
+
* can also spread its result and override `key` with a custom
|
|
413
|
+
* derivation.
|
|
414
|
+
*
|
|
415
|
+
* Requires a pro license on the server.
|
|
416
|
+
*/
|
|
417
|
+
batch?: {
|
|
418
|
+
key: string | ((input: EnqueueInput) => string);
|
|
419
|
+
when: string;
|
|
420
|
+
fold: string;
|
|
421
|
+
};
|
|
373
422
|
}
|
package/dist/unique-key.d.ts
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module
|
|
8
8
|
*/
|
|
9
|
-
import { type Hash } from "node:crypto";
|
|
10
9
|
import type { JobFunction } from "./handler.ts";
|
|
11
10
|
/**
|
|
12
11
|
* Build a function that computes a unique key from a subset of the payload.
|
|
@@ -24,6 +23,11 @@ import type { JobFunction } from "./handler.ts";
|
|
|
24
23
|
* that represent the same logical event), write your own plain function
|
|
25
24
|
* with whatever key format you want; it will pass through unchanged.
|
|
26
25
|
*
|
|
26
|
+
* For the newer `(input) => string` shape used by `client.enqueue({...})`
|
|
27
|
+
* directly, use {@link payloadHasher} instead — this function is a
|
|
28
|
+
* `(fn, payload)`-signature adapter around it that fits the
|
|
29
|
+
* `zizqOptions.uniqueKey` slot on job functions.
|
|
30
|
+
*
|
|
27
31
|
* @example Unique by specific payload fields
|
|
28
32
|
* ```ts
|
|
29
33
|
* import { uniqueKey } from "@zizq-labs/zizq";
|
|
@@ -44,15 +48,3 @@ import type { JobFunction } from "./handler.ts";
|
|
|
44
48
|
* ```
|
|
45
49
|
*/
|
|
46
50
|
export declare function uniqueKey(...fields: string[]): (fn: JobFunction, payload: unknown) => string;
|
|
47
|
-
/**
|
|
48
|
-
* Stream a JSON-compatible value into a crypto hash as canonical JSON:
|
|
49
|
-
* object keys sorted, arrays in order, primitives emitted via `JSON.stringify`.
|
|
50
|
-
*
|
|
51
|
-
* The resulting byte stream is unambiguous because strings are quoted,
|
|
52
|
-
* `null`/`true`/`false` are fixed tokens, and commas separate items within
|
|
53
|
-
* containers (so `[1,2]` and `[12]` hash differently).
|
|
54
|
-
*
|
|
55
|
-
* The input must already be normalised JSON data, as from
|
|
56
|
-
* `JSON.parse(JSON.stringify(x))`.
|
|
57
|
-
*/
|
|
58
|
-
export declare function hashInto(hash: Hash, value: unknown): void;
|
package/dist/unique-key.js
CHANGED
|
@@ -1,14 +1,6 @@
|
|
|
1
1
|
// Copyright (c) 2026 Chris Corbyn <chris@zizq.io>
|
|
2
2
|
// Licensed under the MIT License. See LICENSE file for details.
|
|
3
|
-
|
|
4
|
-
* Helpers for building unique keys from job payloads.
|
|
5
|
-
*
|
|
6
|
-
* The returned functions are suitable for assigning to `zizqOptions.uniqueKey`
|
|
7
|
-
* on a job function.
|
|
8
|
-
*
|
|
9
|
-
* @module
|
|
10
|
-
*/
|
|
11
|
-
import { createHash } from "node:crypto";
|
|
3
|
+
import { payloadHasher } from "./payload-hasher.js";
|
|
12
4
|
/**
|
|
13
5
|
* Build a function that computes a unique key from a subset of the payload.
|
|
14
6
|
*
|
|
@@ -25,6 +17,11 @@ import { createHash } from "node:crypto";
|
|
|
25
17
|
* that represent the same logical event), write your own plain function
|
|
26
18
|
* with whatever key format you want; it will pass through unchanged.
|
|
27
19
|
*
|
|
20
|
+
* For the newer `(input) => string` shape used by `client.enqueue({...})`
|
|
21
|
+
* directly, use {@link payloadHasher} instead — this function is a
|
|
22
|
+
* `(fn, payload)`-signature adapter around it that fits the
|
|
23
|
+
* `zizqOptions.uniqueKey` slot on job functions.
|
|
24
|
+
*
|
|
28
25
|
* @example Unique by specific payload fields
|
|
29
26
|
* ```ts
|
|
30
27
|
* import { uniqueKey } from "@zizq-labs/zizq";
|
|
@@ -45,78 +42,24 @@ import { createHash } from "node:crypto";
|
|
|
45
42
|
* ```
|
|
46
43
|
*/
|
|
47
44
|
export function uniqueKey(...fields) {
|
|
45
|
+
const only = fields.length === 0 ? undefined : fields.map((f) => `.${f}`);
|
|
46
|
+
const hasher = payloadHasher({ only });
|
|
48
47
|
return (fn, payload) => {
|
|
49
48
|
const jobType = fn.zizqOptions?.type ?? fn.name;
|
|
50
49
|
if (!jobType) {
|
|
51
50
|
throw new Error("uniqueKey: job function must have a name or zizqOptions.type");
|
|
52
51
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
52
|
+
if (fields.length > 0) {
|
|
53
|
+
if (payload === null ||
|
|
54
|
+
typeof payload !== "object" ||
|
|
55
|
+
Array.isArray(payload)) {
|
|
56
|
+
throw new Error(`uniqueKey: cannot pick fields from a non-object payload (got ${payload === null
|
|
57
|
+
? "null"
|
|
58
|
+
: Array.isArray(payload)
|
|
59
|
+
? "array"
|
|
60
|
+
: typeof payload}). Use uniqueKey() with no fields to hash the whole payload, or write a custom resolver.`);
|
|
60
61
|
}
|
|
61
|
-
input = pick(payload, fields);
|
|
62
62
|
}
|
|
63
|
-
|
|
64
|
-
// NaN, undefined, BigInt, functions, etc.) and detect circular
|
|
65
|
-
// references. The result is plain JSON-compatible data.
|
|
66
|
-
const normalised = JSON.parse(JSON.stringify(input));
|
|
67
|
-
const hash = createHash("sha256");
|
|
68
|
-
hashInto(hash, normalised);
|
|
69
|
-
return `${jobType}:${hash.digest("hex")}`;
|
|
63
|
+
return hasher({ type: fn, queue: "", payload });
|
|
70
64
|
};
|
|
71
65
|
}
|
|
72
|
-
// --- Internal helpers ---
|
|
73
|
-
/** Extract the given top-level fields from an object, preserving order. */
|
|
74
|
-
function pick(obj, fields) {
|
|
75
|
-
const result = {};
|
|
76
|
-
for (const field of fields) {
|
|
77
|
-
if (field in obj)
|
|
78
|
-
result[field] = obj[field];
|
|
79
|
-
}
|
|
80
|
-
return result;
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* Stream a JSON-compatible value into a crypto hash as canonical JSON:
|
|
84
|
-
* object keys sorted, arrays in order, primitives emitted via `JSON.stringify`.
|
|
85
|
-
*
|
|
86
|
-
* The resulting byte stream is unambiguous because strings are quoted,
|
|
87
|
-
* `null`/`true`/`false` are fixed tokens, and commas separate items within
|
|
88
|
-
* containers (so `[1,2]` and `[12]` hash differently).
|
|
89
|
-
*
|
|
90
|
-
* The input must already be normalised JSON data, as from
|
|
91
|
-
* `JSON.parse(JSON.stringify(x))`.
|
|
92
|
-
*/
|
|
93
|
-
export function hashInto(hash, value) {
|
|
94
|
-
// Bare number, string, boolean or null. Use JSON repr.
|
|
95
|
-
if (value === null || typeof value !== "object") {
|
|
96
|
-
hash.update(JSON.stringify(value));
|
|
97
|
-
return;
|
|
98
|
-
}
|
|
99
|
-
// Arrays hash in the original order, with "[" and "]" markers.
|
|
100
|
-
// We don't worry about the trailing comma; we just need a stable digest.
|
|
101
|
-
if (Array.isArray(value)) {
|
|
102
|
-
hash.update("[");
|
|
103
|
-
for (const item of value) {
|
|
104
|
-
hashInto(hash, item);
|
|
105
|
-
hash.update(",");
|
|
106
|
-
}
|
|
107
|
-
hash.update("]");
|
|
108
|
-
return;
|
|
109
|
-
}
|
|
110
|
-
// Objects hash in key-sorted order, with "{" and "}" markers.
|
|
111
|
-
// We don't worry about the trailing comma; we just need a stable digest.
|
|
112
|
-
hash.update("{");
|
|
113
|
-
const obj = value;
|
|
114
|
-
const keys = Object.keys(obj).sort();
|
|
115
|
-
for (const key of keys) {
|
|
116
|
-
hash.update(JSON.stringify(key));
|
|
117
|
-
hash.update(":");
|
|
118
|
-
hashInto(hash, obj[key]);
|
|
119
|
-
hash.update(",");
|
|
120
|
-
}
|
|
121
|
-
hash.update("}");
|
|
122
|
-
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zizq-labs/zizq",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Node.js client for the Zizq job queue server",
|
|
5
5
|
"homepage": "https://zizq.io",
|
|
6
6
|
"repository": {
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"scripts": {
|
|
35
35
|
"build": "tsc",
|
|
36
36
|
"typecheck": "tsc --noEmit",
|
|
37
|
-
"test": "node --test --experimental-strip-types 'src/**/*.test.ts'"
|
|
37
|
+
"test": "node --test --experimental-strip-types --import ./src/test-setup.ts 'src/**/*.test.ts'"
|
|
38
38
|
},
|
|
39
39
|
"engines": {
|
|
40
40
|
"node": ">=22.19"
|