@gravito/stream 1.0.3 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -5
- package/dist/index.cjs +2956 -238
- package/dist/index.d.cts +563 -123
- package/dist/index.d.ts +563 -123
- package/dist/index.js +2955 -238
- package/package.json +7 -3
package/dist/index.js
CHANGED
|
@@ -125,6 +125,99 @@ var init_DatabaseDriver = __esm({
|
|
|
125
125
|
}
|
|
126
126
|
return job;
|
|
127
127
|
}
|
|
128
|
+
/**
|
|
129
|
+
* Pop multiple jobs from the queue.
|
|
130
|
+
*/
|
|
131
|
+
async popMany(queue, count) {
|
|
132
|
+
if (count <= 1) {
|
|
133
|
+
const job = await this.pop(queue);
|
|
134
|
+
return job ? [job] : [];
|
|
135
|
+
}
|
|
136
|
+
try {
|
|
137
|
+
const result = await this.dbService.execute(
|
|
138
|
+
`SELECT id, payload, attempts, created_at, available_at
|
|
139
|
+
FROM ${this.tableName}
|
|
140
|
+
WHERE queue = $1
|
|
141
|
+
AND available_at <= NOW()
|
|
142
|
+
AND (reserved_at IS NULL OR reserved_at < NOW() - INTERVAL '5 minutes')
|
|
143
|
+
ORDER BY created_at ASC
|
|
144
|
+
LIMIT ${count}
|
|
145
|
+
FOR UPDATE SKIP LOCKED`,
|
|
146
|
+
[queue]
|
|
147
|
+
);
|
|
148
|
+
const rows = Array.isArray(result) ? result : result ? [result] : [];
|
|
149
|
+
if (!rows || rows.length === 0) {
|
|
150
|
+
return [];
|
|
151
|
+
}
|
|
152
|
+
const validRows = rows.filter((r) => r?.id);
|
|
153
|
+
if (validRows.length === 0) {
|
|
154
|
+
return [];
|
|
155
|
+
}
|
|
156
|
+
const ids = validRows.map((r) => r.id);
|
|
157
|
+
await this.dbService.execute(
|
|
158
|
+
`UPDATE ${this.tableName}
|
|
159
|
+
SET reserved_at = NOW()
|
|
160
|
+
WHERE id IN (${ids.map((_, i) => `$${i + 1}`).join(", ")})`,
|
|
161
|
+
ids
|
|
162
|
+
);
|
|
163
|
+
return validRows.map((row) => {
|
|
164
|
+
const createdAt = new Date(row.created_at).getTime();
|
|
165
|
+
try {
|
|
166
|
+
const parsed = JSON.parse(row.payload);
|
|
167
|
+
return {
|
|
168
|
+
...parsed,
|
|
169
|
+
id: row.id,
|
|
170
|
+
attempts: row.attempts
|
|
171
|
+
};
|
|
172
|
+
} catch (_e) {
|
|
173
|
+
return {
|
|
174
|
+
id: row.id,
|
|
175
|
+
type: "class",
|
|
176
|
+
data: row.payload,
|
|
177
|
+
createdAt,
|
|
178
|
+
attempts: row.attempts
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
} catch (_e) {
|
|
183
|
+
const firstJob = await this.pop(queue);
|
|
184
|
+
return firstJob ? [firstJob] : [];
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Get queue statistics.
|
|
189
|
+
*/
|
|
190
|
+
async stats(queue) {
|
|
191
|
+
const failedQueue = `failed:${queue}`;
|
|
192
|
+
try {
|
|
193
|
+
const pendingRes = await this.dbService.execute(
|
|
194
|
+
`SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue = $1 AND available_at <= NOW() AND reserved_at IS NULL`,
|
|
195
|
+
[queue]
|
|
196
|
+
);
|
|
197
|
+
const delayedRes = await this.dbService.execute(
|
|
198
|
+
`SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue = $1 AND available_at > NOW()`,
|
|
199
|
+
[queue]
|
|
200
|
+
);
|
|
201
|
+
const reservedRes = await this.dbService.execute(
|
|
202
|
+
`SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue = $1 AND reserved_at IS NOT NULL`,
|
|
203
|
+
[queue]
|
|
204
|
+
);
|
|
205
|
+
const failedRes = await this.dbService.execute(
|
|
206
|
+
`SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue = $1`,
|
|
207
|
+
[failedQueue]
|
|
208
|
+
);
|
|
209
|
+
return {
|
|
210
|
+
queue,
|
|
211
|
+
size: pendingRes[0]?.count || 0,
|
|
212
|
+
delayed: delayedRes[0]?.count || 0,
|
|
213
|
+
reserved: reservedRes[0]?.count || 0,
|
|
214
|
+
failed: failedRes[0]?.count || 0
|
|
215
|
+
};
|
|
216
|
+
} catch (err) {
|
|
217
|
+
console.error("[DatabaseDriver] Failed to get stats:", err);
|
|
218
|
+
return { queue, size: 0, delayed: 0, reserved: 0, failed: 0 };
|
|
219
|
+
}
|
|
220
|
+
}
|
|
128
221
|
/**
|
|
129
222
|
* Get queue size.
|
|
130
223
|
*/
|
|
@@ -145,21 +238,44 @@ var init_DatabaseDriver = __esm({
|
|
|
145
238
|
async clear(queue) {
|
|
146
239
|
await this.dbService.execute(`DELETE FROM ${this.tableName} WHERE queue = $1`, [queue]);
|
|
147
240
|
}
|
|
241
|
+
/**
|
|
242
|
+
* Pop a job from the queue (blocking).
|
|
243
|
+
* Simple polling fallback for databases.
|
|
244
|
+
*/
|
|
245
|
+
async popBlocking(queue, timeout) {
|
|
246
|
+
const start = Date.now();
|
|
247
|
+
const timeoutMs = timeout * 1e3;
|
|
248
|
+
while (true) {
|
|
249
|
+
const job = await this.pop(queue);
|
|
250
|
+
if (job) {
|
|
251
|
+
return job;
|
|
252
|
+
}
|
|
253
|
+
if (timeout > 0 && Date.now() - start >= timeoutMs) {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
257
|
+
}
|
|
258
|
+
}
|
|
148
259
|
/**
|
|
149
260
|
* Push multiple jobs.
|
|
261
|
+
* Optimizes by using a single multi-row insert if possible.
|
|
150
262
|
*/
|
|
151
263
|
async pushMany(queue, jobs) {
|
|
152
264
|
if (jobs.length === 0) {
|
|
153
265
|
return;
|
|
154
266
|
}
|
|
155
267
|
await this.dbService.transaction(async (tx) => {
|
|
156
|
-
for (
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
268
|
+
for (let i = 0; i < jobs.length; i += 100) {
|
|
269
|
+
const batch = jobs.slice(i, i + 100);
|
|
270
|
+
for (const job of batch) {
|
|
271
|
+
const availableAt = job.delaySeconds ? new Date(Date.now() + job.delaySeconds * 1e3) : /* @__PURE__ */ new Date();
|
|
272
|
+
const payload = JSON.stringify(job);
|
|
273
|
+
await tx.execute(
|
|
274
|
+
`INSERT INTO ${this.tableName} (queue, payload, attempts, available_at, created_at)
|
|
275
|
+
VALUES ($1, $2, $3, $4, $5)`,
|
|
276
|
+
[queue, payload, job.attempts ?? 0, availableAt.toISOString(), (/* @__PURE__ */ new Date()).toISOString()]
|
|
277
|
+
);
|
|
278
|
+
}
|
|
163
279
|
}
|
|
164
280
|
});
|
|
165
281
|
}
|
|
@@ -335,25 +451,29 @@ var init_KafkaDriver = __esm({
|
|
|
335
451
|
await consumer.connect();
|
|
336
452
|
await consumer.subscribe({ topics: [queue] });
|
|
337
453
|
await consumer.run({
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
454
|
+
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning }) => {
|
|
455
|
+
for (const message of batch.messages) {
|
|
456
|
+
if (!isRunning() || !message.value) {
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
try {
|
|
460
|
+
const payload = JSON.parse(message.value.toString());
|
|
461
|
+
const job = {
|
|
462
|
+
id: payload.id,
|
|
463
|
+
type: payload.type,
|
|
464
|
+
data: payload.data,
|
|
465
|
+
className: payload.className,
|
|
466
|
+
createdAt: payload.createdAt,
|
|
467
|
+
delaySeconds: payload.delaySeconds,
|
|
468
|
+
attempts: payload.attempts,
|
|
469
|
+
maxAttempts: payload.maxAttempts
|
|
470
|
+
};
|
|
471
|
+
await callback(job);
|
|
472
|
+
resolveOffset(message.offset);
|
|
473
|
+
await heartbeat();
|
|
474
|
+
} catch (error) {
|
|
475
|
+
console.error("[KafkaDriver] Error processing message:", error);
|
|
476
|
+
}
|
|
357
477
|
}
|
|
358
478
|
}
|
|
359
479
|
});
|
|
@@ -438,6 +558,25 @@ var init_RabbitMQDriver = __esm({
|
|
|
438
558
|
job._raw = msg;
|
|
439
559
|
return job;
|
|
440
560
|
}
|
|
561
|
+
/**
|
|
562
|
+
* Pop multiple jobs.
|
|
563
|
+
* Uses channel.get() in a loop (no native batch get in AMQP).
|
|
564
|
+
*/
|
|
565
|
+
async popMany(queue, count) {
|
|
566
|
+
const channel = await this.ensureChannel();
|
|
567
|
+
await channel.assertQueue(queue, { durable: true });
|
|
568
|
+
const results = [];
|
|
569
|
+
for (let i = 0; i < count; i++) {
|
|
570
|
+
const msg = await channel.get(queue, { noAck: false });
|
|
571
|
+
if (!msg) {
|
|
572
|
+
break;
|
|
573
|
+
}
|
|
574
|
+
const job = JSON.parse(msg.content.toString());
|
|
575
|
+
job._raw = msg;
|
|
576
|
+
results.push(job);
|
|
577
|
+
}
|
|
578
|
+
return results;
|
|
579
|
+
}
|
|
441
580
|
/**
|
|
442
581
|
* Acknowledge a message.
|
|
443
582
|
*/
|
|
@@ -467,6 +606,9 @@ var init_RabbitMQDriver = __esm({
|
|
|
467
606
|
async subscribe(queue, callback, options = {}) {
|
|
468
607
|
const channel = await this.ensureChannel();
|
|
469
608
|
await channel.assertQueue(queue, { durable: true });
|
|
609
|
+
if (options.prefetch) {
|
|
610
|
+
await channel.prefetch(options.prefetch);
|
|
611
|
+
}
|
|
470
612
|
if (this.exchange) {
|
|
471
613
|
await channel.bindQueue(queue, this.exchange, "");
|
|
472
614
|
}
|
|
@@ -551,6 +693,63 @@ var init_RedisDriver = __esm({
|
|
|
551
693
|
else
|
|
552
694
|
return redis.call('SREM', activeSet, groupId)
|
|
553
695
|
end
|
|
696
|
+
`;
|
|
697
|
+
// Lua Logic:
|
|
698
|
+
// Iterate priorities.
|
|
699
|
+
// Check delayed.
|
|
700
|
+
// Check paused.
|
|
701
|
+
// RPOP count.
|
|
702
|
+
static POP_MANY_SCRIPT = `
|
|
703
|
+
local queue = KEYS[1]
|
|
704
|
+
local prefix = ARGV[1]
|
|
705
|
+
local count = tonumber(ARGV[2])
|
|
706
|
+
local now = tonumber(ARGV[3])
|
|
707
|
+
|
|
708
|
+
local priorities = {'critical', 'high', 'default', 'low'}
|
|
709
|
+
local result = {}
|
|
710
|
+
|
|
711
|
+
for _, priority in ipairs(priorities) do
|
|
712
|
+
if #result >= count then break end
|
|
713
|
+
|
|
714
|
+
local key = prefix .. queue
|
|
715
|
+
if priority ~= 'default' then
|
|
716
|
+
key = key .. ':' .. priority
|
|
717
|
+
end
|
|
718
|
+
|
|
719
|
+
-- Check Delayed (Move to Ready if due)
|
|
720
|
+
local delayKey = key .. ":delayed"
|
|
721
|
+
-- Optimization: Only check delayed if we need more items
|
|
722
|
+
-- Fetch up to (count - #result) delayed items
|
|
723
|
+
local needed = count - #result
|
|
724
|
+
local delayed = redis.call("ZRANGEBYSCORE", delayKey, 0, now, "LIMIT", 0, needed)
|
|
725
|
+
|
|
726
|
+
for _, job in ipairs(delayed) do
|
|
727
|
+
redis.call("ZREM", delayKey, job)
|
|
728
|
+
-- We return it directly, assuming we want to process it now.
|
|
729
|
+
-- Alternative: LPUSH to list and RPOP? No, direct return is faster.
|
|
730
|
+
table.insert(result, job)
|
|
731
|
+
needed = needed - 1
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
if #result >= count then break end
|
|
735
|
+
|
|
736
|
+
-- Check Paused
|
|
737
|
+
local isPaused = redis.call("GET", key .. ":paused")
|
|
738
|
+
if isPaused ~= "1" then
|
|
739
|
+
needed = count - #result
|
|
740
|
+
-- Loop RPOP to get items
|
|
741
|
+
for i = 1, needed do
|
|
742
|
+
local job = redis.call("RPOP", key)
|
|
743
|
+
if job then
|
|
744
|
+
table.insert(result, job)
|
|
745
|
+
else
|
|
746
|
+
break
|
|
747
|
+
end
|
|
748
|
+
end
|
|
749
|
+
end
|
|
750
|
+
end
|
|
751
|
+
|
|
752
|
+
return result
|
|
554
753
|
`;
|
|
555
754
|
constructor(config) {
|
|
556
755
|
this.client = config.client;
|
|
@@ -561,7 +760,6 @@ var init_RedisDriver = __esm({
|
|
|
561
760
|
);
|
|
562
761
|
}
|
|
563
762
|
if (typeof this.client.defineCommand === "function") {
|
|
564
|
-
;
|
|
565
763
|
this.client.defineCommand("pushGroupJob", {
|
|
566
764
|
numberOfKeys: 3,
|
|
567
765
|
lua: _RedisDriver.PUSH_SCRIPT
|
|
@@ -570,6 +768,10 @@ var init_RedisDriver = __esm({
|
|
|
570
768
|
numberOfKeys: 3,
|
|
571
769
|
lua: _RedisDriver.COMPLETE_SCRIPT
|
|
572
770
|
});
|
|
771
|
+
this.client.defineCommand("popMany", {
|
|
772
|
+
numberOfKeys: 1,
|
|
773
|
+
lua: _RedisDriver.POP_MANY_SCRIPT
|
|
774
|
+
});
|
|
573
775
|
}
|
|
574
776
|
}
|
|
575
777
|
/**
|
|
@@ -636,31 +838,70 @@ var init_RedisDriver = __esm({
|
|
|
636
838
|
}
|
|
637
839
|
}
|
|
638
840
|
/**
|
|
639
|
-
* Pop a job (
|
|
640
|
-
*
|
|
841
|
+
* Pop a job from a queue (non-blocking).
|
|
842
|
+
* Optimized with Lua script for atomic priority polling.
|
|
641
843
|
*/
|
|
642
844
|
async pop(queue) {
|
|
845
|
+
const priorities = ["critical", "high", "default", "low"];
|
|
846
|
+
const keys = [];
|
|
847
|
+
for (const p of priorities) {
|
|
848
|
+
keys.push(this.getKey(queue, p === "default" ? void 0 : p));
|
|
849
|
+
}
|
|
850
|
+
const script = `
|
|
851
|
+
local now = tonumber(ARGV[1])
|
|
852
|
+
for i, key in ipairs(KEYS) do
|
|
853
|
+
-- 1. Check delayed
|
|
854
|
+
local delayKey = key .. ":delayed"
|
|
855
|
+
local delayed = redis.call("ZRANGEBYSCORE", delayKey, 0, now, "LIMIT", 0, 1)
|
|
856
|
+
if delayed[1] then
|
|
857
|
+
redis.call("ZREM", delayKey, delayed[1])
|
|
858
|
+
return {key, delayed[1]}
|
|
859
|
+
end
|
|
860
|
+
|
|
861
|
+
-- 2. Check paused
|
|
862
|
+
local isPaused = redis.call("GET", key .. ":paused")
|
|
863
|
+
if isPaused ~= "1" then
|
|
864
|
+
-- 3. RPOP
|
|
865
|
+
local payload = redis.call("RPOP", key)
|
|
866
|
+
if payload then
|
|
867
|
+
return {key, payload}
|
|
868
|
+
end
|
|
869
|
+
end
|
|
870
|
+
end
|
|
871
|
+
return nil
|
|
872
|
+
`;
|
|
873
|
+
try {
|
|
874
|
+
const result = await this.client.eval(script, keys.length, ...keys, Date.now().toString());
|
|
875
|
+
if (result?.[1]) {
|
|
876
|
+
return this.parsePayload(result[1]);
|
|
877
|
+
}
|
|
878
|
+
} catch (err) {
|
|
879
|
+
console.error("[RedisDriver] Lua pop error:", err);
|
|
880
|
+
return this.popManualFallback(queue);
|
|
881
|
+
}
|
|
882
|
+
return null;
|
|
883
|
+
}
|
|
884
|
+
/**
|
|
885
|
+
* Manual fallback for pop if Lua fails.
|
|
886
|
+
*/
|
|
887
|
+
async popManualFallback(queue) {
|
|
643
888
|
const priorities = ["critical", "high", void 0, "low"];
|
|
644
889
|
for (const priority of priorities) {
|
|
645
890
|
const key = this.getKey(queue, priority);
|
|
646
891
|
const delayKey = `${key}:delayed`;
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
return this.parsePayload(payload2);
|
|
656
|
-
}
|
|
892
|
+
const now = Date.now();
|
|
893
|
+
const delayedJobs = await this.client.zrange?.(delayKey, 0, 0, "WITHSCORES");
|
|
894
|
+
if (delayedJobs && delayedJobs.length >= 2) {
|
|
895
|
+
const score = parseFloat(delayedJobs[1]);
|
|
896
|
+
if (score <= now) {
|
|
897
|
+
const payload2 = delayedJobs[0];
|
|
898
|
+
await this.client.zrem?.(delayKey, payload2);
|
|
899
|
+
return this.parsePayload(payload2);
|
|
657
900
|
}
|
|
658
901
|
}
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
continue;
|
|
663
|
-
}
|
|
902
|
+
const isPaused = await this.client.get?.(`${key}:paused`);
|
|
903
|
+
if (isPaused === "1") {
|
|
904
|
+
continue;
|
|
664
905
|
}
|
|
665
906
|
const payload = await this.client.rpop(key);
|
|
666
907
|
if (payload) {
|
|
@@ -669,6 +910,31 @@ var init_RedisDriver = __esm({
|
|
|
669
910
|
}
|
|
670
911
|
return null;
|
|
671
912
|
}
|
|
913
|
+
/**
|
|
914
|
+
* Pop a job from the queue (blocking).
|
|
915
|
+
* Uses BRPOP for efficiency. Supports multiple queues and priorities.
|
|
916
|
+
*/
|
|
917
|
+
async popBlocking(queues, timeout) {
|
|
918
|
+
const queueList = Array.isArray(queues) ? queues : [queues];
|
|
919
|
+
const priorities = ["critical", "high", void 0, "low"];
|
|
920
|
+
const keys = [];
|
|
921
|
+
for (const q of queueList) {
|
|
922
|
+
for (const p of priorities) {
|
|
923
|
+
keys.push(this.getKey(q, p));
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
if (typeof this.client.brpop !== "function") {
|
|
927
|
+
return this.pop(queueList[0]);
|
|
928
|
+
}
|
|
929
|
+
try {
|
|
930
|
+
const result = await this.client.brpop(...keys, timeout);
|
|
931
|
+
if (result && Array.isArray(result) && result.length >= 2) {
|
|
932
|
+
return this.parsePayload(result[1]);
|
|
933
|
+
}
|
|
934
|
+
} catch (_e) {
|
|
935
|
+
}
|
|
936
|
+
return null;
|
|
937
|
+
}
|
|
672
938
|
/**
|
|
673
939
|
* Parse Redis payload.
|
|
674
940
|
*/
|
|
@@ -718,11 +984,57 @@ var init_RedisDriver = __esm({
|
|
|
718
984
|
const delayKey = `${key}:delayed`;
|
|
719
985
|
const activeSetKey = `${this.prefix}active`;
|
|
720
986
|
await this.client.del(key);
|
|
721
|
-
if (
|
|
987
|
+
if (this.client.del) {
|
|
722
988
|
await this.client.del(delayKey);
|
|
723
989
|
await this.client.del(activeSetKey);
|
|
724
990
|
}
|
|
725
991
|
}
|
|
992
|
+
/**
|
|
993
|
+
* Get queue statistics.
|
|
994
|
+
* Optimized with Redis Pipeline to fetch all priorities and DLQ stats in one trip.
|
|
995
|
+
*/
|
|
996
|
+
async stats(queue) {
|
|
997
|
+
const priorities = ["critical", "high", "default", "low"];
|
|
998
|
+
const stats = {
|
|
999
|
+
queue,
|
|
1000
|
+
size: 0,
|
|
1001
|
+
delayed: 0,
|
|
1002
|
+
failed: 0
|
|
1003
|
+
};
|
|
1004
|
+
const keys = [];
|
|
1005
|
+
for (const p of priorities) {
|
|
1006
|
+
keys.push(this.getKey(queue, p === "default" ? void 0 : p));
|
|
1007
|
+
}
|
|
1008
|
+
try {
|
|
1009
|
+
if (typeof this.client.pipeline === "function") {
|
|
1010
|
+
const pipe = this.client.pipeline();
|
|
1011
|
+
for (const key of keys) {
|
|
1012
|
+
pipe.llen(key);
|
|
1013
|
+
pipe.zcard(`${key}:delayed`);
|
|
1014
|
+
}
|
|
1015
|
+
pipe.llen(`${this.getKey(queue)}:failed`);
|
|
1016
|
+
const results = await pipe.exec();
|
|
1017
|
+
if (results) {
|
|
1018
|
+
let i = 0;
|
|
1019
|
+
for (const _p of priorities) {
|
|
1020
|
+
stats.size += results[i][1] || 0;
|
|
1021
|
+
stats.delayed += results[i + 1][1] || 0;
|
|
1022
|
+
i += 2;
|
|
1023
|
+
}
|
|
1024
|
+
stats.failed = results[i][1] || 0;
|
|
1025
|
+
}
|
|
1026
|
+
} else {
|
|
1027
|
+
for (const key of keys) {
|
|
1028
|
+
stats.size += await this.client.llen?.(key) || 0;
|
|
1029
|
+
stats.delayed += await this.client.zcard?.(`${key}:delayed`) || 0;
|
|
1030
|
+
}
|
|
1031
|
+
stats.failed = await this.client.llen?.(`${this.getKey(queue)}:failed`) || 0;
|
|
1032
|
+
}
|
|
1033
|
+
} catch (err) {
|
|
1034
|
+
console.error("[RedisDriver] Failed to get stats:", err);
|
|
1035
|
+
}
|
|
1036
|
+
return stats;
|
|
1037
|
+
}
|
|
726
1038
|
/**
|
|
727
1039
|
* Push multiple jobs.
|
|
728
1040
|
*/
|
|
@@ -733,6 +1045,43 @@ var init_RedisDriver = __esm({
|
|
|
733
1045
|
const hasGroup = jobs.some((j) => j.groupId);
|
|
734
1046
|
const hasPriority = jobs.some((j) => j.priority);
|
|
735
1047
|
if (hasGroup || hasPriority) {
|
|
1048
|
+
if (typeof this.client.pipeline === "function") {
|
|
1049
|
+
const pipe = this.client.pipeline();
|
|
1050
|
+
for (const job of jobs) {
|
|
1051
|
+
const priority = job.priority;
|
|
1052
|
+
const key2 = this.getKey(queue, priority);
|
|
1053
|
+
const groupId = job.groupId;
|
|
1054
|
+
const payload = JSON.stringify({
|
|
1055
|
+
id: job.id,
|
|
1056
|
+
type: job.type,
|
|
1057
|
+
data: job.data,
|
|
1058
|
+
className: job.className,
|
|
1059
|
+
createdAt: job.createdAt,
|
|
1060
|
+
delaySeconds: job.delaySeconds,
|
|
1061
|
+
attempts: job.attempts,
|
|
1062
|
+
maxAttempts: job.maxAttempts,
|
|
1063
|
+
groupId,
|
|
1064
|
+
priority,
|
|
1065
|
+
error: job.error,
|
|
1066
|
+
failedAt: job.failedAt
|
|
1067
|
+
});
|
|
1068
|
+
if (groupId) {
|
|
1069
|
+
const activeSetKey = `${this.prefix}active`;
|
|
1070
|
+
const pendingListKey = `${this.prefix}pending:${groupId}`;
|
|
1071
|
+
pipe.pushGroupJob(key2, activeSetKey, pendingListKey, groupId, payload);
|
|
1072
|
+
} else {
|
|
1073
|
+
if (job.delaySeconds && job.delaySeconds > 0) {
|
|
1074
|
+
const delayKey = `${key2}:delayed`;
|
|
1075
|
+
const score = Date.now() + job.delaySeconds * 1e3;
|
|
1076
|
+
pipe.zadd(delayKey, score, payload);
|
|
1077
|
+
} else {
|
|
1078
|
+
pipe.lpush(key2, payload);
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
await pipe.exec();
|
|
1083
|
+
return;
|
|
1084
|
+
}
|
|
736
1085
|
for (const job of jobs) {
|
|
737
1086
|
await this.push(queue, job, {
|
|
738
1087
|
groupId: job.groupId,
|
|
@@ -760,17 +1109,80 @@ var init_RedisDriver = __esm({
|
|
|
760
1109
|
}
|
|
761
1110
|
/**
|
|
762
1111
|
* Pop multiple jobs.
|
|
1112
|
+
* Atomic operation across multiple priority levels.
|
|
763
1113
|
*/
|
|
764
1114
|
async popMany(queue, count) {
|
|
765
|
-
|
|
1115
|
+
if (count <= 0) {
|
|
1116
|
+
return [];
|
|
1117
|
+
}
|
|
1118
|
+
if (count === 1) {
|
|
1119
|
+
const job = await this.pop(queue);
|
|
1120
|
+
return job ? [job] : [];
|
|
1121
|
+
}
|
|
1122
|
+
if (typeof this.client.popMany === "function") {
|
|
1123
|
+
try {
|
|
1124
|
+
const result = await this.client.popMany(queue, this.prefix, count, Date.now().toString());
|
|
1125
|
+
if (Array.isArray(result) && result.length > 0) {
|
|
1126
|
+
return result.map((p) => this.parsePayload(p));
|
|
1127
|
+
} else if (Array.isArray(result) && result.length === 0) {
|
|
1128
|
+
} else {
|
|
1129
|
+
}
|
|
1130
|
+
if (Array.isArray(result)) {
|
|
1131
|
+
return result.map((p) => this.parsePayload(p));
|
|
1132
|
+
}
|
|
1133
|
+
} catch (err) {
|
|
1134
|
+
console.error("[RedisDriver] Lua popMany error:", err);
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
const priorities = ["critical", "high", "default", "low"];
|
|
766
1138
|
const results = [];
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
if (
|
|
770
|
-
results.push(this.parsePayload(payload));
|
|
771
|
-
} else {
|
|
1139
|
+
let remaining = count;
|
|
1140
|
+
for (const priority of priorities) {
|
|
1141
|
+
if (remaining <= 0) {
|
|
772
1142
|
break;
|
|
773
1143
|
}
|
|
1144
|
+
const key = this.getKey(queue, priority === "default" ? void 0 : priority);
|
|
1145
|
+
const isPaused = await this.client.get?.(`${key}:paused`);
|
|
1146
|
+
if (isPaused === "1") {
|
|
1147
|
+
continue;
|
|
1148
|
+
}
|
|
1149
|
+
let fetched = [];
|
|
1150
|
+
try {
|
|
1151
|
+
const reply = await this.client.rpop(key, remaining);
|
|
1152
|
+
if (reply) {
|
|
1153
|
+
fetched = Array.isArray(reply) ? reply : [reply];
|
|
1154
|
+
}
|
|
1155
|
+
} catch (_e) {
|
|
1156
|
+
if (typeof this.client.pipeline === "function") {
|
|
1157
|
+
const pipeline = this.client.pipeline();
|
|
1158
|
+
for (let i = 0; i < remaining; i++) {
|
|
1159
|
+
pipeline.rpop(key);
|
|
1160
|
+
}
|
|
1161
|
+
const replies = await pipeline.exec();
|
|
1162
|
+
if (replies) {
|
|
1163
|
+
fetched = replies.map((r) => r[1]).filter((r) => r !== null);
|
|
1164
|
+
}
|
|
1165
|
+
} else {
|
|
1166
|
+
for (let i = 0; i < remaining; i++) {
|
|
1167
|
+
const res = await this.client.rpop(key);
|
|
1168
|
+
if (res) {
|
|
1169
|
+
fetched.push(res);
|
|
1170
|
+
} else {
|
|
1171
|
+
break;
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
if (fetched.length > 0) {
|
|
1177
|
+
for (const payload of fetched) {
|
|
1178
|
+
try {
|
|
1179
|
+
results.push(this.parsePayload(payload));
|
|
1180
|
+
} catch (e) {
|
|
1181
|
+
console.error("[RedisDriver] Failed to parse job payload:", e);
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
remaining -= fetched.length;
|
|
1185
|
+
}
|
|
774
1186
|
}
|
|
775
1187
|
return results;
|
|
776
1188
|
}
|
|
@@ -814,7 +1226,7 @@ var init_RedisDriver = __esm({
|
|
|
814
1226
|
const client = this.client;
|
|
815
1227
|
if (typeof client.incr === "function") {
|
|
816
1228
|
const current = await client.incr(windowKey);
|
|
817
|
-
if (current === 1) {
|
|
1229
|
+
if (current === 1 && client.expire) {
|
|
818
1230
|
await client.expire(windowKey, Math.ceil(config.duration / 1e3) + 1);
|
|
819
1231
|
}
|
|
820
1232
|
return current <= config.max;
|
|
@@ -826,6 +1238,9 @@ var init_RedisDriver = __esm({
|
|
|
826
1238
|
*/
|
|
827
1239
|
async getFailed(queue, start = 0, end = -1) {
|
|
828
1240
|
const key = `${this.getKey(queue)}:failed`;
|
|
1241
|
+
if (typeof this.client.lrange !== "function") {
|
|
1242
|
+
return [];
|
|
1243
|
+
}
|
|
829
1244
|
const payloads = await this.client.lrange(key, start, end);
|
|
830
1245
|
return payloads.map((p) => this.parsePayload(p));
|
|
831
1246
|
}
|
|
@@ -837,6 +1252,9 @@ var init_RedisDriver = __esm({
|
|
|
837
1252
|
const failedKey = `${this.getKey(queue)}:failed`;
|
|
838
1253
|
let retried = 0;
|
|
839
1254
|
for (let i = 0; i < count; i++) {
|
|
1255
|
+
if (typeof this.client.rpop !== "function") {
|
|
1256
|
+
break;
|
|
1257
|
+
}
|
|
840
1258
|
const payload = await this.client.rpop(failedKey);
|
|
841
1259
|
if (!payload) {
|
|
842
1260
|
break;
|
|
@@ -959,6 +1377,40 @@ var init_SQSDriver = __esm({
|
|
|
959
1377
|
...message.ReceiptHandle && { receiptHandle: message.ReceiptHandle }
|
|
960
1378
|
};
|
|
961
1379
|
}
|
|
1380
|
+
/**
|
|
1381
|
+
* Pop multiple jobs.
|
|
1382
|
+
* Leverages SQS MaxNumberOfMessages (up to 10).
|
|
1383
|
+
*/
|
|
1384
|
+
async popMany(queue, count) {
|
|
1385
|
+
const { ReceiveMessageCommand } = await import("@aws-sdk/client-sqs");
|
|
1386
|
+
const queueUrl = await this.getQueueUrl(queue);
|
|
1387
|
+
const limit = Math.min(count, 10);
|
|
1388
|
+
const response = await this.client.send(
|
|
1389
|
+
new ReceiveMessageCommand({
|
|
1390
|
+
QueueUrl: queueUrl,
|
|
1391
|
+
MaxNumberOfMessages: limit,
|
|
1392
|
+
WaitTimeSeconds: this.waitTimeSeconds,
|
|
1393
|
+
VisibilityTimeout: this.visibilityTimeout
|
|
1394
|
+
})
|
|
1395
|
+
);
|
|
1396
|
+
if (!response.Messages || response.Messages.length === 0) {
|
|
1397
|
+
return [];
|
|
1398
|
+
}
|
|
1399
|
+
return response.Messages.map((message) => {
|
|
1400
|
+
const payload = JSON.parse(message.Body ?? "{}");
|
|
1401
|
+
return {
|
|
1402
|
+
id: payload.id ?? message.MessageId,
|
|
1403
|
+
type: payload.type,
|
|
1404
|
+
data: payload.data,
|
|
1405
|
+
className: payload.className,
|
|
1406
|
+
createdAt: payload.createdAt,
|
|
1407
|
+
delaySeconds: payload.delaySeconds,
|
|
1408
|
+
attempts: payload.attempts,
|
|
1409
|
+
maxAttempts: payload.maxAttempts,
|
|
1410
|
+
receiptHandle: message.ReceiptHandle
|
|
1411
|
+
};
|
|
1412
|
+
});
|
|
1413
|
+
}
|
|
962
1414
|
/**
|
|
963
1415
|
* Get queue size (approximate).
|
|
964
1416
|
*/
|
|
@@ -1062,6 +1514,1830 @@ var init_SQSDriver = __esm({
|
|
|
1062
1514
|
}
|
|
1063
1515
|
});
|
|
1064
1516
|
|
|
1517
|
+
// src/persistence/BufferedPersistence.ts
|
|
1518
|
+
var BufferedPersistence_exports = {};
|
|
1519
|
+
__export(BufferedPersistence_exports, {
|
|
1520
|
+
BufferedPersistence: () => BufferedPersistence
|
|
1521
|
+
});
|
|
1522
|
+
var BufferedPersistence;
|
|
1523
|
+
var init_BufferedPersistence = __esm({
|
|
1524
|
+
"src/persistence/BufferedPersistence.ts"() {
|
|
1525
|
+
"use strict";
|
|
1526
|
+
BufferedPersistence = class {
|
|
1527
|
+
constructor(adapter, options = {}) {
|
|
1528
|
+
this.adapter = adapter;
|
|
1529
|
+
this.maxBufferSize = options.maxBufferSize ?? 50;
|
|
1530
|
+
this.flushInterval = options.flushInterval ?? 5e3;
|
|
1531
|
+
}
|
|
1532
|
+
jobBuffer = [];
|
|
1533
|
+
logBuffer = [];
|
|
1534
|
+
flushTimer = null;
|
|
1535
|
+
maxBufferSize;
|
|
1536
|
+
flushInterval;
|
|
1537
|
+
async archive(queue, job, status) {
|
|
1538
|
+
this.jobBuffer.push({ queue, job, status });
|
|
1539
|
+
if (this.jobBuffer.length >= this.maxBufferSize) {
|
|
1540
|
+
this.flush().catch((err) => {
|
|
1541
|
+
console.error("[BufferedPersistence] Auto-flush failed (jobs):", err.message || err);
|
|
1542
|
+
});
|
|
1543
|
+
} else {
|
|
1544
|
+
this.ensureFlushTimer();
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
async find(queue, id) {
|
|
1548
|
+
return this.adapter.find(queue, id);
|
|
1549
|
+
}
|
|
1550
|
+
async list(queue, options) {
|
|
1551
|
+
return this.adapter.list(queue, options);
|
|
1552
|
+
}
|
|
1553
|
+
async archiveMany(jobs) {
|
|
1554
|
+
if (this.adapter.archiveMany) {
|
|
1555
|
+
return this.adapter.archiveMany(jobs);
|
|
1556
|
+
}
|
|
1557
|
+
for (const item of jobs) {
|
|
1558
|
+
await this.adapter.archive(item.queue, item.job, item.status);
|
|
1559
|
+
}
|
|
1560
|
+
}
|
|
1561
|
+
async cleanup(days) {
|
|
1562
|
+
return this.adapter.cleanup(days);
|
|
1563
|
+
}
|
|
1564
|
+
async flush() {
|
|
1565
|
+
if (this.flushTimer) {
|
|
1566
|
+
clearTimeout(this.flushTimer);
|
|
1567
|
+
this.flushTimer = null;
|
|
1568
|
+
}
|
|
1569
|
+
const jobs = [...this.jobBuffer];
|
|
1570
|
+
const logs = [...this.logBuffer];
|
|
1571
|
+
this.jobBuffer = [];
|
|
1572
|
+
this.logBuffer = [];
|
|
1573
|
+
const promises = [];
|
|
1574
|
+
if (jobs.length > 0) {
|
|
1575
|
+
if (this.adapter.archiveMany) {
|
|
1576
|
+
promises.push(this.adapter.archiveMany(jobs));
|
|
1577
|
+
} else {
|
|
1578
|
+
promises.push(
|
|
1579
|
+
(async () => {
|
|
1580
|
+
for (const item of jobs) {
|
|
1581
|
+
await this.adapter.archive(item.queue, item.job, item.status);
|
|
1582
|
+
}
|
|
1583
|
+
})()
|
|
1584
|
+
);
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
if (logs.length > 0) {
|
|
1588
|
+
if (this.adapter.archiveLogMany) {
|
|
1589
|
+
promises.push(this.adapter.archiveLogMany(logs));
|
|
1590
|
+
} else {
|
|
1591
|
+
promises.push(
|
|
1592
|
+
(async () => {
|
|
1593
|
+
for (const log of logs) {
|
|
1594
|
+
await this.adapter.archiveLog(log);
|
|
1595
|
+
}
|
|
1596
|
+
})()
|
|
1597
|
+
);
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
await Promise.all(promises);
|
|
1601
|
+
}
|
|
1602
|
+
async count(queue, options) {
|
|
1603
|
+
return this.adapter.count(queue, options);
|
|
1604
|
+
}
|
|
1605
|
+
async archiveLog(log) {
|
|
1606
|
+
this.logBuffer.push(log);
|
|
1607
|
+
if (this.logBuffer.length >= this.maxBufferSize) {
|
|
1608
|
+
this.flush().catch((err) => {
|
|
1609
|
+
console.error("[BufferedPersistence] Auto-flush failed (logs):", err.message || err);
|
|
1610
|
+
});
|
|
1611
|
+
} else {
|
|
1612
|
+
this.ensureFlushTimer();
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
async archiveLogMany(logs) {
|
|
1616
|
+
if (this.adapter.archiveLogMany) {
|
|
1617
|
+
return this.adapter.archiveLogMany(logs);
|
|
1618
|
+
}
|
|
1619
|
+
for (const log of logs) {
|
|
1620
|
+
await this.adapter.archiveLog(log);
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
async listLogs(options) {
|
|
1624
|
+
return this.adapter.listLogs(options);
|
|
1625
|
+
}
|
|
1626
|
+
async countLogs(options) {
|
|
1627
|
+
return this.adapter.countLogs(options);
|
|
1628
|
+
}
|
|
1629
|
+
ensureFlushTimer() {
|
|
1630
|
+
if (this.flushTimer) {
|
|
1631
|
+
return;
|
|
1632
|
+
}
|
|
1633
|
+
this.flushTimer = setTimeout(() => {
|
|
1634
|
+
this.flush().catch((err) => {
|
|
1635
|
+
console.error("[BufferedPersistence] Interval flush failed:", err.message || err);
|
|
1636
|
+
});
|
|
1637
|
+
}, this.flushInterval);
|
|
1638
|
+
}
|
|
1639
|
+
};
|
|
1640
|
+
}
|
|
1641
|
+
});
|
|
1642
|
+
|
|
1643
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/utf8.mjs
|
|
1644
|
+
function utf8Count(str) {
|
|
1645
|
+
const strLength = str.length;
|
|
1646
|
+
let byteLength = 0;
|
|
1647
|
+
let pos = 0;
|
|
1648
|
+
while (pos < strLength) {
|
|
1649
|
+
let value = str.charCodeAt(pos++);
|
|
1650
|
+
if ((value & 4294967168) === 0) {
|
|
1651
|
+
byteLength++;
|
|
1652
|
+
continue;
|
|
1653
|
+
} else if ((value & 4294965248) === 0) {
|
|
1654
|
+
byteLength += 2;
|
|
1655
|
+
} else {
|
|
1656
|
+
if (value >= 55296 && value <= 56319) {
|
|
1657
|
+
if (pos < strLength) {
|
|
1658
|
+
const extra = str.charCodeAt(pos);
|
|
1659
|
+
if ((extra & 64512) === 56320) {
|
|
1660
|
+
++pos;
|
|
1661
|
+
value = ((value & 1023) << 10) + (extra & 1023) + 65536;
|
|
1662
|
+
}
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
if ((value & 4294901760) === 0) {
|
|
1666
|
+
byteLength += 3;
|
|
1667
|
+
} else {
|
|
1668
|
+
byteLength += 4;
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
return byteLength;
|
|
1673
|
+
}
|
|
1674
|
+
function utf8EncodeJs(str, output, outputOffset) {
|
|
1675
|
+
const strLength = str.length;
|
|
1676
|
+
let offset = outputOffset;
|
|
1677
|
+
let pos = 0;
|
|
1678
|
+
while (pos < strLength) {
|
|
1679
|
+
let value = str.charCodeAt(pos++);
|
|
1680
|
+
if ((value & 4294967168) === 0) {
|
|
1681
|
+
output[offset++] = value;
|
|
1682
|
+
continue;
|
|
1683
|
+
} else if ((value & 4294965248) === 0) {
|
|
1684
|
+
output[offset++] = value >> 6 & 31 | 192;
|
|
1685
|
+
} else {
|
|
1686
|
+
if (value >= 55296 && value <= 56319) {
|
|
1687
|
+
if (pos < strLength) {
|
|
1688
|
+
const extra = str.charCodeAt(pos);
|
|
1689
|
+
if ((extra & 64512) === 56320) {
|
|
1690
|
+
++pos;
|
|
1691
|
+
value = ((value & 1023) << 10) + (extra & 1023) + 65536;
|
|
1692
|
+
}
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
if ((value & 4294901760) === 0) {
|
|
1696
|
+
output[offset++] = value >> 12 & 15 | 224;
|
|
1697
|
+
output[offset++] = value >> 6 & 63 | 128;
|
|
1698
|
+
} else {
|
|
1699
|
+
output[offset++] = value >> 18 & 7 | 240;
|
|
1700
|
+
output[offset++] = value >> 12 & 63 | 128;
|
|
1701
|
+
output[offset++] = value >> 6 & 63 | 128;
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
output[offset++] = value & 63 | 128;
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
function utf8EncodeTE(str, output, outputOffset) {
|
|
1708
|
+
sharedTextEncoder.encodeInto(str, output.subarray(outputOffset));
|
|
1709
|
+
}
|
|
1710
|
+
function utf8Encode(str, output, outputOffset) {
|
|
1711
|
+
if (str.length > TEXT_ENCODER_THRESHOLD) {
|
|
1712
|
+
utf8EncodeTE(str, output, outputOffset);
|
|
1713
|
+
} else {
|
|
1714
|
+
utf8EncodeJs(str, output, outputOffset);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
function utf8DecodeJs(bytes, inputOffset, byteLength) {
|
|
1718
|
+
let offset = inputOffset;
|
|
1719
|
+
const end = offset + byteLength;
|
|
1720
|
+
const units = [];
|
|
1721
|
+
let result = "";
|
|
1722
|
+
while (offset < end) {
|
|
1723
|
+
const byte1 = bytes[offset++];
|
|
1724
|
+
if ((byte1 & 128) === 0) {
|
|
1725
|
+
units.push(byte1);
|
|
1726
|
+
} else if ((byte1 & 224) === 192) {
|
|
1727
|
+
const byte2 = bytes[offset++] & 63;
|
|
1728
|
+
units.push((byte1 & 31) << 6 | byte2);
|
|
1729
|
+
} else if ((byte1 & 240) === 224) {
|
|
1730
|
+
const byte2 = bytes[offset++] & 63;
|
|
1731
|
+
const byte3 = bytes[offset++] & 63;
|
|
1732
|
+
units.push((byte1 & 31) << 12 | byte2 << 6 | byte3);
|
|
1733
|
+
} else if ((byte1 & 248) === 240) {
|
|
1734
|
+
const byte2 = bytes[offset++] & 63;
|
|
1735
|
+
const byte3 = bytes[offset++] & 63;
|
|
1736
|
+
const byte4 = bytes[offset++] & 63;
|
|
1737
|
+
let unit = (byte1 & 7) << 18 | byte2 << 12 | byte3 << 6 | byte4;
|
|
1738
|
+
if (unit > 65535) {
|
|
1739
|
+
unit -= 65536;
|
|
1740
|
+
units.push(unit >>> 10 & 1023 | 55296);
|
|
1741
|
+
unit = 56320 | unit & 1023;
|
|
1742
|
+
}
|
|
1743
|
+
units.push(unit);
|
|
1744
|
+
} else {
|
|
1745
|
+
units.push(byte1);
|
|
1746
|
+
}
|
|
1747
|
+
if (units.length >= CHUNK_SIZE) {
|
|
1748
|
+
result += String.fromCharCode(...units);
|
|
1749
|
+
units.length = 0;
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
if (units.length > 0) {
|
|
1753
|
+
result += String.fromCharCode(...units);
|
|
1754
|
+
}
|
|
1755
|
+
return result;
|
|
1756
|
+
}
|
|
1757
|
+
function utf8DecodeTD(bytes, inputOffset, byteLength) {
|
|
1758
|
+
const stringBytes = bytes.subarray(inputOffset, inputOffset + byteLength);
|
|
1759
|
+
return sharedTextDecoder.decode(stringBytes);
|
|
1760
|
+
}
|
|
1761
|
+
function utf8Decode(bytes, inputOffset, byteLength) {
|
|
1762
|
+
if (byteLength > TEXT_DECODER_THRESHOLD) {
|
|
1763
|
+
return utf8DecodeTD(bytes, inputOffset, byteLength);
|
|
1764
|
+
} else {
|
|
1765
|
+
return utf8DecodeJs(bytes, inputOffset, byteLength);
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
var sharedTextEncoder, TEXT_ENCODER_THRESHOLD, CHUNK_SIZE, sharedTextDecoder, TEXT_DECODER_THRESHOLD;
|
|
1769
|
+
var init_utf8 = __esm({
|
|
1770
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/utf8.mjs"() {
|
|
1771
|
+
"use strict";
|
|
1772
|
+
sharedTextEncoder = new TextEncoder();
|
|
1773
|
+
TEXT_ENCODER_THRESHOLD = 50;
|
|
1774
|
+
CHUNK_SIZE = 4096;
|
|
1775
|
+
sharedTextDecoder = new TextDecoder();
|
|
1776
|
+
TEXT_DECODER_THRESHOLD = 200;
|
|
1777
|
+
}
|
|
1778
|
+
});
|
|
1779
|
+
|
|
1780
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/ExtData.mjs
|
|
1781
|
+
var ExtData;
|
|
1782
|
+
var init_ExtData = __esm({
|
|
1783
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/ExtData.mjs"() {
|
|
1784
|
+
"use strict";
|
|
1785
|
+
ExtData = class {
|
|
1786
|
+
type;
|
|
1787
|
+
data;
|
|
1788
|
+
constructor(type, data) {
|
|
1789
|
+
this.type = type;
|
|
1790
|
+
this.data = data;
|
|
1791
|
+
}
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
});
|
|
1795
|
+
|
|
1796
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/DecodeError.mjs
|
|
1797
|
+
var DecodeError;
|
|
1798
|
+
var init_DecodeError = __esm({
|
|
1799
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/DecodeError.mjs"() {
|
|
1800
|
+
"use strict";
|
|
1801
|
+
DecodeError = class _DecodeError extends Error {
|
|
1802
|
+
constructor(message) {
|
|
1803
|
+
super(message);
|
|
1804
|
+
const proto = Object.create(_DecodeError.prototype);
|
|
1805
|
+
Object.setPrototypeOf(this, proto);
|
|
1806
|
+
Object.defineProperty(this, "name", {
|
|
1807
|
+
configurable: true,
|
|
1808
|
+
enumerable: false,
|
|
1809
|
+
value: _DecodeError.name
|
|
1810
|
+
});
|
|
1811
|
+
}
|
|
1812
|
+
};
|
|
1813
|
+
}
|
|
1814
|
+
});
|
|
1815
|
+
|
|
1816
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/int.mjs
|
|
1817
|
+
function setUint64(view, offset, value) {
|
|
1818
|
+
const high = value / 4294967296;
|
|
1819
|
+
const low = value;
|
|
1820
|
+
view.setUint32(offset, high);
|
|
1821
|
+
view.setUint32(offset + 4, low);
|
|
1822
|
+
}
|
|
1823
|
+
function setInt64(view, offset, value) {
|
|
1824
|
+
const high = Math.floor(value / 4294967296);
|
|
1825
|
+
const low = value;
|
|
1826
|
+
view.setUint32(offset, high);
|
|
1827
|
+
view.setUint32(offset + 4, low);
|
|
1828
|
+
}
|
|
1829
|
+
function getInt64(view, offset) {
|
|
1830
|
+
const high = view.getInt32(offset);
|
|
1831
|
+
const low = view.getUint32(offset + 4);
|
|
1832
|
+
return high * 4294967296 + low;
|
|
1833
|
+
}
|
|
1834
|
+
function getUint64(view, offset) {
|
|
1835
|
+
const high = view.getUint32(offset);
|
|
1836
|
+
const low = view.getUint32(offset + 4);
|
|
1837
|
+
return high * 4294967296 + low;
|
|
1838
|
+
}
|
|
1839
|
+
var UINT32_MAX;
|
|
1840
|
+
var init_int = __esm({
|
|
1841
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/int.mjs"() {
|
|
1842
|
+
"use strict";
|
|
1843
|
+
UINT32_MAX = 4294967295;
|
|
1844
|
+
}
|
|
1845
|
+
});
|
|
1846
|
+
|
|
1847
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/timestamp.mjs
|
|
1848
|
+
function encodeTimeSpecToTimestamp({ sec, nsec }) {
|
|
1849
|
+
if (sec >= 0 && nsec >= 0 && sec <= TIMESTAMP64_MAX_SEC) {
|
|
1850
|
+
if (nsec === 0 && sec <= TIMESTAMP32_MAX_SEC) {
|
|
1851
|
+
const rv = new Uint8Array(4);
|
|
1852
|
+
const view = new DataView(rv.buffer);
|
|
1853
|
+
view.setUint32(0, sec);
|
|
1854
|
+
return rv;
|
|
1855
|
+
} else {
|
|
1856
|
+
const secHigh = sec / 4294967296;
|
|
1857
|
+
const secLow = sec & 4294967295;
|
|
1858
|
+
const rv = new Uint8Array(8);
|
|
1859
|
+
const view = new DataView(rv.buffer);
|
|
1860
|
+
view.setUint32(0, nsec << 2 | secHigh & 3);
|
|
1861
|
+
view.setUint32(4, secLow);
|
|
1862
|
+
return rv;
|
|
1863
|
+
}
|
|
1864
|
+
} else {
|
|
1865
|
+
const rv = new Uint8Array(12);
|
|
1866
|
+
const view = new DataView(rv.buffer);
|
|
1867
|
+
view.setUint32(0, nsec);
|
|
1868
|
+
setInt64(view, 4, sec);
|
|
1869
|
+
return rv;
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
function encodeDateToTimeSpec(date) {
|
|
1873
|
+
const msec = date.getTime();
|
|
1874
|
+
const sec = Math.floor(msec / 1e3);
|
|
1875
|
+
const nsec = (msec - sec * 1e3) * 1e6;
|
|
1876
|
+
const nsecInSec = Math.floor(nsec / 1e9);
|
|
1877
|
+
return {
|
|
1878
|
+
sec: sec + nsecInSec,
|
|
1879
|
+
nsec: nsec - nsecInSec * 1e9
|
|
1880
|
+
};
|
|
1881
|
+
}
|
|
1882
|
+
function encodeTimestampExtension(object) {
|
|
1883
|
+
if (object instanceof Date) {
|
|
1884
|
+
const timeSpec = encodeDateToTimeSpec(object);
|
|
1885
|
+
return encodeTimeSpecToTimestamp(timeSpec);
|
|
1886
|
+
} else {
|
|
1887
|
+
return null;
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
function decodeTimestampToTimeSpec(data) {
|
|
1891
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
1892
|
+
switch (data.byteLength) {
|
|
1893
|
+
case 4: {
|
|
1894
|
+
const sec = view.getUint32(0);
|
|
1895
|
+
const nsec = 0;
|
|
1896
|
+
return { sec, nsec };
|
|
1897
|
+
}
|
|
1898
|
+
case 8: {
|
|
1899
|
+
const nsec30AndSecHigh2 = view.getUint32(0);
|
|
1900
|
+
const secLow32 = view.getUint32(4);
|
|
1901
|
+
const sec = (nsec30AndSecHigh2 & 3) * 4294967296 + secLow32;
|
|
1902
|
+
const nsec = nsec30AndSecHigh2 >>> 2;
|
|
1903
|
+
return { sec, nsec };
|
|
1904
|
+
}
|
|
1905
|
+
case 12: {
|
|
1906
|
+
const sec = getInt64(view, 4);
|
|
1907
|
+
const nsec = view.getUint32(0);
|
|
1908
|
+
return { sec, nsec };
|
|
1909
|
+
}
|
|
1910
|
+
default:
|
|
1911
|
+
throw new DecodeError(`Unrecognized data size for timestamp (expected 4, 8, or 12): ${data.length}`);
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
function decodeTimestampExtension(data) {
|
|
1915
|
+
const timeSpec = decodeTimestampToTimeSpec(data);
|
|
1916
|
+
return new Date(timeSpec.sec * 1e3 + timeSpec.nsec / 1e6);
|
|
1917
|
+
}
|
|
1918
|
+
var EXT_TIMESTAMP, TIMESTAMP32_MAX_SEC, TIMESTAMP64_MAX_SEC, timestampExtension;
|
|
1919
|
+
var init_timestamp = __esm({
|
|
1920
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/timestamp.mjs"() {
|
|
1921
|
+
"use strict";
|
|
1922
|
+
init_DecodeError();
|
|
1923
|
+
init_int();
|
|
1924
|
+
EXT_TIMESTAMP = -1;
|
|
1925
|
+
TIMESTAMP32_MAX_SEC = 4294967296 - 1;
|
|
1926
|
+
TIMESTAMP64_MAX_SEC = 17179869184 - 1;
|
|
1927
|
+
timestampExtension = {
|
|
1928
|
+
type: EXT_TIMESTAMP,
|
|
1929
|
+
encode: encodeTimestampExtension,
|
|
1930
|
+
decode: decodeTimestampExtension
|
|
1931
|
+
};
|
|
1932
|
+
}
|
|
1933
|
+
});
|
|
1934
|
+
|
|
1935
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/ExtensionCodec.mjs
|
|
1936
|
+
var ExtensionCodec;
|
|
1937
|
+
var init_ExtensionCodec = __esm({
|
|
1938
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/ExtensionCodec.mjs"() {
|
|
1939
|
+
"use strict";
|
|
1940
|
+
init_ExtData();
|
|
1941
|
+
init_timestamp();
|
|
1942
|
+
ExtensionCodec = class _ExtensionCodec {
|
|
1943
|
+
static defaultCodec = new _ExtensionCodec();
|
|
1944
|
+
// ensures ExtensionCodecType<X> matches ExtensionCodec<X>
|
|
1945
|
+
// this will make type errors a lot more clear
|
|
1946
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention
|
|
1947
|
+
__brand;
|
|
1948
|
+
// built-in extensions
|
|
1949
|
+
builtInEncoders = [];
|
|
1950
|
+
builtInDecoders = [];
|
|
1951
|
+
// custom extensions
|
|
1952
|
+
encoders = [];
|
|
1953
|
+
decoders = [];
|
|
1954
|
+
constructor() {
|
|
1955
|
+
this.register(timestampExtension);
|
|
1956
|
+
}
|
|
1957
|
+
register({ type, encode: encode2, decode: decode2 }) {
|
|
1958
|
+
if (type >= 0) {
|
|
1959
|
+
this.encoders[type] = encode2;
|
|
1960
|
+
this.decoders[type] = decode2;
|
|
1961
|
+
} else {
|
|
1962
|
+
const index = -1 - type;
|
|
1963
|
+
this.builtInEncoders[index] = encode2;
|
|
1964
|
+
this.builtInDecoders[index] = decode2;
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1967
|
+
tryToEncode(object, context) {
|
|
1968
|
+
for (let i = 0; i < this.builtInEncoders.length; i++) {
|
|
1969
|
+
const encodeExt = this.builtInEncoders[i];
|
|
1970
|
+
if (encodeExt != null) {
|
|
1971
|
+
const data = encodeExt(object, context);
|
|
1972
|
+
if (data != null) {
|
|
1973
|
+
const type = -1 - i;
|
|
1974
|
+
return new ExtData(type, data);
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
for (let i = 0; i < this.encoders.length; i++) {
|
|
1979
|
+
const encodeExt = this.encoders[i];
|
|
1980
|
+
if (encodeExt != null) {
|
|
1981
|
+
const data = encodeExt(object, context);
|
|
1982
|
+
if (data != null) {
|
|
1983
|
+
const type = i;
|
|
1984
|
+
return new ExtData(type, data);
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
if (object instanceof ExtData) {
|
|
1989
|
+
return object;
|
|
1990
|
+
}
|
|
1991
|
+
return null;
|
|
1992
|
+
}
|
|
1993
|
+
decode(data, type, context) {
|
|
1994
|
+
const decodeExt = type < 0 ? this.builtInDecoders[-1 - type] : this.decoders[type];
|
|
1995
|
+
if (decodeExt) {
|
|
1996
|
+
return decodeExt(data, type, context);
|
|
1997
|
+
} else {
|
|
1998
|
+
return new ExtData(type, data);
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
};
|
|
2002
|
+
}
|
|
2003
|
+
});
|
|
2004
|
+
|
|
2005
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/typedArrays.mjs
|
|
2006
|
+
function isArrayBufferLike(buffer) {
|
|
2007
|
+
return buffer instanceof ArrayBuffer || typeof SharedArrayBuffer !== "undefined" && buffer instanceof SharedArrayBuffer;
|
|
2008
|
+
}
|
|
2009
|
+
function ensureUint8Array(buffer) {
|
|
2010
|
+
if (buffer instanceof Uint8Array) {
|
|
2011
|
+
return buffer;
|
|
2012
|
+
} else if (ArrayBuffer.isView(buffer)) {
|
|
2013
|
+
return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
|
|
2014
|
+
} else if (isArrayBufferLike(buffer)) {
|
|
2015
|
+
return new Uint8Array(buffer);
|
|
2016
|
+
} else {
|
|
2017
|
+
return Uint8Array.from(buffer);
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
var init_typedArrays = __esm({
|
|
2021
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/typedArrays.mjs"() {
|
|
2022
|
+
"use strict";
|
|
2023
|
+
}
|
|
2024
|
+
});
|
|
2025
|
+
|
|
2026
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/Encoder.mjs
|
|
2027
|
+
var DEFAULT_MAX_DEPTH, DEFAULT_INITIAL_BUFFER_SIZE, Encoder;
|
|
2028
|
+
var init_Encoder = __esm({
|
|
2029
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/Encoder.mjs"() {
|
|
2030
|
+
"use strict";
|
|
2031
|
+
init_utf8();
|
|
2032
|
+
init_ExtensionCodec();
|
|
2033
|
+
init_int();
|
|
2034
|
+
init_typedArrays();
|
|
2035
|
+
DEFAULT_MAX_DEPTH = 100;
|
|
2036
|
+
DEFAULT_INITIAL_BUFFER_SIZE = 2048;
|
|
2037
|
+
Encoder = class _Encoder {
|
|
2038
|
+
extensionCodec;
|
|
2039
|
+
context;
|
|
2040
|
+
useBigInt64;
|
|
2041
|
+
maxDepth;
|
|
2042
|
+
initialBufferSize;
|
|
2043
|
+
sortKeys;
|
|
2044
|
+
forceFloat32;
|
|
2045
|
+
ignoreUndefined;
|
|
2046
|
+
forceIntegerToFloat;
|
|
2047
|
+
pos;
|
|
2048
|
+
view;
|
|
2049
|
+
bytes;
|
|
2050
|
+
entered = false;
|
|
2051
|
+
constructor(options) {
|
|
2052
|
+
this.extensionCodec = options?.extensionCodec ?? ExtensionCodec.defaultCodec;
|
|
2053
|
+
this.context = options?.context;
|
|
2054
|
+
this.useBigInt64 = options?.useBigInt64 ?? false;
|
|
2055
|
+
this.maxDepth = options?.maxDepth ?? DEFAULT_MAX_DEPTH;
|
|
2056
|
+
this.initialBufferSize = options?.initialBufferSize ?? DEFAULT_INITIAL_BUFFER_SIZE;
|
|
2057
|
+
this.sortKeys = options?.sortKeys ?? false;
|
|
2058
|
+
this.forceFloat32 = options?.forceFloat32 ?? false;
|
|
2059
|
+
this.ignoreUndefined = options?.ignoreUndefined ?? false;
|
|
2060
|
+
this.forceIntegerToFloat = options?.forceIntegerToFloat ?? false;
|
|
2061
|
+
this.pos = 0;
|
|
2062
|
+
this.view = new DataView(new ArrayBuffer(this.initialBufferSize));
|
|
2063
|
+
this.bytes = new Uint8Array(this.view.buffer);
|
|
2064
|
+
}
|
|
2065
|
+
clone() {
|
|
2066
|
+
return new _Encoder({
|
|
2067
|
+
extensionCodec: this.extensionCodec,
|
|
2068
|
+
context: this.context,
|
|
2069
|
+
useBigInt64: this.useBigInt64,
|
|
2070
|
+
maxDepth: this.maxDepth,
|
|
2071
|
+
initialBufferSize: this.initialBufferSize,
|
|
2072
|
+
sortKeys: this.sortKeys,
|
|
2073
|
+
forceFloat32: this.forceFloat32,
|
|
2074
|
+
ignoreUndefined: this.ignoreUndefined,
|
|
2075
|
+
forceIntegerToFloat: this.forceIntegerToFloat
|
|
2076
|
+
});
|
|
2077
|
+
}
|
|
2078
|
+
reinitializeState() {
|
|
2079
|
+
this.pos = 0;
|
|
2080
|
+
}
|
|
2081
|
+
/**
|
|
2082
|
+
* This is almost equivalent to {@link Encoder#encode}, but it returns an reference of the encoder's internal buffer and thus much faster than {@link Encoder#encode}.
|
|
2083
|
+
*
|
|
2084
|
+
* @returns Encodes the object and returns a shared reference the encoder's internal buffer.
|
|
2085
|
+
*/
|
|
2086
|
+
encodeSharedRef(object) {
|
|
2087
|
+
if (this.entered) {
|
|
2088
|
+
const instance = this.clone();
|
|
2089
|
+
return instance.encodeSharedRef(object);
|
|
2090
|
+
}
|
|
2091
|
+
try {
|
|
2092
|
+
this.entered = true;
|
|
2093
|
+
this.reinitializeState();
|
|
2094
|
+
this.doEncode(object, 1);
|
|
2095
|
+
return this.bytes.subarray(0, this.pos);
|
|
2096
|
+
} finally {
|
|
2097
|
+
this.entered = false;
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
/**
|
|
2101
|
+
* @returns Encodes the object and returns a copy of the encoder's internal buffer.
|
|
2102
|
+
*/
|
|
2103
|
+
encode(object) {
|
|
2104
|
+
if (this.entered) {
|
|
2105
|
+
const instance = this.clone();
|
|
2106
|
+
return instance.encode(object);
|
|
2107
|
+
}
|
|
2108
|
+
try {
|
|
2109
|
+
this.entered = true;
|
|
2110
|
+
this.reinitializeState();
|
|
2111
|
+
this.doEncode(object, 1);
|
|
2112
|
+
return this.bytes.slice(0, this.pos);
|
|
2113
|
+
} finally {
|
|
2114
|
+
this.entered = false;
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
doEncode(object, depth) {
|
|
2118
|
+
if (depth > this.maxDepth) {
|
|
2119
|
+
throw new Error(`Too deep objects in depth ${depth}`);
|
|
2120
|
+
}
|
|
2121
|
+
if (object == null) {
|
|
2122
|
+
this.encodeNil();
|
|
2123
|
+
} else if (typeof object === "boolean") {
|
|
2124
|
+
this.encodeBoolean(object);
|
|
2125
|
+
} else if (typeof object === "number") {
|
|
2126
|
+
if (!this.forceIntegerToFloat) {
|
|
2127
|
+
this.encodeNumber(object);
|
|
2128
|
+
} else {
|
|
2129
|
+
this.encodeNumberAsFloat(object);
|
|
2130
|
+
}
|
|
2131
|
+
} else if (typeof object === "string") {
|
|
2132
|
+
this.encodeString(object);
|
|
2133
|
+
} else if (this.useBigInt64 && typeof object === "bigint") {
|
|
2134
|
+
this.encodeBigInt64(object);
|
|
2135
|
+
} else {
|
|
2136
|
+
this.encodeObject(object, depth);
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
ensureBufferSizeToWrite(sizeToWrite) {
|
|
2140
|
+
const requiredSize = this.pos + sizeToWrite;
|
|
2141
|
+
if (this.view.byteLength < requiredSize) {
|
|
2142
|
+
this.resizeBuffer(requiredSize * 2);
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
resizeBuffer(newSize) {
|
|
2146
|
+
const newBuffer = new ArrayBuffer(newSize);
|
|
2147
|
+
const newBytes = new Uint8Array(newBuffer);
|
|
2148
|
+
const newView = new DataView(newBuffer);
|
|
2149
|
+
newBytes.set(this.bytes);
|
|
2150
|
+
this.view = newView;
|
|
2151
|
+
this.bytes = newBytes;
|
|
2152
|
+
}
|
|
2153
|
+
encodeNil() {
|
|
2154
|
+
this.writeU8(192);
|
|
2155
|
+
}
|
|
2156
|
+
encodeBoolean(object) {
|
|
2157
|
+
if (object === false) {
|
|
2158
|
+
this.writeU8(194);
|
|
2159
|
+
} else {
|
|
2160
|
+
this.writeU8(195);
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
encodeNumber(object) {
|
|
2164
|
+
if (!this.forceIntegerToFloat && Number.isSafeInteger(object)) {
|
|
2165
|
+
if (object >= 0) {
|
|
2166
|
+
if (object < 128) {
|
|
2167
|
+
this.writeU8(object);
|
|
2168
|
+
} else if (object < 256) {
|
|
2169
|
+
this.writeU8(204);
|
|
2170
|
+
this.writeU8(object);
|
|
2171
|
+
} else if (object < 65536) {
|
|
2172
|
+
this.writeU8(205);
|
|
2173
|
+
this.writeU16(object);
|
|
2174
|
+
} else if (object < 4294967296) {
|
|
2175
|
+
this.writeU8(206);
|
|
2176
|
+
this.writeU32(object);
|
|
2177
|
+
} else if (!this.useBigInt64) {
|
|
2178
|
+
this.writeU8(207);
|
|
2179
|
+
this.writeU64(object);
|
|
2180
|
+
} else {
|
|
2181
|
+
this.encodeNumberAsFloat(object);
|
|
2182
|
+
}
|
|
2183
|
+
} else {
|
|
2184
|
+
if (object >= -32) {
|
|
2185
|
+
this.writeU8(224 | object + 32);
|
|
2186
|
+
} else if (object >= -128) {
|
|
2187
|
+
this.writeU8(208);
|
|
2188
|
+
this.writeI8(object);
|
|
2189
|
+
} else if (object >= -32768) {
|
|
2190
|
+
this.writeU8(209);
|
|
2191
|
+
this.writeI16(object);
|
|
2192
|
+
} else if (object >= -2147483648) {
|
|
2193
|
+
this.writeU8(210);
|
|
2194
|
+
this.writeI32(object);
|
|
2195
|
+
} else if (!this.useBigInt64) {
|
|
2196
|
+
this.writeU8(211);
|
|
2197
|
+
this.writeI64(object);
|
|
2198
|
+
} else {
|
|
2199
|
+
this.encodeNumberAsFloat(object);
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
} else {
|
|
2203
|
+
this.encodeNumberAsFloat(object);
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
encodeNumberAsFloat(object) {
|
|
2207
|
+
if (this.forceFloat32) {
|
|
2208
|
+
this.writeU8(202);
|
|
2209
|
+
this.writeF32(object);
|
|
2210
|
+
} else {
|
|
2211
|
+
this.writeU8(203);
|
|
2212
|
+
this.writeF64(object);
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2215
|
+
encodeBigInt64(object) {
|
|
2216
|
+
if (object >= BigInt(0)) {
|
|
2217
|
+
this.writeU8(207);
|
|
2218
|
+
this.writeBigUint64(object);
|
|
2219
|
+
} else {
|
|
2220
|
+
this.writeU8(211);
|
|
2221
|
+
this.writeBigInt64(object);
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
writeStringHeader(byteLength) {
|
|
2225
|
+
if (byteLength < 32) {
|
|
2226
|
+
this.writeU8(160 + byteLength);
|
|
2227
|
+
} else if (byteLength < 256) {
|
|
2228
|
+
this.writeU8(217);
|
|
2229
|
+
this.writeU8(byteLength);
|
|
2230
|
+
} else if (byteLength < 65536) {
|
|
2231
|
+
this.writeU8(218);
|
|
2232
|
+
this.writeU16(byteLength);
|
|
2233
|
+
} else if (byteLength < 4294967296) {
|
|
2234
|
+
this.writeU8(219);
|
|
2235
|
+
this.writeU32(byteLength);
|
|
2236
|
+
} else {
|
|
2237
|
+
throw new Error(`Too long string: ${byteLength} bytes in UTF-8`);
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
encodeString(object) {
|
|
2241
|
+
const maxHeaderSize = 1 + 4;
|
|
2242
|
+
const byteLength = utf8Count(object);
|
|
2243
|
+
this.ensureBufferSizeToWrite(maxHeaderSize + byteLength);
|
|
2244
|
+
this.writeStringHeader(byteLength);
|
|
2245
|
+
utf8Encode(object, this.bytes, this.pos);
|
|
2246
|
+
this.pos += byteLength;
|
|
2247
|
+
}
|
|
2248
|
+
encodeObject(object, depth) {
|
|
2249
|
+
const ext = this.extensionCodec.tryToEncode(object, this.context);
|
|
2250
|
+
if (ext != null) {
|
|
2251
|
+
this.encodeExtension(ext);
|
|
2252
|
+
} else if (Array.isArray(object)) {
|
|
2253
|
+
this.encodeArray(object, depth);
|
|
2254
|
+
} else if (ArrayBuffer.isView(object)) {
|
|
2255
|
+
this.encodeBinary(object);
|
|
2256
|
+
} else if (typeof object === "object") {
|
|
2257
|
+
this.encodeMap(object, depth);
|
|
2258
|
+
} else {
|
|
2259
|
+
throw new Error(`Unrecognized object: ${Object.prototype.toString.apply(object)}`);
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
encodeBinary(object) {
|
|
2263
|
+
const size = object.byteLength;
|
|
2264
|
+
if (size < 256) {
|
|
2265
|
+
this.writeU8(196);
|
|
2266
|
+
this.writeU8(size);
|
|
2267
|
+
} else if (size < 65536) {
|
|
2268
|
+
this.writeU8(197);
|
|
2269
|
+
this.writeU16(size);
|
|
2270
|
+
} else if (size < 4294967296) {
|
|
2271
|
+
this.writeU8(198);
|
|
2272
|
+
this.writeU32(size);
|
|
2273
|
+
} else {
|
|
2274
|
+
throw new Error(`Too large binary: ${size}`);
|
|
2275
|
+
}
|
|
2276
|
+
const bytes = ensureUint8Array(object);
|
|
2277
|
+
this.writeU8a(bytes);
|
|
2278
|
+
}
|
|
2279
|
+
encodeArray(object, depth) {
|
|
2280
|
+
const size = object.length;
|
|
2281
|
+
if (size < 16) {
|
|
2282
|
+
this.writeU8(144 + size);
|
|
2283
|
+
} else if (size < 65536) {
|
|
2284
|
+
this.writeU8(220);
|
|
2285
|
+
this.writeU16(size);
|
|
2286
|
+
} else if (size < 4294967296) {
|
|
2287
|
+
this.writeU8(221);
|
|
2288
|
+
this.writeU32(size);
|
|
2289
|
+
} else {
|
|
2290
|
+
throw new Error(`Too large array: ${size}`);
|
|
2291
|
+
}
|
|
2292
|
+
for (const item of object) {
|
|
2293
|
+
this.doEncode(item, depth + 1);
|
|
2294
|
+
}
|
|
2295
|
+
}
|
|
2296
|
+
countWithoutUndefined(object, keys) {
|
|
2297
|
+
let count = 0;
|
|
2298
|
+
for (const key of keys) {
|
|
2299
|
+
if (object[key] !== void 0) {
|
|
2300
|
+
count++;
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2303
|
+
return count;
|
|
2304
|
+
}
|
|
2305
|
+
encodeMap(object, depth) {
|
|
2306
|
+
const keys = Object.keys(object);
|
|
2307
|
+
if (this.sortKeys) {
|
|
2308
|
+
keys.sort();
|
|
2309
|
+
}
|
|
2310
|
+
const size = this.ignoreUndefined ? this.countWithoutUndefined(object, keys) : keys.length;
|
|
2311
|
+
if (size < 16) {
|
|
2312
|
+
this.writeU8(128 + size);
|
|
2313
|
+
} else if (size < 65536) {
|
|
2314
|
+
this.writeU8(222);
|
|
2315
|
+
this.writeU16(size);
|
|
2316
|
+
} else if (size < 4294967296) {
|
|
2317
|
+
this.writeU8(223);
|
|
2318
|
+
this.writeU32(size);
|
|
2319
|
+
} else {
|
|
2320
|
+
throw new Error(`Too large map object: ${size}`);
|
|
2321
|
+
}
|
|
2322
|
+
for (const key of keys) {
|
|
2323
|
+
const value = object[key];
|
|
2324
|
+
if (!(this.ignoreUndefined && value === void 0)) {
|
|
2325
|
+
this.encodeString(key);
|
|
2326
|
+
this.doEncode(value, depth + 1);
|
|
2327
|
+
}
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
encodeExtension(ext) {
|
|
2331
|
+
if (typeof ext.data === "function") {
|
|
2332
|
+
const data = ext.data(this.pos + 6);
|
|
2333
|
+
const size2 = data.length;
|
|
2334
|
+
if (size2 >= 4294967296) {
|
|
2335
|
+
throw new Error(`Too large extension object: ${size2}`);
|
|
2336
|
+
}
|
|
2337
|
+
this.writeU8(201);
|
|
2338
|
+
this.writeU32(size2);
|
|
2339
|
+
this.writeI8(ext.type);
|
|
2340
|
+
this.writeU8a(data);
|
|
2341
|
+
return;
|
|
2342
|
+
}
|
|
2343
|
+
const size = ext.data.length;
|
|
2344
|
+
if (size === 1) {
|
|
2345
|
+
this.writeU8(212);
|
|
2346
|
+
} else if (size === 2) {
|
|
2347
|
+
this.writeU8(213);
|
|
2348
|
+
} else if (size === 4) {
|
|
2349
|
+
this.writeU8(214);
|
|
2350
|
+
} else if (size === 8) {
|
|
2351
|
+
this.writeU8(215);
|
|
2352
|
+
} else if (size === 16) {
|
|
2353
|
+
this.writeU8(216);
|
|
2354
|
+
} else if (size < 256) {
|
|
2355
|
+
this.writeU8(199);
|
|
2356
|
+
this.writeU8(size);
|
|
2357
|
+
} else if (size < 65536) {
|
|
2358
|
+
this.writeU8(200);
|
|
2359
|
+
this.writeU16(size);
|
|
2360
|
+
} else if (size < 4294967296) {
|
|
2361
|
+
this.writeU8(201);
|
|
2362
|
+
this.writeU32(size);
|
|
2363
|
+
} else {
|
|
2364
|
+
throw new Error(`Too large extension object: ${size}`);
|
|
2365
|
+
}
|
|
2366
|
+
this.writeI8(ext.type);
|
|
2367
|
+
this.writeU8a(ext.data);
|
|
2368
|
+
}
|
|
2369
|
+
writeU8(value) {
|
|
2370
|
+
this.ensureBufferSizeToWrite(1);
|
|
2371
|
+
this.view.setUint8(this.pos, value);
|
|
2372
|
+
this.pos++;
|
|
2373
|
+
}
|
|
2374
|
+
writeU8a(values) {
|
|
2375
|
+
const size = values.length;
|
|
2376
|
+
this.ensureBufferSizeToWrite(size);
|
|
2377
|
+
this.bytes.set(values, this.pos);
|
|
2378
|
+
this.pos += size;
|
|
2379
|
+
}
|
|
2380
|
+
writeI8(value) {
|
|
2381
|
+
this.ensureBufferSizeToWrite(1);
|
|
2382
|
+
this.view.setInt8(this.pos, value);
|
|
2383
|
+
this.pos++;
|
|
2384
|
+
}
|
|
2385
|
+
writeU16(value) {
|
|
2386
|
+
this.ensureBufferSizeToWrite(2);
|
|
2387
|
+
this.view.setUint16(this.pos, value);
|
|
2388
|
+
this.pos += 2;
|
|
2389
|
+
}
|
|
2390
|
+
writeI16(value) {
|
|
2391
|
+
this.ensureBufferSizeToWrite(2);
|
|
2392
|
+
this.view.setInt16(this.pos, value);
|
|
2393
|
+
this.pos += 2;
|
|
2394
|
+
}
|
|
2395
|
+
writeU32(value) {
|
|
2396
|
+
this.ensureBufferSizeToWrite(4);
|
|
2397
|
+
this.view.setUint32(this.pos, value);
|
|
2398
|
+
this.pos += 4;
|
|
2399
|
+
}
|
|
2400
|
+
writeI32(value) {
|
|
2401
|
+
this.ensureBufferSizeToWrite(4);
|
|
2402
|
+
this.view.setInt32(this.pos, value);
|
|
2403
|
+
this.pos += 4;
|
|
2404
|
+
}
|
|
2405
|
+
writeF32(value) {
|
|
2406
|
+
this.ensureBufferSizeToWrite(4);
|
|
2407
|
+
this.view.setFloat32(this.pos, value);
|
|
2408
|
+
this.pos += 4;
|
|
2409
|
+
}
|
|
2410
|
+
writeF64(value) {
|
|
2411
|
+
this.ensureBufferSizeToWrite(8);
|
|
2412
|
+
this.view.setFloat64(this.pos, value);
|
|
2413
|
+
this.pos += 8;
|
|
2414
|
+
}
|
|
2415
|
+
writeU64(value) {
|
|
2416
|
+
this.ensureBufferSizeToWrite(8);
|
|
2417
|
+
setUint64(this.view, this.pos, value);
|
|
2418
|
+
this.pos += 8;
|
|
2419
|
+
}
|
|
2420
|
+
writeI64(value) {
|
|
2421
|
+
this.ensureBufferSizeToWrite(8);
|
|
2422
|
+
setInt64(this.view, this.pos, value);
|
|
2423
|
+
this.pos += 8;
|
|
2424
|
+
}
|
|
2425
|
+
writeBigUint64(value) {
|
|
2426
|
+
this.ensureBufferSizeToWrite(8);
|
|
2427
|
+
this.view.setBigUint64(this.pos, value);
|
|
2428
|
+
this.pos += 8;
|
|
2429
|
+
}
|
|
2430
|
+
writeBigInt64(value) {
|
|
2431
|
+
this.ensureBufferSizeToWrite(8);
|
|
2432
|
+
this.view.setBigInt64(this.pos, value);
|
|
2433
|
+
this.pos += 8;
|
|
2434
|
+
}
|
|
2435
|
+
};
|
|
2436
|
+
}
|
|
2437
|
+
});
|
|
2438
|
+
|
|
2439
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/encode.mjs
|
|
2440
|
+
function encode(value, options) {
|
|
2441
|
+
const encoder = new Encoder(options);
|
|
2442
|
+
return encoder.encodeSharedRef(value);
|
|
2443
|
+
}
|
|
2444
|
+
var init_encode = __esm({
|
|
2445
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/encode.mjs"() {
|
|
2446
|
+
"use strict";
|
|
2447
|
+
init_Encoder();
|
|
2448
|
+
}
|
|
2449
|
+
});
|
|
2450
|
+
|
|
2451
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/prettyByte.mjs
|
|
2452
|
+
function prettyByte(byte) {
|
|
2453
|
+
return `${byte < 0 ? "-" : ""}0x${Math.abs(byte).toString(16).padStart(2, "0")}`;
|
|
2454
|
+
}
|
|
2455
|
+
var init_prettyByte = __esm({
|
|
2456
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/prettyByte.mjs"() {
|
|
2457
|
+
"use strict";
|
|
2458
|
+
}
|
|
2459
|
+
});
|
|
2460
|
+
|
|
2461
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/CachedKeyDecoder.mjs
|
|
2462
|
+
var DEFAULT_MAX_KEY_LENGTH, DEFAULT_MAX_LENGTH_PER_KEY, CachedKeyDecoder;
|
|
2463
|
+
var init_CachedKeyDecoder = __esm({
|
|
2464
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/CachedKeyDecoder.mjs"() {
|
|
2465
|
+
"use strict";
|
|
2466
|
+
init_utf8();
|
|
2467
|
+
DEFAULT_MAX_KEY_LENGTH = 16;
|
|
2468
|
+
DEFAULT_MAX_LENGTH_PER_KEY = 16;
|
|
2469
|
+
CachedKeyDecoder = class {
|
|
2470
|
+
hit = 0;
|
|
2471
|
+
miss = 0;
|
|
2472
|
+
caches;
|
|
2473
|
+
maxKeyLength;
|
|
2474
|
+
maxLengthPerKey;
|
|
2475
|
+
constructor(maxKeyLength = DEFAULT_MAX_KEY_LENGTH, maxLengthPerKey = DEFAULT_MAX_LENGTH_PER_KEY) {
|
|
2476
|
+
this.maxKeyLength = maxKeyLength;
|
|
2477
|
+
this.maxLengthPerKey = maxLengthPerKey;
|
|
2478
|
+
this.caches = [];
|
|
2479
|
+
for (let i = 0; i < this.maxKeyLength; i++) {
|
|
2480
|
+
this.caches.push([]);
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
canBeCached(byteLength) {
|
|
2484
|
+
return byteLength > 0 && byteLength <= this.maxKeyLength;
|
|
2485
|
+
}
|
|
2486
|
+
find(bytes, inputOffset, byteLength) {
|
|
2487
|
+
const records = this.caches[byteLength - 1];
|
|
2488
|
+
FIND_CHUNK: for (const record of records) {
|
|
2489
|
+
const recordBytes = record.bytes;
|
|
2490
|
+
for (let j = 0; j < byteLength; j++) {
|
|
2491
|
+
if (recordBytes[j] !== bytes[inputOffset + j]) {
|
|
2492
|
+
continue FIND_CHUNK;
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
return record.str;
|
|
2496
|
+
}
|
|
2497
|
+
return null;
|
|
2498
|
+
}
|
|
2499
|
+
store(bytes, value) {
|
|
2500
|
+
const records = this.caches[bytes.length - 1];
|
|
2501
|
+
const record = { bytes, str: value };
|
|
2502
|
+
if (records.length >= this.maxLengthPerKey) {
|
|
2503
|
+
records[Math.random() * records.length | 0] = record;
|
|
2504
|
+
} else {
|
|
2505
|
+
records.push(record);
|
|
2506
|
+
}
|
|
2507
|
+
}
|
|
2508
|
+
decode(bytes, inputOffset, byteLength) {
|
|
2509
|
+
const cachedValue = this.find(bytes, inputOffset, byteLength);
|
|
2510
|
+
if (cachedValue != null) {
|
|
2511
|
+
this.hit++;
|
|
2512
|
+
return cachedValue;
|
|
2513
|
+
}
|
|
2514
|
+
this.miss++;
|
|
2515
|
+
const str = utf8DecodeJs(bytes, inputOffset, byteLength);
|
|
2516
|
+
const slicedCopyOfBytes = Uint8Array.prototype.slice.call(bytes, inputOffset, inputOffset + byteLength);
|
|
2517
|
+
this.store(slicedCopyOfBytes, str);
|
|
2518
|
+
return str;
|
|
2519
|
+
}
|
|
2520
|
+
};
|
|
2521
|
+
}
|
|
2522
|
+
});
|
|
2523
|
+
|
|
2524
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/Decoder.mjs
|
|
2525
|
+
var STATE_ARRAY, STATE_MAP_KEY, STATE_MAP_VALUE, mapKeyConverter, StackPool, HEAD_BYTE_REQUIRED, EMPTY_VIEW, EMPTY_BYTES, MORE_DATA, sharedCachedKeyDecoder, Decoder;
|
|
2526
|
+
var init_Decoder = __esm({
|
|
2527
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/Decoder.mjs"() {
|
|
2528
|
+
"use strict";
|
|
2529
|
+
init_prettyByte();
|
|
2530
|
+
init_ExtensionCodec();
|
|
2531
|
+
init_int();
|
|
2532
|
+
init_utf8();
|
|
2533
|
+
init_typedArrays();
|
|
2534
|
+
init_CachedKeyDecoder();
|
|
2535
|
+
init_DecodeError();
|
|
2536
|
+
STATE_ARRAY = "array";
|
|
2537
|
+
STATE_MAP_KEY = "map_key";
|
|
2538
|
+
STATE_MAP_VALUE = "map_value";
|
|
2539
|
+
mapKeyConverter = (key) => {
|
|
2540
|
+
if (typeof key === "string" || typeof key === "number") {
|
|
2541
|
+
return key;
|
|
2542
|
+
}
|
|
2543
|
+
throw new DecodeError("The type of key must be string or number but " + typeof key);
|
|
2544
|
+
};
|
|
2545
|
+
StackPool = class {
|
|
2546
|
+
stack = [];
|
|
2547
|
+
stackHeadPosition = -1;
|
|
2548
|
+
get length() {
|
|
2549
|
+
return this.stackHeadPosition + 1;
|
|
2550
|
+
}
|
|
2551
|
+
top() {
|
|
2552
|
+
return this.stack[this.stackHeadPosition];
|
|
2553
|
+
}
|
|
2554
|
+
pushArrayState(size) {
|
|
2555
|
+
const state = this.getUninitializedStateFromPool();
|
|
2556
|
+
state.type = STATE_ARRAY;
|
|
2557
|
+
state.position = 0;
|
|
2558
|
+
state.size = size;
|
|
2559
|
+
state.array = new Array(size);
|
|
2560
|
+
}
|
|
2561
|
+
pushMapState(size) {
|
|
2562
|
+
const state = this.getUninitializedStateFromPool();
|
|
2563
|
+
state.type = STATE_MAP_KEY;
|
|
2564
|
+
state.readCount = 0;
|
|
2565
|
+
state.size = size;
|
|
2566
|
+
state.map = {};
|
|
2567
|
+
}
|
|
2568
|
+
getUninitializedStateFromPool() {
|
|
2569
|
+
this.stackHeadPosition++;
|
|
2570
|
+
if (this.stackHeadPosition === this.stack.length) {
|
|
2571
|
+
const partialState = {
|
|
2572
|
+
type: void 0,
|
|
2573
|
+
size: 0,
|
|
2574
|
+
array: void 0,
|
|
2575
|
+
position: 0,
|
|
2576
|
+
readCount: 0,
|
|
2577
|
+
map: void 0,
|
|
2578
|
+
key: null
|
|
2579
|
+
};
|
|
2580
|
+
this.stack.push(partialState);
|
|
2581
|
+
}
|
|
2582
|
+
return this.stack[this.stackHeadPosition];
|
|
2583
|
+
}
|
|
2584
|
+
release(state) {
|
|
2585
|
+
const topStackState = this.stack[this.stackHeadPosition];
|
|
2586
|
+
if (topStackState !== state) {
|
|
2587
|
+
throw new Error("Invalid stack state. Released state is not on top of the stack.");
|
|
2588
|
+
}
|
|
2589
|
+
if (state.type === STATE_ARRAY) {
|
|
2590
|
+
const partialState = state;
|
|
2591
|
+
partialState.size = 0;
|
|
2592
|
+
partialState.array = void 0;
|
|
2593
|
+
partialState.position = 0;
|
|
2594
|
+
partialState.type = void 0;
|
|
2595
|
+
}
|
|
2596
|
+
if (state.type === STATE_MAP_KEY || state.type === STATE_MAP_VALUE) {
|
|
2597
|
+
const partialState = state;
|
|
2598
|
+
partialState.size = 0;
|
|
2599
|
+
partialState.map = void 0;
|
|
2600
|
+
partialState.readCount = 0;
|
|
2601
|
+
partialState.type = void 0;
|
|
2602
|
+
}
|
|
2603
|
+
this.stackHeadPosition--;
|
|
2604
|
+
}
|
|
2605
|
+
reset() {
|
|
2606
|
+
this.stack.length = 0;
|
|
2607
|
+
this.stackHeadPosition = -1;
|
|
2608
|
+
}
|
|
2609
|
+
};
|
|
2610
|
+
HEAD_BYTE_REQUIRED = -1;
|
|
2611
|
+
EMPTY_VIEW = new DataView(new ArrayBuffer(0));
|
|
2612
|
+
EMPTY_BYTES = new Uint8Array(EMPTY_VIEW.buffer);
|
|
2613
|
+
try {
|
|
2614
|
+
EMPTY_VIEW.getInt8(0);
|
|
2615
|
+
} catch (e) {
|
|
2616
|
+
if (!(e instanceof RangeError)) {
|
|
2617
|
+
throw new Error("This module is not supported in the current JavaScript engine because DataView does not throw RangeError on out-of-bounds access");
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2620
|
+
MORE_DATA = new RangeError("Insufficient data");
|
|
2621
|
+
sharedCachedKeyDecoder = new CachedKeyDecoder();
|
|
2622
|
+
Decoder = class _Decoder {
|
|
2623
|
+
extensionCodec;
|
|
2624
|
+
context;
|
|
2625
|
+
useBigInt64;
|
|
2626
|
+
rawStrings;
|
|
2627
|
+
maxStrLength;
|
|
2628
|
+
maxBinLength;
|
|
2629
|
+
maxArrayLength;
|
|
2630
|
+
maxMapLength;
|
|
2631
|
+
maxExtLength;
|
|
2632
|
+
keyDecoder;
|
|
2633
|
+
mapKeyConverter;
|
|
2634
|
+
totalPos = 0;
|
|
2635
|
+
pos = 0;
|
|
2636
|
+
view = EMPTY_VIEW;
|
|
2637
|
+
bytes = EMPTY_BYTES;
|
|
2638
|
+
headByte = HEAD_BYTE_REQUIRED;
|
|
2639
|
+
stack = new StackPool();
|
|
2640
|
+
entered = false;
|
|
2641
|
+
constructor(options) {
|
|
2642
|
+
this.extensionCodec = options?.extensionCodec ?? ExtensionCodec.defaultCodec;
|
|
2643
|
+
this.context = options?.context;
|
|
2644
|
+
this.useBigInt64 = options?.useBigInt64 ?? false;
|
|
2645
|
+
this.rawStrings = options?.rawStrings ?? false;
|
|
2646
|
+
this.maxStrLength = options?.maxStrLength ?? UINT32_MAX;
|
|
2647
|
+
this.maxBinLength = options?.maxBinLength ?? UINT32_MAX;
|
|
2648
|
+
this.maxArrayLength = options?.maxArrayLength ?? UINT32_MAX;
|
|
2649
|
+
this.maxMapLength = options?.maxMapLength ?? UINT32_MAX;
|
|
2650
|
+
this.maxExtLength = options?.maxExtLength ?? UINT32_MAX;
|
|
2651
|
+
this.keyDecoder = options?.keyDecoder !== void 0 ? options.keyDecoder : sharedCachedKeyDecoder;
|
|
2652
|
+
this.mapKeyConverter = options?.mapKeyConverter ?? mapKeyConverter;
|
|
2653
|
+
}
|
|
2654
|
+
clone() {
|
|
2655
|
+
return new _Decoder({
|
|
2656
|
+
extensionCodec: this.extensionCodec,
|
|
2657
|
+
context: this.context,
|
|
2658
|
+
useBigInt64: this.useBigInt64,
|
|
2659
|
+
rawStrings: this.rawStrings,
|
|
2660
|
+
maxStrLength: this.maxStrLength,
|
|
2661
|
+
maxBinLength: this.maxBinLength,
|
|
2662
|
+
maxArrayLength: this.maxArrayLength,
|
|
2663
|
+
maxMapLength: this.maxMapLength,
|
|
2664
|
+
maxExtLength: this.maxExtLength,
|
|
2665
|
+
keyDecoder: this.keyDecoder
|
|
2666
|
+
});
|
|
2667
|
+
}
|
|
2668
|
+
reinitializeState() {
|
|
2669
|
+
this.totalPos = 0;
|
|
2670
|
+
this.headByte = HEAD_BYTE_REQUIRED;
|
|
2671
|
+
this.stack.reset();
|
|
2672
|
+
}
|
|
2673
|
+
setBuffer(buffer) {
|
|
2674
|
+
const bytes = ensureUint8Array(buffer);
|
|
2675
|
+
this.bytes = bytes;
|
|
2676
|
+
this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2677
|
+
this.pos = 0;
|
|
2678
|
+
}
|
|
2679
|
+
appendBuffer(buffer) {
|
|
2680
|
+
if (this.headByte === HEAD_BYTE_REQUIRED && !this.hasRemaining(1)) {
|
|
2681
|
+
this.setBuffer(buffer);
|
|
2682
|
+
} else {
|
|
2683
|
+
const remainingData = this.bytes.subarray(this.pos);
|
|
2684
|
+
const newData = ensureUint8Array(buffer);
|
|
2685
|
+
const newBuffer = new Uint8Array(remainingData.length + newData.length);
|
|
2686
|
+
newBuffer.set(remainingData);
|
|
2687
|
+
newBuffer.set(newData, remainingData.length);
|
|
2688
|
+
this.setBuffer(newBuffer);
|
|
2689
|
+
}
|
|
2690
|
+
}
|
|
2691
|
+
hasRemaining(size) {
|
|
2692
|
+
return this.view.byteLength - this.pos >= size;
|
|
2693
|
+
}
|
|
2694
|
+
createExtraByteError(posToShow) {
|
|
2695
|
+
const { view, pos } = this;
|
|
2696
|
+
return new RangeError(`Extra ${view.byteLength - pos} of ${view.byteLength} byte(s) found at buffer[${posToShow}]`);
|
|
2697
|
+
}
|
|
2698
|
+
/**
|
|
2699
|
+
* @throws {@link DecodeError}
|
|
2700
|
+
* @throws {@link RangeError}
|
|
2701
|
+
*/
|
|
2702
|
+
decode(buffer) {
|
|
2703
|
+
if (this.entered) {
|
|
2704
|
+
const instance = this.clone();
|
|
2705
|
+
return instance.decode(buffer);
|
|
2706
|
+
}
|
|
2707
|
+
try {
|
|
2708
|
+
this.entered = true;
|
|
2709
|
+
this.reinitializeState();
|
|
2710
|
+
this.setBuffer(buffer);
|
|
2711
|
+
const object = this.doDecodeSync();
|
|
2712
|
+
if (this.hasRemaining(1)) {
|
|
2713
|
+
throw this.createExtraByteError(this.pos);
|
|
2714
|
+
}
|
|
2715
|
+
return object;
|
|
2716
|
+
} finally {
|
|
2717
|
+
this.entered = false;
|
|
2718
|
+
}
|
|
2719
|
+
}
|
|
2720
|
+
*decodeMulti(buffer) {
|
|
2721
|
+
if (this.entered) {
|
|
2722
|
+
const instance = this.clone();
|
|
2723
|
+
yield* instance.decodeMulti(buffer);
|
|
2724
|
+
return;
|
|
2725
|
+
}
|
|
2726
|
+
try {
|
|
2727
|
+
this.entered = true;
|
|
2728
|
+
this.reinitializeState();
|
|
2729
|
+
this.setBuffer(buffer);
|
|
2730
|
+
while (this.hasRemaining(1)) {
|
|
2731
|
+
yield this.doDecodeSync();
|
|
2732
|
+
}
|
|
2733
|
+
} finally {
|
|
2734
|
+
this.entered = false;
|
|
2735
|
+
}
|
|
2736
|
+
}
|
|
2737
|
+
async decodeAsync(stream) {
|
|
2738
|
+
if (this.entered) {
|
|
2739
|
+
const instance = this.clone();
|
|
2740
|
+
return instance.decodeAsync(stream);
|
|
2741
|
+
}
|
|
2742
|
+
try {
|
|
2743
|
+
this.entered = true;
|
|
2744
|
+
let decoded = false;
|
|
2745
|
+
let object;
|
|
2746
|
+
for await (const buffer of stream) {
|
|
2747
|
+
if (decoded) {
|
|
2748
|
+
this.entered = false;
|
|
2749
|
+
throw this.createExtraByteError(this.totalPos);
|
|
2750
|
+
}
|
|
2751
|
+
this.appendBuffer(buffer);
|
|
2752
|
+
try {
|
|
2753
|
+
object = this.doDecodeSync();
|
|
2754
|
+
decoded = true;
|
|
2755
|
+
} catch (e) {
|
|
2756
|
+
if (!(e instanceof RangeError)) {
|
|
2757
|
+
throw e;
|
|
2758
|
+
}
|
|
2759
|
+
}
|
|
2760
|
+
this.totalPos += this.pos;
|
|
2761
|
+
}
|
|
2762
|
+
if (decoded) {
|
|
2763
|
+
if (this.hasRemaining(1)) {
|
|
2764
|
+
throw this.createExtraByteError(this.totalPos);
|
|
2765
|
+
}
|
|
2766
|
+
return object;
|
|
2767
|
+
}
|
|
2768
|
+
const { headByte, pos, totalPos } = this;
|
|
2769
|
+
throw new RangeError(`Insufficient data in parsing ${prettyByte(headByte)} at ${totalPos} (${pos} in the current buffer)`);
|
|
2770
|
+
} finally {
|
|
2771
|
+
this.entered = false;
|
|
2772
|
+
}
|
|
2773
|
+
}
|
|
2774
|
+
decodeArrayStream(stream) {
|
|
2775
|
+
return this.decodeMultiAsync(stream, true);
|
|
2776
|
+
}
|
|
2777
|
+
decodeStream(stream) {
|
|
2778
|
+
return this.decodeMultiAsync(stream, false);
|
|
2779
|
+
}
|
|
2780
|
+
async *decodeMultiAsync(stream, isArray) {
|
|
2781
|
+
if (this.entered) {
|
|
2782
|
+
const instance = this.clone();
|
|
2783
|
+
yield* instance.decodeMultiAsync(stream, isArray);
|
|
2784
|
+
return;
|
|
2785
|
+
}
|
|
2786
|
+
try {
|
|
2787
|
+
this.entered = true;
|
|
2788
|
+
let isArrayHeaderRequired = isArray;
|
|
2789
|
+
let arrayItemsLeft = -1;
|
|
2790
|
+
for await (const buffer of stream) {
|
|
2791
|
+
if (isArray && arrayItemsLeft === 0) {
|
|
2792
|
+
throw this.createExtraByteError(this.totalPos);
|
|
2793
|
+
}
|
|
2794
|
+
this.appendBuffer(buffer);
|
|
2795
|
+
if (isArrayHeaderRequired) {
|
|
2796
|
+
arrayItemsLeft = this.readArraySize();
|
|
2797
|
+
isArrayHeaderRequired = false;
|
|
2798
|
+
this.complete();
|
|
2799
|
+
}
|
|
2800
|
+
try {
|
|
2801
|
+
while (true) {
|
|
2802
|
+
yield this.doDecodeSync();
|
|
2803
|
+
if (--arrayItemsLeft === 0) {
|
|
2804
|
+
break;
|
|
2805
|
+
}
|
|
2806
|
+
}
|
|
2807
|
+
} catch (e) {
|
|
2808
|
+
if (!(e instanceof RangeError)) {
|
|
2809
|
+
throw e;
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2812
|
+
this.totalPos += this.pos;
|
|
2813
|
+
}
|
|
2814
|
+
} finally {
|
|
2815
|
+
this.entered = false;
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2818
|
+
doDecodeSync() {
|
|
2819
|
+
DECODE: while (true) {
|
|
2820
|
+
const headByte = this.readHeadByte();
|
|
2821
|
+
let object;
|
|
2822
|
+
if (headByte >= 224) {
|
|
2823
|
+
object = headByte - 256;
|
|
2824
|
+
} else if (headByte < 192) {
|
|
2825
|
+
if (headByte < 128) {
|
|
2826
|
+
object = headByte;
|
|
2827
|
+
} else if (headByte < 144) {
|
|
2828
|
+
const size = headByte - 128;
|
|
2829
|
+
if (size !== 0) {
|
|
2830
|
+
this.pushMapState(size);
|
|
2831
|
+
this.complete();
|
|
2832
|
+
continue DECODE;
|
|
2833
|
+
} else {
|
|
2834
|
+
object = {};
|
|
2835
|
+
}
|
|
2836
|
+
} else if (headByte < 160) {
|
|
2837
|
+
const size = headByte - 144;
|
|
2838
|
+
if (size !== 0) {
|
|
2839
|
+
this.pushArrayState(size);
|
|
2840
|
+
this.complete();
|
|
2841
|
+
continue DECODE;
|
|
2842
|
+
} else {
|
|
2843
|
+
object = [];
|
|
2844
|
+
}
|
|
2845
|
+
} else {
|
|
2846
|
+
const byteLength = headByte - 160;
|
|
2847
|
+
object = this.decodeString(byteLength, 0);
|
|
2848
|
+
}
|
|
2849
|
+
} else if (headByte === 192) {
|
|
2850
|
+
object = null;
|
|
2851
|
+
} else if (headByte === 194) {
|
|
2852
|
+
object = false;
|
|
2853
|
+
} else if (headByte === 195) {
|
|
2854
|
+
object = true;
|
|
2855
|
+
} else if (headByte === 202) {
|
|
2856
|
+
object = this.readF32();
|
|
2857
|
+
} else if (headByte === 203) {
|
|
2858
|
+
object = this.readF64();
|
|
2859
|
+
} else if (headByte === 204) {
|
|
2860
|
+
object = this.readU8();
|
|
2861
|
+
} else if (headByte === 205) {
|
|
2862
|
+
object = this.readU16();
|
|
2863
|
+
} else if (headByte === 206) {
|
|
2864
|
+
object = this.readU32();
|
|
2865
|
+
} else if (headByte === 207) {
|
|
2866
|
+
if (this.useBigInt64) {
|
|
2867
|
+
object = this.readU64AsBigInt();
|
|
2868
|
+
} else {
|
|
2869
|
+
object = this.readU64();
|
|
2870
|
+
}
|
|
2871
|
+
} else if (headByte === 208) {
|
|
2872
|
+
object = this.readI8();
|
|
2873
|
+
} else if (headByte === 209) {
|
|
2874
|
+
object = this.readI16();
|
|
2875
|
+
} else if (headByte === 210) {
|
|
2876
|
+
object = this.readI32();
|
|
2877
|
+
} else if (headByte === 211) {
|
|
2878
|
+
if (this.useBigInt64) {
|
|
2879
|
+
object = this.readI64AsBigInt();
|
|
2880
|
+
} else {
|
|
2881
|
+
object = this.readI64();
|
|
2882
|
+
}
|
|
2883
|
+
} else if (headByte === 217) {
|
|
2884
|
+
const byteLength = this.lookU8();
|
|
2885
|
+
object = this.decodeString(byteLength, 1);
|
|
2886
|
+
} else if (headByte === 218) {
|
|
2887
|
+
const byteLength = this.lookU16();
|
|
2888
|
+
object = this.decodeString(byteLength, 2);
|
|
2889
|
+
} else if (headByte === 219) {
|
|
2890
|
+
const byteLength = this.lookU32();
|
|
2891
|
+
object = this.decodeString(byteLength, 4);
|
|
2892
|
+
} else if (headByte === 220) {
|
|
2893
|
+
const size = this.readU16();
|
|
2894
|
+
if (size !== 0) {
|
|
2895
|
+
this.pushArrayState(size);
|
|
2896
|
+
this.complete();
|
|
2897
|
+
continue DECODE;
|
|
2898
|
+
} else {
|
|
2899
|
+
object = [];
|
|
2900
|
+
}
|
|
2901
|
+
} else if (headByte === 221) {
|
|
2902
|
+
const size = this.readU32();
|
|
2903
|
+
if (size !== 0) {
|
|
2904
|
+
this.pushArrayState(size);
|
|
2905
|
+
this.complete();
|
|
2906
|
+
continue DECODE;
|
|
2907
|
+
} else {
|
|
2908
|
+
object = [];
|
|
2909
|
+
}
|
|
2910
|
+
} else if (headByte === 222) {
|
|
2911
|
+
const size = this.readU16();
|
|
2912
|
+
if (size !== 0) {
|
|
2913
|
+
this.pushMapState(size);
|
|
2914
|
+
this.complete();
|
|
2915
|
+
continue DECODE;
|
|
2916
|
+
} else {
|
|
2917
|
+
object = {};
|
|
2918
|
+
}
|
|
2919
|
+
} else if (headByte === 223) {
|
|
2920
|
+
const size = this.readU32();
|
|
2921
|
+
if (size !== 0) {
|
|
2922
|
+
this.pushMapState(size);
|
|
2923
|
+
this.complete();
|
|
2924
|
+
continue DECODE;
|
|
2925
|
+
} else {
|
|
2926
|
+
object = {};
|
|
2927
|
+
}
|
|
2928
|
+
} else if (headByte === 196) {
|
|
2929
|
+
const size = this.lookU8();
|
|
2930
|
+
object = this.decodeBinary(size, 1);
|
|
2931
|
+
} else if (headByte === 197) {
|
|
2932
|
+
const size = this.lookU16();
|
|
2933
|
+
object = this.decodeBinary(size, 2);
|
|
2934
|
+
} else if (headByte === 198) {
|
|
2935
|
+
const size = this.lookU32();
|
|
2936
|
+
object = this.decodeBinary(size, 4);
|
|
2937
|
+
} else if (headByte === 212) {
|
|
2938
|
+
object = this.decodeExtension(1, 0);
|
|
2939
|
+
} else if (headByte === 213) {
|
|
2940
|
+
object = this.decodeExtension(2, 0);
|
|
2941
|
+
} else if (headByte === 214) {
|
|
2942
|
+
object = this.decodeExtension(4, 0);
|
|
2943
|
+
} else if (headByte === 215) {
|
|
2944
|
+
object = this.decodeExtension(8, 0);
|
|
2945
|
+
} else if (headByte === 216) {
|
|
2946
|
+
object = this.decodeExtension(16, 0);
|
|
2947
|
+
} else if (headByte === 199) {
|
|
2948
|
+
const size = this.lookU8();
|
|
2949
|
+
object = this.decodeExtension(size, 1);
|
|
2950
|
+
} else if (headByte === 200) {
|
|
2951
|
+
const size = this.lookU16();
|
|
2952
|
+
object = this.decodeExtension(size, 2);
|
|
2953
|
+
} else if (headByte === 201) {
|
|
2954
|
+
const size = this.lookU32();
|
|
2955
|
+
object = this.decodeExtension(size, 4);
|
|
2956
|
+
} else {
|
|
2957
|
+
throw new DecodeError(`Unrecognized type byte: ${prettyByte(headByte)}`);
|
|
2958
|
+
}
|
|
2959
|
+
this.complete();
|
|
2960
|
+
const stack = this.stack;
|
|
2961
|
+
while (stack.length > 0) {
|
|
2962
|
+
const state = stack.top();
|
|
2963
|
+
if (state.type === STATE_ARRAY) {
|
|
2964
|
+
state.array[state.position] = object;
|
|
2965
|
+
state.position++;
|
|
2966
|
+
if (state.position === state.size) {
|
|
2967
|
+
object = state.array;
|
|
2968
|
+
stack.release(state);
|
|
2969
|
+
} else {
|
|
2970
|
+
continue DECODE;
|
|
2971
|
+
}
|
|
2972
|
+
} else if (state.type === STATE_MAP_KEY) {
|
|
2973
|
+
if (object === "__proto__") {
|
|
2974
|
+
throw new DecodeError("The key __proto__ is not allowed");
|
|
2975
|
+
}
|
|
2976
|
+
state.key = this.mapKeyConverter(object);
|
|
2977
|
+
state.type = STATE_MAP_VALUE;
|
|
2978
|
+
continue DECODE;
|
|
2979
|
+
} else {
|
|
2980
|
+
state.map[state.key] = object;
|
|
2981
|
+
state.readCount++;
|
|
2982
|
+
if (state.readCount === state.size) {
|
|
2983
|
+
object = state.map;
|
|
2984
|
+
stack.release(state);
|
|
2985
|
+
} else {
|
|
2986
|
+
state.key = null;
|
|
2987
|
+
state.type = STATE_MAP_KEY;
|
|
2988
|
+
continue DECODE;
|
|
2989
|
+
}
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2992
|
+
return object;
|
|
2993
|
+
}
|
|
2994
|
+
}
|
|
2995
|
+
readHeadByte() {
|
|
2996
|
+
if (this.headByte === HEAD_BYTE_REQUIRED) {
|
|
2997
|
+
this.headByte = this.readU8();
|
|
2998
|
+
}
|
|
2999
|
+
return this.headByte;
|
|
3000
|
+
}
|
|
3001
|
+
complete() {
|
|
3002
|
+
this.headByte = HEAD_BYTE_REQUIRED;
|
|
3003
|
+
}
|
|
3004
|
+
readArraySize() {
|
|
3005
|
+
const headByte = this.readHeadByte();
|
|
3006
|
+
switch (headByte) {
|
|
3007
|
+
case 220:
|
|
3008
|
+
return this.readU16();
|
|
3009
|
+
case 221:
|
|
3010
|
+
return this.readU32();
|
|
3011
|
+
default: {
|
|
3012
|
+
if (headByte < 160) {
|
|
3013
|
+
return headByte - 144;
|
|
3014
|
+
} else {
|
|
3015
|
+
throw new DecodeError(`Unrecognized array type byte: ${prettyByte(headByte)}`);
|
|
3016
|
+
}
|
|
3017
|
+
}
|
|
3018
|
+
}
|
|
3019
|
+
}
|
|
3020
|
+
pushMapState(size) {
|
|
3021
|
+
if (size > this.maxMapLength) {
|
|
3022
|
+
throw new DecodeError(`Max length exceeded: map length (${size}) > maxMapLengthLength (${this.maxMapLength})`);
|
|
3023
|
+
}
|
|
3024
|
+
this.stack.pushMapState(size);
|
|
3025
|
+
}
|
|
3026
|
+
pushArrayState(size) {
|
|
3027
|
+
if (size > this.maxArrayLength) {
|
|
3028
|
+
throw new DecodeError(`Max length exceeded: array length (${size}) > maxArrayLength (${this.maxArrayLength})`);
|
|
3029
|
+
}
|
|
3030
|
+
this.stack.pushArrayState(size);
|
|
3031
|
+
}
|
|
3032
|
+
decodeString(byteLength, headerOffset) {
|
|
3033
|
+
if (!this.rawStrings || this.stateIsMapKey()) {
|
|
3034
|
+
return this.decodeUtf8String(byteLength, headerOffset);
|
|
3035
|
+
}
|
|
3036
|
+
return this.decodeBinary(byteLength, headerOffset);
|
|
3037
|
+
}
|
|
3038
|
+
/**
|
|
3039
|
+
* @throws {@link RangeError}
|
|
3040
|
+
*/
|
|
3041
|
+
decodeUtf8String(byteLength, headerOffset) {
|
|
3042
|
+
if (byteLength > this.maxStrLength) {
|
|
3043
|
+
throw new DecodeError(`Max length exceeded: UTF-8 byte length (${byteLength}) > maxStrLength (${this.maxStrLength})`);
|
|
3044
|
+
}
|
|
3045
|
+
if (this.bytes.byteLength < this.pos + headerOffset + byteLength) {
|
|
3046
|
+
throw MORE_DATA;
|
|
3047
|
+
}
|
|
3048
|
+
const offset = this.pos + headerOffset;
|
|
3049
|
+
let object;
|
|
3050
|
+
if (this.stateIsMapKey() && this.keyDecoder?.canBeCached(byteLength)) {
|
|
3051
|
+
object = this.keyDecoder.decode(this.bytes, offset, byteLength);
|
|
3052
|
+
} else {
|
|
3053
|
+
object = utf8Decode(this.bytes, offset, byteLength);
|
|
3054
|
+
}
|
|
3055
|
+
this.pos += headerOffset + byteLength;
|
|
3056
|
+
return object;
|
|
3057
|
+
}
|
|
3058
|
+
stateIsMapKey() {
|
|
3059
|
+
if (this.stack.length > 0) {
|
|
3060
|
+
const state = this.stack.top();
|
|
3061
|
+
return state.type === STATE_MAP_KEY;
|
|
3062
|
+
}
|
|
3063
|
+
return false;
|
|
3064
|
+
}
|
|
3065
|
+
/**
|
|
3066
|
+
* @throws {@link RangeError}
|
|
3067
|
+
*/
|
|
3068
|
+
decodeBinary(byteLength, headOffset) {
|
|
3069
|
+
if (byteLength > this.maxBinLength) {
|
|
3070
|
+
throw new DecodeError(`Max length exceeded: bin length (${byteLength}) > maxBinLength (${this.maxBinLength})`);
|
|
3071
|
+
}
|
|
3072
|
+
if (!this.hasRemaining(byteLength + headOffset)) {
|
|
3073
|
+
throw MORE_DATA;
|
|
3074
|
+
}
|
|
3075
|
+
const offset = this.pos + headOffset;
|
|
3076
|
+
const object = this.bytes.subarray(offset, offset + byteLength);
|
|
3077
|
+
this.pos += headOffset + byteLength;
|
|
3078
|
+
return object;
|
|
3079
|
+
}
|
|
3080
|
+
decodeExtension(size, headOffset) {
|
|
3081
|
+
if (size > this.maxExtLength) {
|
|
3082
|
+
throw new DecodeError(`Max length exceeded: ext length (${size}) > maxExtLength (${this.maxExtLength})`);
|
|
3083
|
+
}
|
|
3084
|
+
const extType = this.view.getInt8(this.pos + headOffset);
|
|
3085
|
+
const data = this.decodeBinary(
|
|
3086
|
+
size,
|
|
3087
|
+
headOffset + 1
|
|
3088
|
+
/* extType */
|
|
3089
|
+
);
|
|
3090
|
+
return this.extensionCodec.decode(data, extType, this.context);
|
|
3091
|
+
}
|
|
3092
|
+
lookU8() {
|
|
3093
|
+
return this.view.getUint8(this.pos);
|
|
3094
|
+
}
|
|
3095
|
+
lookU16() {
|
|
3096
|
+
return this.view.getUint16(this.pos);
|
|
3097
|
+
}
|
|
3098
|
+
lookU32() {
|
|
3099
|
+
return this.view.getUint32(this.pos);
|
|
3100
|
+
}
|
|
3101
|
+
readU8() {
|
|
3102
|
+
const value = this.view.getUint8(this.pos);
|
|
3103
|
+
this.pos++;
|
|
3104
|
+
return value;
|
|
3105
|
+
}
|
|
3106
|
+
readI8() {
|
|
3107
|
+
const value = this.view.getInt8(this.pos);
|
|
3108
|
+
this.pos++;
|
|
3109
|
+
return value;
|
|
3110
|
+
}
|
|
3111
|
+
readU16() {
|
|
3112
|
+
const value = this.view.getUint16(this.pos);
|
|
3113
|
+
this.pos += 2;
|
|
3114
|
+
return value;
|
|
3115
|
+
}
|
|
3116
|
+
readI16() {
|
|
3117
|
+
const value = this.view.getInt16(this.pos);
|
|
3118
|
+
this.pos += 2;
|
|
3119
|
+
return value;
|
|
3120
|
+
}
|
|
3121
|
+
readU32() {
|
|
3122
|
+
const value = this.view.getUint32(this.pos);
|
|
3123
|
+
this.pos += 4;
|
|
3124
|
+
return value;
|
|
3125
|
+
}
|
|
3126
|
+
readI32() {
|
|
3127
|
+
const value = this.view.getInt32(this.pos);
|
|
3128
|
+
this.pos += 4;
|
|
3129
|
+
return value;
|
|
3130
|
+
}
|
|
3131
|
+
readU64() {
|
|
3132
|
+
const value = getUint64(this.view, this.pos);
|
|
3133
|
+
this.pos += 8;
|
|
3134
|
+
return value;
|
|
3135
|
+
}
|
|
3136
|
+
readI64() {
|
|
3137
|
+
const value = getInt64(this.view, this.pos);
|
|
3138
|
+
this.pos += 8;
|
|
3139
|
+
return value;
|
|
3140
|
+
}
|
|
3141
|
+
readU64AsBigInt() {
|
|
3142
|
+
const value = this.view.getBigUint64(this.pos);
|
|
3143
|
+
this.pos += 8;
|
|
3144
|
+
return value;
|
|
3145
|
+
}
|
|
3146
|
+
readI64AsBigInt() {
|
|
3147
|
+
const value = this.view.getBigInt64(this.pos);
|
|
3148
|
+
this.pos += 8;
|
|
3149
|
+
return value;
|
|
3150
|
+
}
|
|
3151
|
+
readF32() {
|
|
3152
|
+
const value = this.view.getFloat32(this.pos);
|
|
3153
|
+
this.pos += 4;
|
|
3154
|
+
return value;
|
|
3155
|
+
}
|
|
3156
|
+
readF64() {
|
|
3157
|
+
const value = this.view.getFloat64(this.pos);
|
|
3158
|
+
this.pos += 8;
|
|
3159
|
+
return value;
|
|
3160
|
+
}
|
|
3161
|
+
};
|
|
3162
|
+
}
|
|
3163
|
+
});
|
|
3164
|
+
|
|
3165
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/decode.mjs
|
|
3166
|
+
function decode(buffer, options) {
|
|
3167
|
+
const decoder = new Decoder(options);
|
|
3168
|
+
return decoder.decode(buffer);
|
|
3169
|
+
}
|
|
3170
|
+
function decodeMulti(buffer, options) {
|
|
3171
|
+
const decoder = new Decoder(options);
|
|
3172
|
+
return decoder.decodeMulti(buffer);
|
|
3173
|
+
}
|
|
3174
|
+
var init_decode = __esm({
|
|
3175
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/decode.mjs"() {
|
|
3176
|
+
"use strict";
|
|
3177
|
+
init_Decoder();
|
|
3178
|
+
}
|
|
3179
|
+
});
|
|
3180
|
+
|
|
3181
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/stream.mjs
|
|
3182
|
+
function isAsyncIterable(object) {
|
|
3183
|
+
return object[Symbol.asyncIterator] != null;
|
|
3184
|
+
}
|
|
3185
|
+
async function* asyncIterableFromStream(stream) {
|
|
3186
|
+
const reader = stream.getReader();
|
|
3187
|
+
try {
|
|
3188
|
+
while (true) {
|
|
3189
|
+
const { done, value } = await reader.read();
|
|
3190
|
+
if (done) {
|
|
3191
|
+
return;
|
|
3192
|
+
}
|
|
3193
|
+
yield value;
|
|
3194
|
+
}
|
|
3195
|
+
} finally {
|
|
3196
|
+
reader.releaseLock();
|
|
3197
|
+
}
|
|
3198
|
+
}
|
|
3199
|
+
function ensureAsyncIterable(streamLike) {
|
|
3200
|
+
if (isAsyncIterable(streamLike)) {
|
|
3201
|
+
return streamLike;
|
|
3202
|
+
} else {
|
|
3203
|
+
return asyncIterableFromStream(streamLike);
|
|
3204
|
+
}
|
|
3205
|
+
}
|
|
3206
|
+
var init_stream = __esm({
|
|
3207
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/stream.mjs"() {
|
|
3208
|
+
"use strict";
|
|
3209
|
+
}
|
|
3210
|
+
});
|
|
3211
|
+
|
|
3212
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/decodeAsync.mjs
|
|
3213
|
+
async function decodeAsync(streamLike, options) {
|
|
3214
|
+
const stream = ensureAsyncIterable(streamLike);
|
|
3215
|
+
const decoder = new Decoder(options);
|
|
3216
|
+
return decoder.decodeAsync(stream);
|
|
3217
|
+
}
|
|
3218
|
+
function decodeArrayStream(streamLike, options) {
|
|
3219
|
+
const stream = ensureAsyncIterable(streamLike);
|
|
3220
|
+
const decoder = new Decoder(options);
|
|
3221
|
+
return decoder.decodeArrayStream(stream);
|
|
3222
|
+
}
|
|
3223
|
+
function decodeMultiStream(streamLike, options) {
|
|
3224
|
+
const stream = ensureAsyncIterable(streamLike);
|
|
3225
|
+
const decoder = new Decoder(options);
|
|
3226
|
+
return decoder.decodeStream(stream);
|
|
3227
|
+
}
|
|
3228
|
+
var init_decodeAsync = __esm({
|
|
3229
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/decodeAsync.mjs"() {
|
|
3230
|
+
"use strict";
|
|
3231
|
+
init_Decoder();
|
|
3232
|
+
init_stream();
|
|
3233
|
+
}
|
|
3234
|
+
});
|
|
3235
|
+
|
|
3236
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/index.mjs
|
|
3237
|
+
var dist_exports = {};
|
|
3238
|
+
__export(dist_exports, {
|
|
3239
|
+
DecodeError: () => DecodeError,
|
|
3240
|
+
Decoder: () => Decoder,
|
|
3241
|
+
EXT_TIMESTAMP: () => EXT_TIMESTAMP,
|
|
3242
|
+
Encoder: () => Encoder,
|
|
3243
|
+
ExtData: () => ExtData,
|
|
3244
|
+
ExtensionCodec: () => ExtensionCodec,
|
|
3245
|
+
decode: () => decode,
|
|
3246
|
+
decodeArrayStream: () => decodeArrayStream,
|
|
3247
|
+
decodeAsync: () => decodeAsync,
|
|
3248
|
+
decodeMulti: () => decodeMulti,
|
|
3249
|
+
decodeMultiStream: () => decodeMultiStream,
|
|
3250
|
+
decodeTimestampExtension: () => decodeTimestampExtension,
|
|
3251
|
+
decodeTimestampToTimeSpec: () => decodeTimestampToTimeSpec,
|
|
3252
|
+
encode: () => encode,
|
|
3253
|
+
encodeDateToTimeSpec: () => encodeDateToTimeSpec,
|
|
3254
|
+
encodeTimeSpecToTimestamp: () => encodeTimeSpecToTimestamp,
|
|
3255
|
+
encodeTimestampExtension: () => encodeTimestampExtension
|
|
3256
|
+
});
|
|
3257
|
+
var init_dist = __esm({
|
|
3258
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/index.mjs"() {
|
|
3259
|
+
"use strict";
|
|
3260
|
+
init_encode();
|
|
3261
|
+
init_decode();
|
|
3262
|
+
init_decodeAsync();
|
|
3263
|
+
init_Decoder();
|
|
3264
|
+
init_DecodeError();
|
|
3265
|
+
init_Encoder();
|
|
3266
|
+
init_ExtensionCodec();
|
|
3267
|
+
init_ExtData();
|
|
3268
|
+
init_timestamp();
|
|
3269
|
+
}
|
|
3270
|
+
});
|
|
3271
|
+
|
|
3272
|
+
// src/serializers/MessagePackSerializer.ts
|
|
3273
|
+
var MessagePackSerializer_exports = {};
|
|
3274
|
+
__export(MessagePackSerializer_exports, {
|
|
3275
|
+
MessagePackSerializer: () => MessagePackSerializer
|
|
3276
|
+
});
|
|
3277
|
+
var MessagePackSerializer;
|
|
3278
|
+
var init_MessagePackSerializer = __esm({
|
|
3279
|
+
"src/serializers/MessagePackSerializer.ts"() {
|
|
3280
|
+
"use strict";
|
|
3281
|
+
MessagePackSerializer = class {
|
|
3282
|
+
msgpack;
|
|
3283
|
+
constructor() {
|
|
3284
|
+
try {
|
|
3285
|
+
this.msgpack = (init_dist(), __toCommonJS(dist_exports));
|
|
3286
|
+
} catch (_e) {
|
|
3287
|
+
throw new Error(
|
|
3288
|
+
"MessagePackSerializer requires @msgpack/msgpack. Please install it: bun add @msgpack/msgpack"
|
|
3289
|
+
);
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
serialize(job) {
|
|
3293
|
+
const id = job.id || `${Date.now()}-${crypto.randomUUID()}`;
|
|
3294
|
+
const properties = {};
|
|
3295
|
+
for (const key in job) {
|
|
3296
|
+
if (Object.hasOwn(job, key) && typeof job[key] !== "function") {
|
|
3297
|
+
properties[key] = job[key];
|
|
3298
|
+
}
|
|
3299
|
+
}
|
|
3300
|
+
const encoded = this.msgpack.encode(properties);
|
|
3301
|
+
const data = Buffer.from(encoded).toString("base64");
|
|
3302
|
+
return {
|
|
3303
|
+
id,
|
|
3304
|
+
type: "msgpack",
|
|
3305
|
+
data,
|
|
3306
|
+
createdAt: Date.now(),
|
|
3307
|
+
...job.delaySeconds !== void 0 ? { delaySeconds: job.delaySeconds } : {},
|
|
3308
|
+
attempts: job.attempts ?? 0,
|
|
3309
|
+
...job.maxAttempts !== void 0 ? { maxAttempts: job.maxAttempts } : {},
|
|
3310
|
+
...job.groupId ? { groupId: job.groupId } : {},
|
|
3311
|
+
...job.priority ? { priority: job.priority } : {}
|
|
3312
|
+
};
|
|
3313
|
+
}
|
|
3314
|
+
deserialize(serialized) {
|
|
3315
|
+
if (serialized.type !== "msgpack") {
|
|
3316
|
+
throw new Error('Invalid serialization type: expected "msgpack"');
|
|
3317
|
+
}
|
|
3318
|
+
const buffer = Buffer.from(serialized.data, "base64");
|
|
3319
|
+
const properties = this.msgpack.decode(buffer);
|
|
3320
|
+
const job = /* @__PURE__ */ Object.create({});
|
|
3321
|
+
Object.assign(job, properties);
|
|
3322
|
+
job.id = serialized.id;
|
|
3323
|
+
if (serialized.groupId) {
|
|
3324
|
+
job.groupId = serialized.groupId;
|
|
3325
|
+
}
|
|
3326
|
+
if (serialized.priority) {
|
|
3327
|
+
job.priority = serialized.priority;
|
|
3328
|
+
}
|
|
3329
|
+
if (serialized.delaySeconds !== void 0) {
|
|
3330
|
+
job.delaySeconds = serialized.delaySeconds;
|
|
3331
|
+
}
|
|
3332
|
+
if (serialized.attempts !== void 0) {
|
|
3333
|
+
job.attempts = serialized.attempts;
|
|
3334
|
+
}
|
|
3335
|
+
return job;
|
|
3336
|
+
}
|
|
3337
|
+
};
|
|
3338
|
+
}
|
|
3339
|
+
});
|
|
3340
|
+
|
|
1065
3341
|
// src/Scheduler.ts
|
|
1066
3342
|
var Scheduler_exports = {};
|
|
1067
3343
|
__export(Scheduler_exports, {
|
|
@@ -1080,6 +3356,9 @@ var init_Scheduler = __esm({
|
|
|
1080
3356
|
prefix;
|
|
1081
3357
|
get client() {
|
|
1082
3358
|
const driver = this.manager.getDriver(this.manager.getDefaultConnection());
|
|
3359
|
+
if (!driver || !("client" in driver)) {
|
|
3360
|
+
throw new Error("[Scheduler] Driver does not support Redis client access");
|
|
3361
|
+
}
|
|
1083
3362
|
return driver.client;
|
|
1084
3363
|
}
|
|
1085
3364
|
/**
|
|
@@ -1092,7 +3371,11 @@ var init_Scheduler = __esm({
|
|
|
1092
3371
|
nextRun,
|
|
1093
3372
|
enabled: true
|
|
1094
3373
|
};
|
|
1095
|
-
const
|
|
3374
|
+
const client = this.client;
|
|
3375
|
+
if (typeof client.pipeline !== "function") {
|
|
3376
|
+
throw new Error("[Scheduler] Redis client does not support pipeline");
|
|
3377
|
+
}
|
|
3378
|
+
const pipe = client.pipeline();
|
|
1096
3379
|
pipe.hset(`${this.prefix}schedule:${config.id}`, {
|
|
1097
3380
|
...fullConfig,
|
|
1098
3381
|
job: JSON.stringify(fullConfig.job)
|
|
@@ -1104,7 +3387,11 @@ var init_Scheduler = __esm({
|
|
|
1104
3387
|
* Remove a scheduled job.
|
|
1105
3388
|
*/
|
|
1106
3389
|
async remove(id) {
|
|
1107
|
-
const
|
|
3390
|
+
const client = this.client;
|
|
3391
|
+
if (typeof client.pipeline !== "function") {
|
|
3392
|
+
throw new Error("[Scheduler] Redis client does not support pipeline");
|
|
3393
|
+
}
|
|
3394
|
+
const pipe = client.pipeline();
|
|
1108
3395
|
pipe.del(`${this.prefix}schedule:${id}`);
|
|
1109
3396
|
pipe.zrem(`${this.prefix}schedules`, id);
|
|
1110
3397
|
await pipe.exec();
|
|
@@ -1113,10 +3400,14 @@ var init_Scheduler = __esm({
|
|
|
1113
3400
|
* List all scheduled jobs.
|
|
1114
3401
|
*/
|
|
1115
3402
|
async list() {
|
|
1116
|
-
const
|
|
3403
|
+
const client = this.client;
|
|
3404
|
+
if (typeof client.zrange !== "function") {
|
|
3405
|
+
throw new Error("[Scheduler] Redis client does not support zrange");
|
|
3406
|
+
}
|
|
3407
|
+
const ids = await client.zrange(`${this.prefix}schedules`, 0, -1);
|
|
1117
3408
|
const configs = [];
|
|
1118
3409
|
for (const id of ids) {
|
|
1119
|
-
const data = await
|
|
3410
|
+
const data = await client.hgetall?.(`${this.prefix}schedule:${id}`);
|
|
1120
3411
|
if (data?.id) {
|
|
1121
3412
|
configs.push({
|
|
1122
3413
|
...data,
|
|
@@ -1133,7 +3424,8 @@ var init_Scheduler = __esm({
|
|
|
1133
3424
|
* Run a scheduled job immediately (out of schedule).
|
|
1134
3425
|
*/
|
|
1135
3426
|
async runNow(id) {
|
|
1136
|
-
const
|
|
3427
|
+
const client = this.client;
|
|
3428
|
+
const data = await client.hgetall?.(`${this.prefix}schedule:${id}`);
|
|
1137
3429
|
if (data?.id) {
|
|
1138
3430
|
const serialized = JSON.parse(data.job);
|
|
1139
3431
|
const serializer = this.manager.getSerializer();
|
|
@@ -1146,14 +3438,21 @@ var init_Scheduler = __esm({
|
|
|
1146
3438
|
* This should be called periodically (e.g. every minute).
|
|
1147
3439
|
*/
|
|
1148
3440
|
async tick() {
|
|
3441
|
+
const client = this.client;
|
|
3442
|
+
if (typeof client.zrangebyscore !== "function") {
|
|
3443
|
+
throw new Error("[Scheduler] Redis client does not support zrangebyscore");
|
|
3444
|
+
}
|
|
1149
3445
|
const now = Date.now();
|
|
1150
|
-
const dueIds = await
|
|
3446
|
+
const dueIds = await client.zrangebyscore(`${this.prefix}schedules`, 0, now);
|
|
1151
3447
|
let fired = 0;
|
|
1152
3448
|
for (const id of dueIds) {
|
|
1153
3449
|
const lockKey = `${this.prefix}lock:schedule:${id}:${Math.floor(now / 1e3)}`;
|
|
1154
|
-
|
|
3450
|
+
if (typeof client.set !== "function") {
|
|
3451
|
+
continue;
|
|
3452
|
+
}
|
|
3453
|
+
const lock = await client.set(lockKey, "1", "EX", 10, "NX");
|
|
1155
3454
|
if (lock === "OK") {
|
|
1156
|
-
const data = await
|
|
3455
|
+
const data = await client.hgetall?.(`${this.prefix}schedule:${id}`);
|
|
1157
3456
|
if (data?.id && data.enabled === "true") {
|
|
1158
3457
|
try {
|
|
1159
3458
|
const serializedJob = JSON.parse(data.job);
|
|
@@ -1161,16 +3460,19 @@ var init_Scheduler = __esm({
|
|
|
1161
3460
|
const driver = this.manager.getDriver(connection);
|
|
1162
3461
|
await driver.push(data.queue, serializedJob);
|
|
1163
3462
|
const nextRun = parser.parse(data.cron).next().getTime();
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
3463
|
+
if (typeof client.pipeline === "function") {
|
|
3464
|
+
const pipe = client.pipeline();
|
|
3465
|
+
pipe.hset(`${this.prefix}schedule:${id}`, {
|
|
3466
|
+
lastRun: now,
|
|
3467
|
+
nextRun
|
|
3468
|
+
});
|
|
3469
|
+
pipe.zadd(`${this.prefix}schedules`, nextRun, id);
|
|
3470
|
+
await pipe.exec();
|
|
3471
|
+
}
|
|
1171
3472
|
fired++;
|
|
1172
3473
|
} catch (err) {
|
|
1173
|
-
|
|
3474
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
3475
|
+
console.error(`[Scheduler] Failed to process schedule ${id}:`, error.message);
|
|
1174
3476
|
}
|
|
1175
3477
|
}
|
|
1176
3478
|
}
|
|
@@ -1181,6 +3483,10 @@ var init_Scheduler = __esm({
|
|
|
1181
3483
|
}
|
|
1182
3484
|
});
|
|
1183
3485
|
|
|
3486
|
+
// src/Consumer.ts
|
|
3487
|
+
import { EventEmitter } from "events";
|
|
3488
|
+
import pLimit from "p-limit";
|
|
3489
|
+
|
|
1184
3490
|
// src/Worker.ts
|
|
1185
3491
|
var Worker = class {
|
|
1186
3492
|
constructor(options = {}) {
|
|
@@ -1238,18 +3544,40 @@ var Worker = class {
|
|
|
1238
3544
|
};
|
|
1239
3545
|
|
|
1240
3546
|
// src/Consumer.ts
|
|
1241
|
-
var Consumer = class {
|
|
3547
|
+
var Consumer = class extends EventEmitter {
|
|
1242
3548
|
constructor(queueManager, options) {
|
|
3549
|
+
super();
|
|
1243
3550
|
this.queueManager = queueManager;
|
|
1244
3551
|
this.options = options;
|
|
1245
3552
|
}
|
|
1246
3553
|
running = false;
|
|
1247
3554
|
stopRequested = false;
|
|
1248
|
-
workerId = `worker-${
|
|
3555
|
+
workerId = `worker-${crypto.randomUUID()}`;
|
|
1249
3556
|
heartbeatTimer = null;
|
|
3557
|
+
groupLimiters = /* @__PURE__ */ new Map();
|
|
3558
|
+
stats = {
|
|
3559
|
+
processed: 0,
|
|
3560
|
+
failed: 0,
|
|
3561
|
+
retried: 0,
|
|
3562
|
+
active: 0
|
|
3563
|
+
};
|
|
1250
3564
|
get connectionName() {
|
|
1251
3565
|
return this.options.connection ?? this.queueManager.getDefaultConnection();
|
|
1252
3566
|
}
|
|
3567
|
+
/**
|
|
3568
|
+
* Log debug message.
|
|
3569
|
+
*/
|
|
3570
|
+
log(message, data) {
|
|
3571
|
+
if (this.options.debug) {
|
|
3572
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
3573
|
+
const prefix = `[Consumer:${this.workerId}] [${timestamp}]`;
|
|
3574
|
+
if (data) {
|
|
3575
|
+
console.log(prefix, message, data);
|
|
3576
|
+
} else {
|
|
3577
|
+
console.log(prefix, message);
|
|
3578
|
+
}
|
|
3579
|
+
}
|
|
3580
|
+
}
|
|
1253
3581
|
/**
|
|
1254
3582
|
* Start the consumer loop.
|
|
1255
3583
|
*/
|
|
@@ -1260,19 +3588,36 @@ var Consumer = class {
|
|
|
1260
3588
|
this.running = true;
|
|
1261
3589
|
this.stopRequested = false;
|
|
1262
3590
|
const worker = new Worker(this.options.workerOptions);
|
|
1263
|
-
|
|
3591
|
+
let currentPollInterval = this.options.pollInterval ?? 1e3;
|
|
3592
|
+
const minPollInterval = this.options.minPollInterval ?? 100;
|
|
3593
|
+
const maxPollInterval = this.options.maxPollInterval ?? 5e3;
|
|
3594
|
+
const backoffMultiplier = this.options.backoffMultiplier ?? 1.5;
|
|
1264
3595
|
const keepAlive = this.options.keepAlive ?? true;
|
|
1265
|
-
|
|
3596
|
+
const concurrency = this.options.concurrency ?? 1;
|
|
3597
|
+
const batchSize = this.options.batchSize ?? 1;
|
|
3598
|
+
const useBlocking = this.options.useBlocking ?? true;
|
|
3599
|
+
const blockingTimeout = this.options.blockingTimeout ?? 5;
|
|
3600
|
+
this.log("Started", {
|
|
1266
3601
|
queues: this.options.queues,
|
|
1267
3602
|
connection: this.options.connection,
|
|
1268
|
-
workerId: this.workerId
|
|
3603
|
+
workerId: this.workerId,
|
|
3604
|
+
concurrency,
|
|
3605
|
+
batchSize
|
|
1269
3606
|
});
|
|
1270
3607
|
if (this.options.monitor) {
|
|
1271
3608
|
this.startHeartbeat();
|
|
1272
|
-
await this.publishLog(
|
|
3609
|
+
await this.publishLog(
|
|
3610
|
+
"info",
|
|
3611
|
+
`Consumer started on [${this.options.queues.join(", ")}] with concurrency ${concurrency}`
|
|
3612
|
+
);
|
|
1273
3613
|
}
|
|
1274
3614
|
while (this.running && !this.stopRequested) {
|
|
1275
|
-
|
|
3615
|
+
const capacity = concurrency - this.stats.active;
|
|
3616
|
+
if (capacity <= 0) {
|
|
3617
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
3618
|
+
continue;
|
|
3619
|
+
}
|
|
3620
|
+
const eligibleQueues = [];
|
|
1276
3621
|
for (const queue of this.options.queues) {
|
|
1277
3622
|
if (this.options.rateLimits?.[queue]) {
|
|
1278
3623
|
const limit = this.options.rateLimits[queue];
|
|
@@ -1288,60 +3633,74 @@ var Consumer = class {
|
|
|
1288
3633
|
console.error(`[Consumer] Error checking rate limit for "${queue}":`, err);
|
|
1289
3634
|
}
|
|
1290
3635
|
}
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
3636
|
+
eligibleQueues.push(queue);
|
|
3637
|
+
}
|
|
3638
|
+
if (eligibleQueues.length === 0) {
|
|
3639
|
+
await new Promise((resolve) => setTimeout(resolve, currentPollInterval));
|
|
3640
|
+
continue;
|
|
3641
|
+
}
|
|
3642
|
+
let jobs = [];
|
|
3643
|
+
let didBlock = false;
|
|
3644
|
+
try {
|
|
3645
|
+
const currentBatchSize = Math.min(batchSize, capacity);
|
|
3646
|
+
const driver = this.queueManager.getDriver(this.connectionName);
|
|
3647
|
+
if (currentBatchSize > 1) {
|
|
3648
|
+
for (const queue of eligibleQueues) {
|
|
3649
|
+
const fetched = await this.queueManager.popMany(
|
|
3650
|
+
queue,
|
|
3651
|
+
currentBatchSize,
|
|
3652
|
+
this.connectionName
|
|
3653
|
+
);
|
|
3654
|
+
if (fetched.length > 0) {
|
|
3655
|
+
jobs = fetched;
|
|
3656
|
+
break;
|
|
1297
3657
|
}
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
if (this.options.monitor) {
|
|
1317
|
-
await this.publishLog(
|
|
1318
|
-
"warning",
|
|
1319
|
-
`Job retrying in ${delaySec}s (Attempt ${job.attempts}/${maxAttempts})`,
|
|
1320
|
-
job.id
|
|
1321
|
-
);
|
|
1322
|
-
}
|
|
1323
|
-
} else {
|
|
1324
|
-
await this.queueManager.fail(job, err).catch((dlqErr) => {
|
|
1325
|
-
console.error(`[Consumer] Error moving job to DLQ:`, dlqErr);
|
|
1326
|
-
});
|
|
3658
|
+
}
|
|
3659
|
+
} else {
|
|
3660
|
+
if (useBlocking && driver.popBlocking) {
|
|
3661
|
+
didBlock = true;
|
|
3662
|
+
const job = await this.queueManager.popBlocking(
|
|
3663
|
+
eligibleQueues,
|
|
3664
|
+
blockingTimeout,
|
|
3665
|
+
this.connectionName
|
|
3666
|
+
);
|
|
3667
|
+
if (job) {
|
|
3668
|
+
jobs.push(job);
|
|
3669
|
+
}
|
|
3670
|
+
} else {
|
|
3671
|
+
for (const queue of eligibleQueues) {
|
|
3672
|
+
const job = await this.queueManager.pop(queue, this.connectionName);
|
|
3673
|
+
if (job) {
|
|
3674
|
+
jobs.push(job);
|
|
3675
|
+
break;
|
|
1327
3676
|
}
|
|
1328
|
-
} finally {
|
|
1329
|
-
await this.queueManager.complete(job).catch((err) => {
|
|
1330
|
-
console.error(`[Consumer] Error completing job in queue "${queue}":`, err);
|
|
1331
|
-
});
|
|
1332
3677
|
}
|
|
1333
3678
|
}
|
|
1334
|
-
} catch (error) {
|
|
1335
|
-
console.error(`[Consumer] Error polling queue "${queue}":`, error);
|
|
1336
3679
|
}
|
|
3680
|
+
if (jobs.length > 0) {
|
|
3681
|
+
this.stats.active += jobs.length;
|
|
3682
|
+
currentPollInterval = minPollInterval;
|
|
3683
|
+
for (const job of jobs) {
|
|
3684
|
+
this.runJob(job, worker).finally(() => {
|
|
3685
|
+
this.stats.active--;
|
|
3686
|
+
});
|
|
3687
|
+
}
|
|
3688
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
3689
|
+
continue;
|
|
3690
|
+
}
|
|
3691
|
+
} catch (error) {
|
|
3692
|
+
console.error("[Consumer] Loop error:", error);
|
|
1337
3693
|
}
|
|
1338
|
-
if (
|
|
3694
|
+
if (this.stats.active === 0 && !keepAlive) {
|
|
1339
3695
|
break;
|
|
1340
3696
|
}
|
|
1341
|
-
if (!this.stopRequested
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
3697
|
+
if (!this.stopRequested) {
|
|
3698
|
+
if (!didBlock) {
|
|
3699
|
+
await new Promise((resolve) => setTimeout(resolve, currentPollInterval));
|
|
3700
|
+
currentPollInterval = Math.min(currentPollInterval * backoffMultiplier, maxPollInterval);
|
|
3701
|
+
}
|
|
3702
|
+
} else {
|
|
3703
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
1345
3704
|
}
|
|
1346
3705
|
}
|
|
1347
3706
|
this.running = false;
|
|
@@ -1349,7 +3708,89 @@ var Consumer = class {
|
|
|
1349
3708
|
if (this.options.monitor) {
|
|
1350
3709
|
await this.publishLog("info", "Consumer stopped");
|
|
1351
3710
|
}
|
|
1352
|
-
|
|
3711
|
+
this.log("Stopped");
|
|
3712
|
+
}
|
|
3713
|
+
/**
|
|
3714
|
+
* Run a job with concurrency controls.
|
|
3715
|
+
*/
|
|
3716
|
+
async runJob(job, worker) {
|
|
3717
|
+
if (!job.groupId || this.options.groupJobsSequential === false) {
|
|
3718
|
+
return this.handleJob(job, worker);
|
|
3719
|
+
}
|
|
3720
|
+
let limiter = this.groupLimiters.get(job.groupId);
|
|
3721
|
+
if (!limiter) {
|
|
3722
|
+
limiter = pLimit(1);
|
|
3723
|
+
this.groupLimiters.set(job.groupId, limiter);
|
|
3724
|
+
}
|
|
3725
|
+
if (limiter.pendingCount > 0) {
|
|
3726
|
+
this.log(`Job ${job.id} queued behind group ${job.groupId}`);
|
|
3727
|
+
}
|
|
3728
|
+
await limiter(async () => {
|
|
3729
|
+
await this.handleJob(job, worker);
|
|
3730
|
+
});
|
|
3731
|
+
if (limiter.activeCount === 0 && limiter.pendingCount === 0) {
|
|
3732
|
+
this.groupLimiters.delete(job.groupId);
|
|
3733
|
+
}
|
|
3734
|
+
}
|
|
3735
|
+
/**
|
|
3736
|
+
* Handle a single job.
|
|
3737
|
+
*/
|
|
3738
|
+
async handleJob(job, worker) {
|
|
3739
|
+
const currentQueue = job.queueName || "default";
|
|
3740
|
+
const startTime = Date.now();
|
|
3741
|
+
this.log(`Processing job ${job.id} from ${currentQueue}`);
|
|
3742
|
+
this.emit("job:started", { job, queue: currentQueue });
|
|
3743
|
+
if (this.options.monitor) {
|
|
3744
|
+
await this.publishLog("info", `Processing job: ${job.id}`, job.id);
|
|
3745
|
+
}
|
|
3746
|
+
try {
|
|
3747
|
+
await worker.process(job);
|
|
3748
|
+
const duration = Date.now() - startTime;
|
|
3749
|
+
this.stats.processed++;
|
|
3750
|
+
this.emit("job:processed", { job, duration, queue: currentQueue });
|
|
3751
|
+
this.log(`Completed job ${job.id} in ${duration}ms`);
|
|
3752
|
+
if (this.options.monitor) {
|
|
3753
|
+
await this.publishLog("success", `Completed job: ${job.id}`, job.id);
|
|
3754
|
+
}
|
|
3755
|
+
} catch (err) {
|
|
3756
|
+
const error = err;
|
|
3757
|
+
const duration = Date.now() - startTime;
|
|
3758
|
+
this.emit("job:failed", { job, error, duration, queue: currentQueue });
|
|
3759
|
+
this.log(`Failed job ${job.id} in ${duration}ms`, { error: error.message });
|
|
3760
|
+
this.stats.failed++;
|
|
3761
|
+
if (this.options.monitor) {
|
|
3762
|
+
await this.publishLog("error", `Job failed: ${job.id} - ${error.message}`, job.id);
|
|
3763
|
+
}
|
|
3764
|
+
const attempts = job.attempts ?? 1;
|
|
3765
|
+
const maxAttempts = job.maxAttempts ?? this.options.workerOptions?.maxAttempts ?? 3;
|
|
3766
|
+
if (attempts < maxAttempts) {
|
|
3767
|
+
job.attempts = attempts + 1;
|
|
3768
|
+
const delayMs = job.getRetryDelay(job.attempts);
|
|
3769
|
+
const delaySec = Math.ceil(delayMs / 1e3);
|
|
3770
|
+
job.delay(delaySec);
|
|
3771
|
+
await this.queueManager.push(job);
|
|
3772
|
+
this.log(`Retrying job ${job.id} in ${delaySec}s (Attempt ${job.attempts}/${maxAttempts})`);
|
|
3773
|
+
this.stats.retried++;
|
|
3774
|
+
this.emit("job:retried", { job, attempt: job.attempts, delay: delaySec });
|
|
3775
|
+
if (this.options.monitor) {
|
|
3776
|
+
await this.publishLog(
|
|
3777
|
+
"warning",
|
|
3778
|
+
`Job retrying in ${delaySec}s (Attempt ${job.attempts}/${maxAttempts})`,
|
|
3779
|
+
job.id
|
|
3780
|
+
);
|
|
3781
|
+
}
|
|
3782
|
+
} else {
|
|
3783
|
+
this.emit("job:failed_permanently", { job, error });
|
|
3784
|
+
this.log(`Job ${job.id} failed permanently`);
|
|
3785
|
+
await this.queueManager.fail(job, error).catch((dlqErr) => {
|
|
3786
|
+
console.error("[Consumer] Error moving job to DLQ:", dlqErr);
|
|
3787
|
+
});
|
|
3788
|
+
}
|
|
3789
|
+
} finally {
|
|
3790
|
+
await this.queueManager.complete(job).catch((err) => {
|
|
3791
|
+
console.error(`[Consumer] Error completing job in queue "${currentQueue}":`, err);
|
|
3792
|
+
});
|
|
3793
|
+
}
|
|
1353
3794
|
}
|
|
1354
3795
|
startHeartbeat() {
|
|
1355
3796
|
const interval = typeof this.options.monitor === "object" ? this.options.monitor.interval ?? 5e3 : 5e3;
|
|
@@ -1369,7 +3810,8 @@ var Consumer = class {
|
|
|
1369
3810
|
rss: Math.floor(mem.rss / 1024 / 1024),
|
|
1370
3811
|
heapUsed: Math.floor(mem.heapUsed / 1024 / 1024),
|
|
1371
3812
|
total: Math.floor(os.totalmem() / 1024 / 1024)
|
|
1372
|
-
}
|
|
3813
|
+
},
|
|
3814
|
+
stats: this.stats
|
|
1373
3815
|
};
|
|
1374
3816
|
await driver.reportHeartbeat(
|
|
1375
3817
|
{
|
|
@@ -1419,7 +3861,7 @@ var Consumer = class {
|
|
|
1419
3861
|
* Stop the consumer loop (graceful shutdown).
|
|
1420
3862
|
*/
|
|
1421
3863
|
async stop() {
|
|
1422
|
-
|
|
3864
|
+
this.log("Stopping...");
|
|
1423
3865
|
this.stopRequested = true;
|
|
1424
3866
|
while (this.running) {
|
|
1425
3867
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
@@ -1431,6 +3873,20 @@ var Consumer = class {
|
|
|
1431
3873
|
isRunning() {
|
|
1432
3874
|
return this.running;
|
|
1433
3875
|
}
|
|
3876
|
+
/**
|
|
3877
|
+
* Get current consumer statistics.
|
|
3878
|
+
*/
|
|
3879
|
+
getStats() {
|
|
3880
|
+
return { ...this.stats };
|
|
3881
|
+
}
|
|
3882
|
+
/**
|
|
3883
|
+
* Reset statistics counters.
|
|
3884
|
+
*/
|
|
3885
|
+
resetStats() {
|
|
3886
|
+
this.stats.processed = 0;
|
|
3887
|
+
this.stats.failed = 0;
|
|
3888
|
+
this.stats.retried = 0;
|
|
3889
|
+
}
|
|
1434
3890
|
};
|
|
1435
3891
|
|
|
1436
3892
|
// src/index.ts
|
|
@@ -1440,6 +3896,10 @@ init_KafkaDriver();
|
|
|
1440
3896
|
// src/drivers/MemoryDriver.ts
|
|
1441
3897
|
var MemoryDriver = class {
|
|
1442
3898
|
queues = /* @__PURE__ */ new Map();
|
|
3899
|
+
maxSize;
|
|
3900
|
+
constructor(config = {}) {
|
|
3901
|
+
this.maxSize = config.maxSize ?? Infinity;
|
|
3902
|
+
}
|
|
1443
3903
|
/**
|
|
1444
3904
|
* Push a job to a queue.
|
|
1445
3905
|
*/
|
|
@@ -1447,7 +3907,11 @@ var MemoryDriver = class {
|
|
|
1447
3907
|
if (!this.queues.has(queue)) {
|
|
1448
3908
|
this.queues.set(queue, []);
|
|
1449
3909
|
}
|
|
1450
|
-
this.queues.get(queue)
|
|
3910
|
+
const q = this.queues.get(queue);
|
|
3911
|
+
if (q.length >= this.maxSize) {
|
|
3912
|
+
throw new Error(`[MemoryDriver] Queue '${queue}' is full (max size: ${this.maxSize})`);
|
|
3913
|
+
}
|
|
3914
|
+
q.push(job);
|
|
1451
3915
|
}
|
|
1452
3916
|
/**
|
|
1453
3917
|
* Pop a job from a queue (FIFO).
|
|
@@ -1478,6 +3942,39 @@ var MemoryDriver = class {
|
|
|
1478
3942
|
async clear(queue) {
|
|
1479
3943
|
this.queues.delete(queue);
|
|
1480
3944
|
}
|
|
3945
|
+
/**
|
|
3946
|
+
* Mark a job as permanently failed.
|
|
3947
|
+
*/
|
|
3948
|
+
async fail(queue, job) {
|
|
3949
|
+
const failedQueue = `failed:${queue}`;
|
|
3950
|
+
if (!this.queues.has(failedQueue)) {
|
|
3951
|
+
this.queues.set(failedQueue, []);
|
|
3952
|
+
}
|
|
3953
|
+
this.queues.get(failedQueue)?.push(job);
|
|
3954
|
+
}
|
|
3955
|
+
/**
|
|
3956
|
+
* Get queue statistics.
|
|
3957
|
+
*/
|
|
3958
|
+
async stats(queue) {
|
|
3959
|
+
const jobs = this.queues.get(queue) || [];
|
|
3960
|
+
const now = Date.now();
|
|
3961
|
+
let pending = 0;
|
|
3962
|
+
let delayed = 0;
|
|
3963
|
+
for (const job of jobs) {
|
|
3964
|
+
const isDelayed = job.delaySeconds && now < job.createdAt + job.delaySeconds * 1e3;
|
|
3965
|
+
if (isDelayed) {
|
|
3966
|
+
delayed++;
|
|
3967
|
+
} else {
|
|
3968
|
+
pending++;
|
|
3969
|
+
}
|
|
3970
|
+
}
|
|
3971
|
+
return {
|
|
3972
|
+
queue,
|
|
3973
|
+
size: pending,
|
|
3974
|
+
delayed,
|
|
3975
|
+
failed: this.queues.get(`failed:${queue}`)?.length || 0
|
|
3976
|
+
};
|
|
3977
|
+
}
|
|
1481
3978
|
/**
|
|
1482
3979
|
* Push multiple jobs.
|
|
1483
3980
|
*/
|
|
@@ -1612,6 +4109,35 @@ var Job = class {
|
|
|
1612
4109
|
}
|
|
1613
4110
|
};
|
|
1614
4111
|
|
|
4112
|
+
// src/serializers/CachedSerializer.ts
|
|
4113
|
+
var CachedSerializer = class {
|
|
4114
|
+
/**
|
|
4115
|
+
* @param delegate - The actual serializer to use for the first serialization
|
|
4116
|
+
*/
|
|
4117
|
+
constructor(delegate) {
|
|
4118
|
+
this.delegate = delegate;
|
|
4119
|
+
}
|
|
4120
|
+
cache = /* @__PURE__ */ new WeakMap();
|
|
4121
|
+
/**
|
|
4122
|
+
* Serialize a job with caching.
|
|
4123
|
+
*/
|
|
4124
|
+
serialize(job) {
|
|
4125
|
+
if (this.cache.has(job)) {
|
|
4126
|
+
return this.cache.get(job);
|
|
4127
|
+
}
|
|
4128
|
+
const serialized = this.delegate.serialize(job);
|
|
4129
|
+
this.cache.set(job, serialized);
|
|
4130
|
+
return serialized;
|
|
4131
|
+
}
|
|
4132
|
+
/**
|
|
4133
|
+
* Deserialize a job.
|
|
4134
|
+
* No caching for deserialization as we get new objects each time from the driver.
|
|
4135
|
+
*/
|
|
4136
|
+
deserialize(serialized) {
|
|
4137
|
+
return this.delegate.deserialize(serialized);
|
|
4138
|
+
}
|
|
4139
|
+
};
|
|
4140
|
+
|
|
1615
4141
|
// src/serializers/ClassNameSerializer.ts
|
|
1616
4142
|
var ClassNameSerializer = class {
|
|
1617
4143
|
/**
|
|
@@ -1638,7 +4164,7 @@ var ClassNameSerializer = class {
|
|
|
1638
4164
|
* Serialize a Job.
|
|
1639
4165
|
*/
|
|
1640
4166
|
serialize(job) {
|
|
1641
|
-
const id = job.id || `${Date.now()}-${
|
|
4167
|
+
const id = job.id || `${Date.now()}-${crypto.randomUUID()}`;
|
|
1642
4168
|
const className = job.constructor.name;
|
|
1643
4169
|
const properties = {};
|
|
1644
4170
|
for (const key in job) {
|
|
@@ -1650,10 +4176,7 @@ var ClassNameSerializer = class {
|
|
|
1650
4176
|
id,
|
|
1651
4177
|
type: "class",
|
|
1652
4178
|
className,
|
|
1653
|
-
data: JSON.stringify(
|
|
1654
|
-
class: className,
|
|
1655
|
-
properties
|
|
1656
|
-
}),
|
|
4179
|
+
data: JSON.stringify(properties),
|
|
1657
4180
|
createdAt: Date.now(),
|
|
1658
4181
|
...job.delaySeconds !== void 0 ? { delaySeconds: job.delaySeconds } : {},
|
|
1659
4182
|
attempts: job.attempts ?? 0,
|
|
@@ -1680,10 +4203,10 @@ var ClassNameSerializer = class {
|
|
|
1680
4203
|
`Job class "${serialized.className}" is not registered. Please register it using serializer.register().`
|
|
1681
4204
|
);
|
|
1682
4205
|
}
|
|
1683
|
-
const
|
|
4206
|
+
const properties = JSON.parse(serialized.data);
|
|
1684
4207
|
const job = new JobClass();
|
|
1685
|
-
if (
|
|
1686
|
-
Object.assign(job,
|
|
4208
|
+
if (properties) {
|
|
4209
|
+
Object.assign(job, properties);
|
|
1687
4210
|
}
|
|
1688
4211
|
job.id = serialized.id;
|
|
1689
4212
|
if (serialized.delaySeconds !== void 0) {
|
|
@@ -1717,14 +4240,17 @@ var JsonSerializer = class {
|
|
|
1717
4240
|
* Serialize a job.
|
|
1718
4241
|
*/
|
|
1719
4242
|
serialize(job) {
|
|
1720
|
-
const id = `${Date.now()}-${
|
|
4243
|
+
const id = job.id || `${Date.now()}-${crypto.randomUUID()}`;
|
|
4244
|
+
const properties = {};
|
|
4245
|
+
for (const key in job) {
|
|
4246
|
+
if (Object.hasOwn(job, key) && typeof job[key] !== "function") {
|
|
4247
|
+
properties[key] = job[key];
|
|
4248
|
+
}
|
|
4249
|
+
}
|
|
1721
4250
|
return {
|
|
1722
4251
|
id,
|
|
1723
4252
|
type: "json",
|
|
1724
|
-
data: JSON.stringify(
|
|
1725
|
-
job: job.constructor.name,
|
|
1726
|
-
properties: { ...job }
|
|
1727
|
-
}),
|
|
4253
|
+
data: JSON.stringify(properties),
|
|
1728
4254
|
createdAt: Date.now(),
|
|
1729
4255
|
...job.delaySeconds !== void 0 ? { delaySeconds: job.delaySeconds } : {},
|
|
1730
4256
|
attempts: job.attempts ?? 0,
|
|
@@ -1735,23 +4261,27 @@ var JsonSerializer = class {
|
|
|
1735
4261
|
}
|
|
1736
4262
|
/**
|
|
1737
4263
|
* Deserialize a job.
|
|
1738
|
-
*
|
|
1739
|
-
* Note: this implementation only restores properties and does not recreate class instances.
|
|
1740
|
-
* For class instances, use `ClassNameSerializer`.
|
|
1741
4264
|
*/
|
|
1742
4265
|
deserialize(serialized) {
|
|
1743
4266
|
if (serialized.type !== "json") {
|
|
1744
4267
|
throw new Error('Invalid serialization type: expected "json"');
|
|
1745
4268
|
}
|
|
1746
|
-
const
|
|
4269
|
+
const properties = JSON.parse(serialized.data);
|
|
1747
4270
|
const job = /* @__PURE__ */ Object.create({});
|
|
1748
|
-
Object.assign(job,
|
|
4271
|
+
Object.assign(job, properties);
|
|
4272
|
+
job.id = serialized.id;
|
|
1749
4273
|
if (serialized.groupId) {
|
|
1750
4274
|
job.groupId = serialized.groupId;
|
|
1751
4275
|
}
|
|
1752
4276
|
if (serialized.priority) {
|
|
1753
4277
|
job.priority = serialized.priority;
|
|
1754
4278
|
}
|
|
4279
|
+
if (serialized.delaySeconds !== void 0) {
|
|
4280
|
+
job.delaySeconds = serialized.delaySeconds;
|
|
4281
|
+
}
|
|
4282
|
+
if (serialized.attempts !== void 0) {
|
|
4283
|
+
job.attempts = serialized.attempts;
|
|
4284
|
+
}
|
|
1755
4285
|
return job;
|
|
1756
4286
|
}
|
|
1757
4287
|
};
|
|
@@ -1764,16 +4294,30 @@ var QueueManager = class {
|
|
|
1764
4294
|
defaultSerializer;
|
|
1765
4295
|
persistence;
|
|
1766
4296
|
scheduler;
|
|
1767
|
-
|
|
4297
|
+
debug;
|
|
1768
4298
|
constructor(config = {}) {
|
|
1769
4299
|
this.persistence = config.persistence;
|
|
1770
4300
|
this.defaultConnection = config.default ?? "default";
|
|
4301
|
+
this.debug = config.debug ?? false;
|
|
4302
|
+
if (this.persistence && (this.persistence.bufferSize || this.persistence.flushInterval)) {
|
|
4303
|
+
const { BufferedPersistence: BufferedPersistence2 } = (init_BufferedPersistence(), __toCommonJS(BufferedPersistence_exports));
|
|
4304
|
+
this.persistence.adapter = new BufferedPersistence2(this.persistence.adapter, {
|
|
4305
|
+
maxBufferSize: this.persistence.bufferSize,
|
|
4306
|
+
flushInterval: this.persistence.flushInterval
|
|
4307
|
+
});
|
|
4308
|
+
}
|
|
1771
4309
|
const serializerType = config.defaultSerializer ?? "class";
|
|
1772
4310
|
if (serializerType === "class") {
|
|
1773
4311
|
this.defaultSerializer = new ClassNameSerializer();
|
|
4312
|
+
} else if (serializerType === "msgpack") {
|
|
4313
|
+
const { MessagePackSerializer: MessagePackSerializer2 } = (init_MessagePackSerializer(), __toCommonJS(MessagePackSerializer_exports));
|
|
4314
|
+
this.defaultSerializer = new MessagePackSerializer2();
|
|
1774
4315
|
} else {
|
|
1775
4316
|
this.defaultSerializer = new JsonSerializer();
|
|
1776
4317
|
}
|
|
4318
|
+
if (config.useSerializationCache) {
|
|
4319
|
+
this.defaultSerializer = new CachedSerializer(this.defaultSerializer);
|
|
4320
|
+
}
|
|
1777
4321
|
if (!this.drivers.has("default")) {
|
|
1778
4322
|
this.drivers.set("default", new MemoryDriver());
|
|
1779
4323
|
}
|
|
@@ -1783,6 +4327,20 @@ var QueueManager = class {
|
|
|
1783
4327
|
}
|
|
1784
4328
|
}
|
|
1785
4329
|
}
|
|
4330
|
+
/**
|
|
4331
|
+
* Log debug message.
|
|
4332
|
+
*/
|
|
4333
|
+
log(message, data) {
|
|
4334
|
+
if (this.debug) {
|
|
4335
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
4336
|
+
const prefix = `[QueueManager] [${timestamp}]`;
|
|
4337
|
+
if (data) {
|
|
4338
|
+
console.log(prefix, message, data);
|
|
4339
|
+
} else {
|
|
4340
|
+
console.log(prefix, message);
|
|
4341
|
+
}
|
|
4342
|
+
}
|
|
4343
|
+
}
|
|
1786
4344
|
/**
|
|
1787
4345
|
* Register a connection.
|
|
1788
4346
|
* @param name - Connection name
|
|
@@ -1796,8 +4354,7 @@ var QueueManager = class {
|
|
|
1796
4354
|
break;
|
|
1797
4355
|
case "database": {
|
|
1798
4356
|
const { DatabaseDriver: DatabaseDriver2 } = (init_DatabaseDriver(), __toCommonJS(DatabaseDriver_exports));
|
|
1799
|
-
|
|
1800
|
-
if (!dbService) {
|
|
4357
|
+
if (!config.dbService) {
|
|
1801
4358
|
throw new Error(
|
|
1802
4359
|
"[QueueManager] DatabaseDriver requires dbService. Please provide a database service that implements DatabaseService interface."
|
|
1803
4360
|
);
|
|
@@ -1805,9 +4362,7 @@ var QueueManager = class {
|
|
|
1805
4362
|
this.drivers.set(
|
|
1806
4363
|
name,
|
|
1807
4364
|
new DatabaseDriver2({
|
|
1808
|
-
|
|
1809
|
-
dbService,
|
|
1810
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4365
|
+
dbService: config.dbService,
|
|
1811
4366
|
table: config.table
|
|
1812
4367
|
})
|
|
1813
4368
|
);
|
|
@@ -1815,8 +4370,7 @@ var QueueManager = class {
|
|
|
1815
4370
|
}
|
|
1816
4371
|
case "redis": {
|
|
1817
4372
|
const { RedisDriver: RedisDriver2 } = (init_RedisDriver(), __toCommonJS(RedisDriver_exports));
|
|
1818
|
-
|
|
1819
|
-
if (!client) {
|
|
4373
|
+
if (!config.client) {
|
|
1820
4374
|
throw new Error(
|
|
1821
4375
|
"[QueueManager] RedisDriver requires client. Please provide Redis client in connection config."
|
|
1822
4376
|
);
|
|
@@ -1824,9 +4378,7 @@ var QueueManager = class {
|
|
|
1824
4378
|
this.drivers.set(
|
|
1825
4379
|
name,
|
|
1826
4380
|
new RedisDriver2({
|
|
1827
|
-
|
|
1828
|
-
client,
|
|
1829
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4381
|
+
client: config.client,
|
|
1830
4382
|
prefix: config.prefix
|
|
1831
4383
|
})
|
|
1832
4384
|
);
|
|
@@ -1834,8 +4386,7 @@ var QueueManager = class {
|
|
|
1834
4386
|
}
|
|
1835
4387
|
case "kafka": {
|
|
1836
4388
|
const { KafkaDriver: KafkaDriver2 } = (init_KafkaDriver(), __toCommonJS(KafkaDriver_exports));
|
|
1837
|
-
|
|
1838
|
-
if (!client) {
|
|
4389
|
+
if (!config.client) {
|
|
1839
4390
|
throw new Error(
|
|
1840
4391
|
"[QueueManager] KafkaDriver requires client. Please provide Kafka client in connection config."
|
|
1841
4392
|
);
|
|
@@ -1843,9 +4394,7 @@ var QueueManager = class {
|
|
|
1843
4394
|
this.drivers.set(
|
|
1844
4395
|
name,
|
|
1845
4396
|
new KafkaDriver2({
|
|
1846
|
-
|
|
1847
|
-
client,
|
|
1848
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4397
|
+
client: config.client,
|
|
1849
4398
|
consumerGroupId: config.consumerGroupId
|
|
1850
4399
|
})
|
|
1851
4400
|
);
|
|
@@ -1853,8 +4402,7 @@ var QueueManager = class {
|
|
|
1853
4402
|
}
|
|
1854
4403
|
case "sqs": {
|
|
1855
4404
|
const { SQSDriver: SQSDriver2 } = (init_SQSDriver(), __toCommonJS(SQSDriver_exports));
|
|
1856
|
-
|
|
1857
|
-
if (!client) {
|
|
4405
|
+
if (!config.client) {
|
|
1858
4406
|
throw new Error(
|
|
1859
4407
|
"[QueueManager] SQSDriver requires client. Please provide SQS client in connection config."
|
|
1860
4408
|
);
|
|
@@ -1862,13 +4410,9 @@ var QueueManager = class {
|
|
|
1862
4410
|
this.drivers.set(
|
|
1863
4411
|
name,
|
|
1864
4412
|
new SQSDriver2({
|
|
1865
|
-
|
|
1866
|
-
client,
|
|
1867
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4413
|
+
client: config.client,
|
|
1868
4414
|
queueUrlPrefix: config.queueUrlPrefix,
|
|
1869
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
1870
4415
|
visibilityTimeout: config.visibilityTimeout,
|
|
1871
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
1872
4416
|
waitTimeSeconds: config.waitTimeSeconds
|
|
1873
4417
|
})
|
|
1874
4418
|
);
|
|
@@ -1876,8 +4420,7 @@ var QueueManager = class {
|
|
|
1876
4420
|
}
|
|
1877
4421
|
case "rabbitmq": {
|
|
1878
4422
|
const { RabbitMQDriver: RabbitMQDriver2 } = (init_RabbitMQDriver(), __toCommonJS(RabbitMQDriver_exports));
|
|
1879
|
-
|
|
1880
|
-
if (!client) {
|
|
4423
|
+
if (!config.client) {
|
|
1881
4424
|
throw new Error(
|
|
1882
4425
|
"[QueueManager] RabbitMQDriver requires client. Please provide RabbitMQ connection/channel in connection config."
|
|
1883
4426
|
);
|
|
@@ -1885,11 +4428,8 @@ var QueueManager = class {
|
|
|
1885
4428
|
this.drivers.set(
|
|
1886
4429
|
name,
|
|
1887
4430
|
new RabbitMQDriver2({
|
|
1888
|
-
|
|
1889
|
-
client,
|
|
1890
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4431
|
+
client: config.client,
|
|
1891
4432
|
exchange: config.exchange,
|
|
1892
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
1893
4433
|
exchangeType: config.exchangeType
|
|
1894
4434
|
})
|
|
1895
4435
|
);
|
|
@@ -1968,6 +4508,11 @@ var QueueManager = class {
|
|
|
1968
4508
|
pushOptions.priority = job.priority;
|
|
1969
4509
|
}
|
|
1970
4510
|
await driver.push(queue, serialized, pushOptions);
|
|
4511
|
+
this.log(`Pushed job to ${queue} (${connection})`, {
|
|
4512
|
+
id: serialized.id,
|
|
4513
|
+
job: serialized.className ?? "json",
|
|
4514
|
+
options: pushOptions
|
|
4515
|
+
});
|
|
1971
4516
|
if (this.persistence?.archiveEnqueued) {
|
|
1972
4517
|
this.persistence.adapter.archive(queue, serialized, "waiting").catch((err) => {
|
|
1973
4518
|
console.error("[QueueManager] Persistence archive failed (waiting):", err);
|
|
@@ -1980,16 +4525,19 @@ var QueueManager = class {
|
|
|
1980
4525
|
*
|
|
1981
4526
|
* @template T - The type of the jobs.
|
|
1982
4527
|
* @param jobs - Array of job instances.
|
|
4528
|
+
* @param options - Bulk push options.
|
|
1983
4529
|
*
|
|
1984
4530
|
* @example
|
|
1985
4531
|
* ```typescript
|
|
1986
|
-
* await manager.pushMany(
|
|
4532
|
+
* await manager.pushMany(jobs, { batchSize: 500, concurrency: 2 });
|
|
1987
4533
|
* ```
|
|
1988
4534
|
*/
|
|
1989
|
-
async pushMany(jobs) {
|
|
4535
|
+
async pushMany(jobs, options = {}) {
|
|
1990
4536
|
if (jobs.length === 0) {
|
|
1991
4537
|
return;
|
|
1992
4538
|
}
|
|
4539
|
+
const batchSize = options.batchSize ?? 100;
|
|
4540
|
+
const concurrency = options.concurrency ?? 1;
|
|
1993
4541
|
const groups = /* @__PURE__ */ new Map();
|
|
1994
4542
|
const serializer = this.getSerializer();
|
|
1995
4543
|
for (const job of jobs) {
|
|
@@ -2002,17 +4550,42 @@ var QueueManager = class {
|
|
|
2002
4550
|
}
|
|
2003
4551
|
groups.get(key)?.push(serialized);
|
|
2004
4552
|
}
|
|
4553
|
+
const processBatch = async (driver, queue, batch) => {
|
|
4554
|
+
if (driver.pushMany) {
|
|
4555
|
+
await driver.pushMany(queue, batch);
|
|
4556
|
+
} else {
|
|
4557
|
+
for (const job of batch) {
|
|
4558
|
+
await driver.push(queue, job);
|
|
4559
|
+
}
|
|
4560
|
+
}
|
|
4561
|
+
};
|
|
2005
4562
|
for (const [key, serializedJobs] of groups.entries()) {
|
|
2006
4563
|
const [connection, queue] = key.split(":");
|
|
2007
4564
|
if (!connection || !queue) {
|
|
2008
4565
|
continue;
|
|
2009
4566
|
}
|
|
2010
4567
|
const driver = this.getDriver(connection);
|
|
2011
|
-
|
|
2012
|
-
|
|
4568
|
+
this.log(`Pushing ${serializedJobs.length} jobs to ${queue} (${connection})`);
|
|
4569
|
+
const chunks = [];
|
|
4570
|
+
for (let i = 0; i < serializedJobs.length; i += batchSize) {
|
|
4571
|
+
chunks.push(serializedJobs.slice(i, i + batchSize));
|
|
4572
|
+
}
|
|
4573
|
+
if (concurrency > 1) {
|
|
4574
|
+
const activePromises = [];
|
|
4575
|
+
for (const chunk of chunks) {
|
|
4576
|
+
const promise = processBatch(driver, queue, chunk);
|
|
4577
|
+
activePromises.push(promise);
|
|
4578
|
+
if (activePromises.length >= concurrency) {
|
|
4579
|
+
await Promise.race(activePromises);
|
|
4580
|
+
}
|
|
4581
|
+
}
|
|
4582
|
+
for (let i = 0; i < chunks.length; i += concurrency) {
|
|
4583
|
+
const batchPromises = chunks.slice(i, i + concurrency).map((chunk) => processBatch(driver, queue, chunk));
|
|
4584
|
+
await Promise.all(batchPromises);
|
|
4585
|
+
}
|
|
2013
4586
|
} else {
|
|
2014
|
-
for (const
|
|
2015
|
-
await driver
|
|
4587
|
+
for (const chunk of chunks) {
|
|
4588
|
+
await processBatch(driver, queue, chunk);
|
|
2016
4589
|
}
|
|
2017
4590
|
}
|
|
2018
4591
|
}
|
|
@@ -2037,6 +4610,7 @@ var QueueManager = class {
|
|
|
2037
4610
|
if (!serialized) {
|
|
2038
4611
|
return null;
|
|
2039
4612
|
}
|
|
4613
|
+
this.log(`Popped job from ${queue} (${connection})`, { id: serialized.id });
|
|
2040
4614
|
try {
|
|
2041
4615
|
return serializer.deserialize(serialized);
|
|
2042
4616
|
} catch (error) {
|
|
@@ -2044,6 +4618,42 @@ var QueueManager = class {
|
|
|
2044
4618
|
return null;
|
|
2045
4619
|
}
|
|
2046
4620
|
}
|
|
4621
|
+
/**
|
|
4622
|
+
* Pop multiple jobs from the queue.
|
|
4623
|
+
*
|
|
4624
|
+
* @param queue - Queue name (default: 'default').
|
|
4625
|
+
* @param count - Number of jobs to pop (default: 10).
|
|
4626
|
+
* @param connection - Connection name (optional).
|
|
4627
|
+
* @returns Array of Job instances.
|
|
4628
|
+
*/
|
|
4629
|
+
async popMany(queue = "default", count = 10, connection = this.defaultConnection) {
|
|
4630
|
+
const driver = this.getDriver(connection);
|
|
4631
|
+
const serializer = this.getSerializer();
|
|
4632
|
+
const results = [];
|
|
4633
|
+
if (driver.popMany) {
|
|
4634
|
+
const serializedJobs = await driver.popMany(queue, count);
|
|
4635
|
+
if (serializedJobs.length > 0) {
|
|
4636
|
+
this.log(`Popped ${serializedJobs.length} jobs from ${queue} (${connection})`);
|
|
4637
|
+
}
|
|
4638
|
+
for (const serialized of serializedJobs) {
|
|
4639
|
+
try {
|
|
4640
|
+
results.push(serializer.deserialize(serialized));
|
|
4641
|
+
} catch (error) {
|
|
4642
|
+
console.error("[QueueManager] Failed to deserialize job:", error);
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
4645
|
+
} else {
|
|
4646
|
+
for (let i = 0; i < count; i++) {
|
|
4647
|
+
const job = await this.pop(queue, connection);
|
|
4648
|
+
if (job) {
|
|
4649
|
+
results.push(job);
|
|
4650
|
+
} else {
|
|
4651
|
+
break;
|
|
4652
|
+
}
|
|
4653
|
+
}
|
|
4654
|
+
}
|
|
4655
|
+
return results;
|
|
4656
|
+
}
|
|
2047
4657
|
/**
|
|
2048
4658
|
* Get queue size.
|
|
2049
4659
|
*
|
|
@@ -2055,6 +4665,35 @@ var QueueManager = class {
|
|
|
2055
4665
|
const driver = this.getDriver(connection);
|
|
2056
4666
|
return driver.size(queue);
|
|
2057
4667
|
}
|
|
4668
|
+
/**
|
|
4669
|
+
* Pop a job from the queue (blocking).
|
|
4670
|
+
*
|
|
4671
|
+
* @param queue - Queue name (default: 'default').
|
|
4672
|
+
* @param timeout - Timeout in seconds (default: 0, wait forever).
|
|
4673
|
+
* @param connection - Connection name (optional).
|
|
4674
|
+
*/
|
|
4675
|
+
async popBlocking(queues = "default", timeout = 0, connection = this.defaultConnection) {
|
|
4676
|
+
const driver = this.getDriver(connection);
|
|
4677
|
+
const serializer = this.getSerializer();
|
|
4678
|
+
if (!driver.popBlocking) {
|
|
4679
|
+
const q = Array.isArray(queues) ? queues[0] : queues;
|
|
4680
|
+
return this.pop(q, connection);
|
|
4681
|
+
}
|
|
4682
|
+
const serialized = await driver.popBlocking(queues, timeout);
|
|
4683
|
+
if (!serialized) {
|
|
4684
|
+
return null;
|
|
4685
|
+
}
|
|
4686
|
+
this.log(
|
|
4687
|
+
`Popped job (blocking) from ${Array.isArray(queues) ? queues.join(",") : queues} (${connection})`,
|
|
4688
|
+
{ id: serialized.id }
|
|
4689
|
+
);
|
|
4690
|
+
try {
|
|
4691
|
+
return serializer.deserialize(serialized);
|
|
4692
|
+
} catch (error) {
|
|
4693
|
+
console.error("[QueueManager] Failed to deserialize job:", error);
|
|
4694
|
+
return null;
|
|
4695
|
+
}
|
|
4696
|
+
}
|
|
2058
4697
|
/**
|
|
2059
4698
|
* Clear all jobs from a queue.
|
|
2060
4699
|
*
|
|
@@ -2065,6 +4704,23 @@ var QueueManager = class {
|
|
|
2065
4704
|
const driver = this.getDriver(connection);
|
|
2066
4705
|
await driver.clear(queue);
|
|
2067
4706
|
}
|
|
4707
|
+
/**
|
|
4708
|
+
* Get queue statistics including size, delayed, and failed job counts.
|
|
4709
|
+
*
|
|
4710
|
+
* @param queue - Queue name (default: 'default').
|
|
4711
|
+
* @param connection - Connection name (optional).
|
|
4712
|
+
* @returns Queue statistics object.
|
|
4713
|
+
*/
|
|
4714
|
+
async stats(queue = "default", connection = this.defaultConnection) {
|
|
4715
|
+
const driver = this.getDriver(connection);
|
|
4716
|
+
if (driver.stats) {
|
|
4717
|
+
return await driver.stats(queue);
|
|
4718
|
+
}
|
|
4719
|
+
return {
|
|
4720
|
+
queue,
|
|
4721
|
+
size: await driver.size(queue)
|
|
4722
|
+
};
|
|
4723
|
+
}
|
|
2068
4724
|
/**
|
|
2069
4725
|
* Mark a job as completed.
|
|
2070
4726
|
* @param job - Job instance
|
|
@@ -2077,6 +4733,7 @@ var QueueManager = class {
|
|
|
2077
4733
|
if (driver.complete) {
|
|
2078
4734
|
const serialized = serializer.serialize(job);
|
|
2079
4735
|
await driver.complete(queue, serialized);
|
|
4736
|
+
this.log(`Completed job ${job.id} in ${queue}`);
|
|
2080
4737
|
if (this.persistence?.archiveCompleted) {
|
|
2081
4738
|
await this.persistence.adapter.archive(queue, serialized, "completed").catch((err) => {
|
|
2082
4739
|
console.error("[QueueManager] Persistence archive failed (completed):", err);
|
|
@@ -2099,6 +4756,7 @@ var QueueManager = class {
|
|
|
2099
4756
|
serialized.error = error.message;
|
|
2100
4757
|
serialized.failedAt = Date.now();
|
|
2101
4758
|
await driver.fail(queue, serialized);
|
|
4759
|
+
this.log(`Failed job ${job.id} in ${queue}`, { error: error.message });
|
|
2102
4760
|
if (this.persistence?.archiveFailed) {
|
|
2103
4761
|
await this.persistence.adapter.archive(queue, serialized, "failed").catch((err) => {
|
|
2104
4762
|
console.error("[QueueManager] Persistence archive failed (failed):", err);
|
|
@@ -2232,39 +4890,51 @@ var OrbitStream = class _OrbitStream {
|
|
|
2232
4890
|
}
|
|
2233
4891
|
};
|
|
2234
4892
|
|
|
4893
|
+
// src/index.ts
|
|
4894
|
+
init_BufferedPersistence();
|
|
4895
|
+
|
|
2235
4896
|
// src/persistence/MySQLPersistence.ts
|
|
2236
4897
|
import { DB, Schema } from "@gravito/atlas";
|
|
2237
4898
|
var MySQLPersistence = class {
|
|
2238
4899
|
/**
|
|
2239
4900
|
* @param db - An Atlas DB instance or compatible QueryBuilder.
|
|
2240
4901
|
* @param table - The name of the table to store archived jobs.
|
|
4902
|
+
* @param logsTable - The name of the table to store system logs.
|
|
4903
|
+
* @param options - Buffering options (Deprecated: Use BufferedPersistence wrapper instead).
|
|
2241
4904
|
*/
|
|
2242
|
-
constructor(db, table = "flux_job_archive", logsTable = "flux_system_logs") {
|
|
4905
|
+
constructor(db, table = "flux_job_archive", logsTable = "flux_system_logs", _options = {}) {
|
|
2243
4906
|
this.db = db;
|
|
2244
4907
|
this.table = table;
|
|
2245
4908
|
this.logsTable = logsTable;
|
|
2246
4909
|
}
|
|
2247
|
-
/**
|
|
2248
|
-
* Archive a job.
|
|
2249
|
-
*/
|
|
2250
4910
|
async archive(queue, job, status) {
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
4911
|
+
await this.archiveMany([{ queue, job, status }]);
|
|
4912
|
+
}
|
|
4913
|
+
async archiveMany(jobs) {
|
|
4914
|
+
if (jobs.length === 0) {
|
|
4915
|
+
return;
|
|
4916
|
+
}
|
|
4917
|
+
const batchSize = 500;
|
|
4918
|
+
for (let i = 0; i < jobs.length; i += batchSize) {
|
|
4919
|
+
const chunk = jobs.slice(i, i + batchSize);
|
|
4920
|
+
try {
|
|
4921
|
+
const records = chunk.map((item) => ({
|
|
4922
|
+
job_id: item.job.id,
|
|
4923
|
+
queue: item.queue,
|
|
4924
|
+
status: item.status,
|
|
4925
|
+
payload: JSON.stringify(item.job),
|
|
4926
|
+
error: item.job.error || null,
|
|
4927
|
+
created_at: new Date(item.job.createdAt),
|
|
4928
|
+
archived_at: /* @__PURE__ */ new Date()
|
|
4929
|
+
}));
|
|
4930
|
+
await this.db.table(this.table).insert(records);
|
|
4931
|
+
} catch (err) {
|
|
4932
|
+
console.error(`[MySQLPersistence] Failed to archive ${chunk.length} jobs:`, err);
|
|
4933
|
+
}
|
|
2263
4934
|
}
|
|
2264
4935
|
}
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
*/
|
|
4936
|
+
async flush() {
|
|
4937
|
+
}
|
|
2268
4938
|
async find(queue, id) {
|
|
2269
4939
|
const row = await this.db.table(this.table).where("queue", queue).where("job_id", id).first();
|
|
2270
4940
|
if (!row) {
|
|
@@ -2302,7 +4972,9 @@ var MySQLPersistence = class {
|
|
|
2302
4972
|
} catch (_e) {
|
|
2303
4973
|
return null;
|
|
2304
4974
|
}
|
|
2305
|
-
}).filter(
|
|
4975
|
+
}).filter(
|
|
4976
|
+
(item) => !!item
|
|
4977
|
+
);
|
|
2306
4978
|
}
|
|
2307
4979
|
/**
|
|
2308
4980
|
* Search jobs from the archive.
|
|
@@ -2322,22 +4994,35 @@ var MySQLPersistence = class {
|
|
|
2322
4994
|
} catch (_e) {
|
|
2323
4995
|
return null;
|
|
2324
4996
|
}
|
|
2325
|
-
}).filter(
|
|
4997
|
+
}).filter(
|
|
4998
|
+
(item) => !!item
|
|
4999
|
+
);
|
|
2326
5000
|
}
|
|
2327
5001
|
/**
|
|
2328
|
-
* Archive a system log message.
|
|
5002
|
+
* Archive a system log message (buffered).
|
|
2329
5003
|
*/
|
|
2330
5004
|
async archiveLog(log) {
|
|
5005
|
+
await this.archiveLogMany([log]);
|
|
5006
|
+
}
|
|
5007
|
+
/**
|
|
5008
|
+
* Archive multiple log messages (direct batch write).
|
|
5009
|
+
*/
|
|
5010
|
+
async archiveLogMany(logs) {
|
|
5011
|
+
if (logs.length === 0) {
|
|
5012
|
+
return;
|
|
5013
|
+
}
|
|
2331
5014
|
try {
|
|
2332
|
-
|
|
5015
|
+
const records = logs.map((log) => ({
|
|
2333
5016
|
level: log.level,
|
|
2334
5017
|
message: log.message,
|
|
2335
5018
|
worker_id: log.workerId,
|
|
2336
5019
|
queue: log.queue || null,
|
|
2337
5020
|
timestamp: log.timestamp
|
|
2338
|
-
});
|
|
5021
|
+
}));
|
|
5022
|
+
await this.db.table(this.logsTable).insert(records);
|
|
2339
5023
|
} catch (err) {
|
|
2340
|
-
|
|
5024
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
5025
|
+
console.error(`[MySQLPersistence] Failed to archive ${logs.length} logs:`, error.message);
|
|
2341
5026
|
}
|
|
2342
5027
|
}
|
|
2343
5028
|
/**
|
|
@@ -2388,8 +5073,8 @@ var MySQLPersistence = class {
|
|
|
2388
5073
|
if (options.endTime) {
|
|
2389
5074
|
query = query.where("timestamp", "<=", options.endTime);
|
|
2390
5075
|
}
|
|
2391
|
-
const result = await query.count(
|
|
2392
|
-
return result
|
|
5076
|
+
const result = await query.count();
|
|
5077
|
+
return Number(result) || 0;
|
|
2393
5078
|
}
|
|
2394
5079
|
/**
|
|
2395
5080
|
* Remove old records from the archive.
|
|
@@ -2401,7 +5086,7 @@ var MySQLPersistence = class {
|
|
|
2401
5086
|
this.db.table(this.table).where("archived_at", "<", threshold).delete(),
|
|
2402
5087
|
this.db.table(this.logsTable).where("timestamp", "<", threshold).delete()
|
|
2403
5088
|
]);
|
|
2404
|
-
return (jobsDeleted || 0) + (logsDeleted || 0);
|
|
5089
|
+
return (Number(jobsDeleted) || 0) + (Number(logsDeleted) || 0);
|
|
2405
5090
|
}
|
|
2406
5091
|
/**
|
|
2407
5092
|
* Count jobs in the archive.
|
|
@@ -2420,8 +5105,8 @@ var MySQLPersistence = class {
|
|
|
2420
5105
|
if (options.endTime) {
|
|
2421
5106
|
query = query.where("archived_at", "<=", options.endTime);
|
|
2422
5107
|
}
|
|
2423
|
-
const result = await query.count(
|
|
2424
|
-
return result
|
|
5108
|
+
const result = await query.count();
|
|
5109
|
+
return Number(result) || 0;
|
|
2425
5110
|
}
|
|
2426
5111
|
/**
|
|
2427
5112
|
* Help script to create the necessary table.
|
|
@@ -2477,33 +5162,49 @@ var SQLitePersistence = class {
|
|
|
2477
5162
|
/**
|
|
2478
5163
|
* @param db - An Atlas DB instance (SQLite driver).
|
|
2479
5164
|
* @param table - The name of the table to store archived jobs.
|
|
5165
|
+
* @param logsTable - The name of the table to store system logs.
|
|
5166
|
+
* @param options - Buffering options (Deprecated: Use BufferedPersistence wrapper instead).
|
|
2480
5167
|
*/
|
|
2481
|
-
constructor(db, table = "flux_job_archive", logsTable = "flux_system_logs") {
|
|
5168
|
+
constructor(db, table = "flux_job_archive", logsTable = "flux_system_logs", _options = {}) {
|
|
2482
5169
|
this.db = db;
|
|
2483
5170
|
this.table = table;
|
|
2484
5171
|
this.logsTable = logsTable;
|
|
2485
5172
|
}
|
|
2486
|
-
/**
|
|
2487
|
-
* Archive a job.
|
|
2488
|
-
*/
|
|
2489
5173
|
async archive(queue, job, status) {
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
5174
|
+
await this.archiveMany([{ queue, job, status }]);
|
|
5175
|
+
}
|
|
5176
|
+
async archiveMany(jobs) {
|
|
5177
|
+
if (jobs.length === 0) {
|
|
5178
|
+
return;
|
|
5179
|
+
}
|
|
5180
|
+
const batchSize = 200;
|
|
5181
|
+
for (let i = 0; i < jobs.length; i += batchSize) {
|
|
5182
|
+
const chunk = jobs.slice(i, i + batchSize);
|
|
5183
|
+
try {
|
|
5184
|
+
const records = chunk.map((item) => ({
|
|
5185
|
+
job_id: item.job.id,
|
|
5186
|
+
queue: item.queue,
|
|
5187
|
+
status: item.status,
|
|
5188
|
+
payload: JSON.stringify(item.job),
|
|
5189
|
+
error: item.job.error || null,
|
|
5190
|
+
created_at: new Date(item.job.createdAt),
|
|
5191
|
+
archived_at: /* @__PURE__ */ new Date()
|
|
5192
|
+
}));
|
|
5193
|
+
if (typeof this.db.transaction === "function") {
|
|
5194
|
+
await this.db.transaction(async (trx) => {
|
|
5195
|
+
await trx.table(this.table).insert(records);
|
|
5196
|
+
});
|
|
5197
|
+
} else {
|
|
5198
|
+
await this.db.table(this.table).insert(records);
|
|
5199
|
+
}
|
|
5200
|
+
} catch (err) {
|
|
5201
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
5202
|
+
console.error(`[SQLitePersistence] Failed to archive ${chunk.length} jobs:`, error.message);
|
|
5203
|
+
}
|
|
2502
5204
|
}
|
|
2503
5205
|
}
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
*/
|
|
5206
|
+
async flush() {
|
|
5207
|
+
}
|
|
2507
5208
|
async find(queue, id) {
|
|
2508
5209
|
const row = await this.db.table(this.table).where("queue", queue).where("job_id", id).first();
|
|
2509
5210
|
if (!row) {
|
|
@@ -2541,7 +5242,9 @@ var SQLitePersistence = class {
|
|
|
2541
5242
|
} catch (_e) {
|
|
2542
5243
|
return null;
|
|
2543
5244
|
}
|
|
2544
|
-
}).filter(
|
|
5245
|
+
}).filter(
|
|
5246
|
+
(item) => !!item
|
|
5247
|
+
);
|
|
2545
5248
|
}
|
|
2546
5249
|
/**
|
|
2547
5250
|
* Search jobs from the archive.
|
|
@@ -2561,22 +5264,35 @@ var SQLitePersistence = class {
|
|
|
2561
5264
|
} catch (_e) {
|
|
2562
5265
|
return null;
|
|
2563
5266
|
}
|
|
2564
|
-
}).filter(
|
|
5267
|
+
}).filter(
|
|
5268
|
+
(item) => !!item
|
|
5269
|
+
);
|
|
2565
5270
|
}
|
|
2566
5271
|
/**
|
|
2567
|
-
* Archive a system log message.
|
|
5272
|
+
* Archive a system log message (buffered).
|
|
2568
5273
|
*/
|
|
2569
5274
|
async archiveLog(log) {
|
|
5275
|
+
await this.archiveLogMany([log]);
|
|
5276
|
+
}
|
|
5277
|
+
/**
|
|
5278
|
+
* Archive multiple log messages (direct batch write).
|
|
5279
|
+
*/
|
|
5280
|
+
async archiveLogMany(logs) {
|
|
5281
|
+
if (logs.length === 0) {
|
|
5282
|
+
return;
|
|
5283
|
+
}
|
|
2570
5284
|
try {
|
|
2571
|
-
|
|
5285
|
+
const records = logs.map((log) => ({
|
|
2572
5286
|
level: log.level,
|
|
2573
5287
|
message: log.message,
|
|
2574
5288
|
worker_id: log.workerId,
|
|
2575
5289
|
queue: log.queue || null,
|
|
2576
5290
|
timestamp: log.timestamp
|
|
2577
|
-
});
|
|
5291
|
+
}));
|
|
5292
|
+
await this.db.table(this.logsTable).insert(records);
|
|
2578
5293
|
} catch (err) {
|
|
2579
|
-
|
|
5294
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
5295
|
+
console.error(`[SQLitePersistence] Failed to archive ${logs.length} logs:`, error.message);
|
|
2580
5296
|
}
|
|
2581
5297
|
}
|
|
2582
5298
|
/**
|
|
@@ -2627,8 +5343,8 @@ var SQLitePersistence = class {
|
|
|
2627
5343
|
if (options.endTime) {
|
|
2628
5344
|
query = query.where("timestamp", "<=", options.endTime);
|
|
2629
5345
|
}
|
|
2630
|
-
const result = await query.count(
|
|
2631
|
-
return result
|
|
5346
|
+
const result = await query.count();
|
|
5347
|
+
return Number(result) || 0;
|
|
2632
5348
|
}
|
|
2633
5349
|
/**
|
|
2634
5350
|
* Remove old records from the archive.
|
|
@@ -2640,7 +5356,7 @@ var SQLitePersistence = class {
|
|
|
2640
5356
|
this.db.table(this.table).where("archived_at", "<", threshold).delete(),
|
|
2641
5357
|
this.db.table(this.logsTable).where("timestamp", "<", threshold).delete()
|
|
2642
5358
|
]);
|
|
2643
|
-
return (jobsDeleted || 0) + (logsDeleted || 0);
|
|
5359
|
+
return (Number(jobsDeleted) || 0) + (Number(logsDeleted) || 0);
|
|
2644
5360
|
}
|
|
2645
5361
|
/**
|
|
2646
5362
|
* Count jobs in the archive.
|
|
@@ -2659,8 +5375,8 @@ var SQLitePersistence = class {
|
|
|
2659
5375
|
if (options.endTime) {
|
|
2660
5376
|
query = query.where("archived_at", "<=", options.endTime);
|
|
2661
5377
|
}
|
|
2662
|
-
const result = await query.count(
|
|
2663
|
-
return result
|
|
5378
|
+
const result = await query.count();
|
|
5379
|
+
return Number(result) || 0;
|
|
2664
5380
|
}
|
|
2665
5381
|
/**
|
|
2666
5382
|
* Setup table for SQLite.
|
|
@@ -2711,6 +5427,7 @@ var SQLitePersistence = class {
|
|
|
2711
5427
|
// src/index.ts
|
|
2712
5428
|
init_Scheduler();
|
|
2713
5429
|
export {
|
|
5430
|
+
BufferedPersistence,
|
|
2714
5431
|
ClassNameSerializer,
|
|
2715
5432
|
Consumer,
|
|
2716
5433
|
DatabaseDriver,
|