@gravito/stream 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +2956 -238
- package/dist/index.d.cts +563 -123
- package/dist/index.d.ts +563 -123
- package/dist/index.js +2955 -238
- package/package.json +3 -2
package/dist/index.cjs
CHANGED
|
@@ -130,6 +130,99 @@ var init_DatabaseDriver = __esm({
|
|
|
130
130
|
}
|
|
131
131
|
return job;
|
|
132
132
|
}
|
|
133
|
+
/**
|
|
134
|
+
* Pop multiple jobs from the queue.
|
|
135
|
+
*/
|
|
136
|
+
async popMany(queue, count) {
|
|
137
|
+
if (count <= 1) {
|
|
138
|
+
const job = await this.pop(queue);
|
|
139
|
+
return job ? [job] : [];
|
|
140
|
+
}
|
|
141
|
+
try {
|
|
142
|
+
const result = await this.dbService.execute(
|
|
143
|
+
`SELECT id, payload, attempts, created_at, available_at
|
|
144
|
+
FROM ${this.tableName}
|
|
145
|
+
WHERE queue = $1
|
|
146
|
+
AND available_at <= NOW()
|
|
147
|
+
AND (reserved_at IS NULL OR reserved_at < NOW() - INTERVAL '5 minutes')
|
|
148
|
+
ORDER BY created_at ASC
|
|
149
|
+
LIMIT ${count}
|
|
150
|
+
FOR UPDATE SKIP LOCKED`,
|
|
151
|
+
[queue]
|
|
152
|
+
);
|
|
153
|
+
const rows = Array.isArray(result) ? result : result ? [result] : [];
|
|
154
|
+
if (!rows || rows.length === 0) {
|
|
155
|
+
return [];
|
|
156
|
+
}
|
|
157
|
+
const validRows = rows.filter((r) => r?.id);
|
|
158
|
+
if (validRows.length === 0) {
|
|
159
|
+
return [];
|
|
160
|
+
}
|
|
161
|
+
const ids = validRows.map((r) => r.id);
|
|
162
|
+
await this.dbService.execute(
|
|
163
|
+
`UPDATE ${this.tableName}
|
|
164
|
+
SET reserved_at = NOW()
|
|
165
|
+
WHERE id IN (${ids.map((_, i) => `$${i + 1}`).join(", ")})`,
|
|
166
|
+
ids
|
|
167
|
+
);
|
|
168
|
+
return validRows.map((row) => {
|
|
169
|
+
const createdAt = new Date(row.created_at).getTime();
|
|
170
|
+
try {
|
|
171
|
+
const parsed = JSON.parse(row.payload);
|
|
172
|
+
return {
|
|
173
|
+
...parsed,
|
|
174
|
+
id: row.id,
|
|
175
|
+
attempts: row.attempts
|
|
176
|
+
};
|
|
177
|
+
} catch (_e) {
|
|
178
|
+
return {
|
|
179
|
+
id: row.id,
|
|
180
|
+
type: "class",
|
|
181
|
+
data: row.payload,
|
|
182
|
+
createdAt,
|
|
183
|
+
attempts: row.attempts
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
});
|
|
187
|
+
} catch (_e) {
|
|
188
|
+
const firstJob = await this.pop(queue);
|
|
189
|
+
return firstJob ? [firstJob] : [];
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Get queue statistics.
|
|
194
|
+
*/
|
|
195
|
+
async stats(queue) {
|
|
196
|
+
const failedQueue = `failed:${queue}`;
|
|
197
|
+
try {
|
|
198
|
+
const pendingRes = await this.dbService.execute(
|
|
199
|
+
`SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue = $1 AND available_at <= NOW() AND reserved_at IS NULL`,
|
|
200
|
+
[queue]
|
|
201
|
+
);
|
|
202
|
+
const delayedRes = await this.dbService.execute(
|
|
203
|
+
`SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue = $1 AND available_at > NOW()`,
|
|
204
|
+
[queue]
|
|
205
|
+
);
|
|
206
|
+
const reservedRes = await this.dbService.execute(
|
|
207
|
+
`SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue = $1 AND reserved_at IS NOT NULL`,
|
|
208
|
+
[queue]
|
|
209
|
+
);
|
|
210
|
+
const failedRes = await this.dbService.execute(
|
|
211
|
+
`SELECT COUNT(*) as count FROM ${this.tableName} WHERE queue = $1`,
|
|
212
|
+
[failedQueue]
|
|
213
|
+
);
|
|
214
|
+
return {
|
|
215
|
+
queue,
|
|
216
|
+
size: pendingRes[0]?.count || 0,
|
|
217
|
+
delayed: delayedRes[0]?.count || 0,
|
|
218
|
+
reserved: reservedRes[0]?.count || 0,
|
|
219
|
+
failed: failedRes[0]?.count || 0
|
|
220
|
+
};
|
|
221
|
+
} catch (err) {
|
|
222
|
+
console.error("[DatabaseDriver] Failed to get stats:", err);
|
|
223
|
+
return { queue, size: 0, delayed: 0, reserved: 0, failed: 0 };
|
|
224
|
+
}
|
|
225
|
+
}
|
|
133
226
|
/**
|
|
134
227
|
* Get queue size.
|
|
135
228
|
*/
|
|
@@ -150,21 +243,44 @@ var init_DatabaseDriver = __esm({
|
|
|
150
243
|
async clear(queue) {
|
|
151
244
|
await this.dbService.execute(`DELETE FROM ${this.tableName} WHERE queue = $1`, [queue]);
|
|
152
245
|
}
|
|
246
|
+
/**
|
|
247
|
+
* Pop a job from the queue (blocking).
|
|
248
|
+
* Simple polling fallback for databases.
|
|
249
|
+
*/
|
|
250
|
+
async popBlocking(queue, timeout) {
|
|
251
|
+
const start = Date.now();
|
|
252
|
+
const timeoutMs = timeout * 1e3;
|
|
253
|
+
while (true) {
|
|
254
|
+
const job = await this.pop(queue);
|
|
255
|
+
if (job) {
|
|
256
|
+
return job;
|
|
257
|
+
}
|
|
258
|
+
if (timeout > 0 && Date.now() - start >= timeoutMs) {
|
|
259
|
+
return null;
|
|
260
|
+
}
|
|
261
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
262
|
+
}
|
|
263
|
+
}
|
|
153
264
|
/**
|
|
154
265
|
* Push multiple jobs.
|
|
266
|
+
* Optimizes by using a single multi-row insert if possible.
|
|
155
267
|
*/
|
|
156
268
|
async pushMany(queue, jobs) {
|
|
157
269
|
if (jobs.length === 0) {
|
|
158
270
|
return;
|
|
159
271
|
}
|
|
160
272
|
await this.dbService.transaction(async (tx) => {
|
|
161
|
-
for (
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
273
|
+
for (let i = 0; i < jobs.length; i += 100) {
|
|
274
|
+
const batch = jobs.slice(i, i + 100);
|
|
275
|
+
for (const job of batch) {
|
|
276
|
+
const availableAt = job.delaySeconds ? new Date(Date.now() + job.delaySeconds * 1e3) : /* @__PURE__ */ new Date();
|
|
277
|
+
const payload = JSON.stringify(job);
|
|
278
|
+
await tx.execute(
|
|
279
|
+
`INSERT INTO ${this.tableName} (queue, payload, attempts, available_at, created_at)
|
|
280
|
+
VALUES ($1, $2, $3, $4, $5)`,
|
|
281
|
+
[queue, payload, job.attempts ?? 0, availableAt.toISOString(), (/* @__PURE__ */ new Date()).toISOString()]
|
|
282
|
+
);
|
|
283
|
+
}
|
|
168
284
|
}
|
|
169
285
|
});
|
|
170
286
|
}
|
|
@@ -340,25 +456,29 @@ var init_KafkaDriver = __esm({
|
|
|
340
456
|
await consumer.connect();
|
|
341
457
|
await consumer.subscribe({ topics: [queue] });
|
|
342
458
|
await consumer.run({
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
459
|
+
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning }) => {
|
|
460
|
+
for (const message of batch.messages) {
|
|
461
|
+
if (!isRunning() || !message.value) {
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
try {
|
|
465
|
+
const payload = JSON.parse(message.value.toString());
|
|
466
|
+
const job = {
|
|
467
|
+
id: payload.id,
|
|
468
|
+
type: payload.type,
|
|
469
|
+
data: payload.data,
|
|
470
|
+
className: payload.className,
|
|
471
|
+
createdAt: payload.createdAt,
|
|
472
|
+
delaySeconds: payload.delaySeconds,
|
|
473
|
+
attempts: payload.attempts,
|
|
474
|
+
maxAttempts: payload.maxAttempts
|
|
475
|
+
};
|
|
476
|
+
await callback(job);
|
|
477
|
+
resolveOffset(message.offset);
|
|
478
|
+
await heartbeat();
|
|
479
|
+
} catch (error) {
|
|
480
|
+
console.error("[KafkaDriver] Error processing message:", error);
|
|
481
|
+
}
|
|
362
482
|
}
|
|
363
483
|
}
|
|
364
484
|
});
|
|
@@ -443,6 +563,25 @@ var init_RabbitMQDriver = __esm({
|
|
|
443
563
|
job._raw = msg;
|
|
444
564
|
return job;
|
|
445
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* Pop multiple jobs.
|
|
568
|
+
* Uses channel.get() in a loop (no native batch get in AMQP).
|
|
569
|
+
*/
|
|
570
|
+
async popMany(queue, count) {
|
|
571
|
+
const channel = await this.ensureChannel();
|
|
572
|
+
await channel.assertQueue(queue, { durable: true });
|
|
573
|
+
const results = [];
|
|
574
|
+
for (let i = 0; i < count; i++) {
|
|
575
|
+
const msg = await channel.get(queue, { noAck: false });
|
|
576
|
+
if (!msg) {
|
|
577
|
+
break;
|
|
578
|
+
}
|
|
579
|
+
const job = JSON.parse(msg.content.toString());
|
|
580
|
+
job._raw = msg;
|
|
581
|
+
results.push(job);
|
|
582
|
+
}
|
|
583
|
+
return results;
|
|
584
|
+
}
|
|
446
585
|
/**
|
|
447
586
|
* Acknowledge a message.
|
|
448
587
|
*/
|
|
@@ -472,6 +611,9 @@ var init_RabbitMQDriver = __esm({
|
|
|
472
611
|
async subscribe(queue, callback, options = {}) {
|
|
473
612
|
const channel = await this.ensureChannel();
|
|
474
613
|
await channel.assertQueue(queue, { durable: true });
|
|
614
|
+
if (options.prefetch) {
|
|
615
|
+
await channel.prefetch(options.prefetch);
|
|
616
|
+
}
|
|
475
617
|
if (this.exchange) {
|
|
476
618
|
await channel.bindQueue(queue, this.exchange, "");
|
|
477
619
|
}
|
|
@@ -556,6 +698,63 @@ var init_RedisDriver = __esm({
|
|
|
556
698
|
else
|
|
557
699
|
return redis.call('SREM', activeSet, groupId)
|
|
558
700
|
end
|
|
701
|
+
`;
|
|
702
|
+
// Lua Logic:
|
|
703
|
+
// Iterate priorities.
|
|
704
|
+
// Check delayed.
|
|
705
|
+
// Check paused.
|
|
706
|
+
// RPOP count.
|
|
707
|
+
static POP_MANY_SCRIPT = `
|
|
708
|
+
local queue = KEYS[1]
|
|
709
|
+
local prefix = ARGV[1]
|
|
710
|
+
local count = tonumber(ARGV[2])
|
|
711
|
+
local now = tonumber(ARGV[3])
|
|
712
|
+
|
|
713
|
+
local priorities = {'critical', 'high', 'default', 'low'}
|
|
714
|
+
local result = {}
|
|
715
|
+
|
|
716
|
+
for _, priority in ipairs(priorities) do
|
|
717
|
+
if #result >= count then break end
|
|
718
|
+
|
|
719
|
+
local key = prefix .. queue
|
|
720
|
+
if priority ~= 'default' then
|
|
721
|
+
key = key .. ':' .. priority
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
-- Check Delayed (Move to Ready if due)
|
|
725
|
+
local delayKey = key .. ":delayed"
|
|
726
|
+
-- Optimization: Only check delayed if we need more items
|
|
727
|
+
-- Fetch up to (count - #result) delayed items
|
|
728
|
+
local needed = count - #result
|
|
729
|
+
local delayed = redis.call("ZRANGEBYSCORE", delayKey, 0, now, "LIMIT", 0, needed)
|
|
730
|
+
|
|
731
|
+
for _, job in ipairs(delayed) do
|
|
732
|
+
redis.call("ZREM", delayKey, job)
|
|
733
|
+
-- We return it directly, assuming we want to process it now.
|
|
734
|
+
-- Alternative: LPUSH to list and RPOP? No, direct return is faster.
|
|
735
|
+
table.insert(result, job)
|
|
736
|
+
needed = needed - 1
|
|
737
|
+
end
|
|
738
|
+
|
|
739
|
+
if #result >= count then break end
|
|
740
|
+
|
|
741
|
+
-- Check Paused
|
|
742
|
+
local isPaused = redis.call("GET", key .. ":paused")
|
|
743
|
+
if isPaused ~= "1" then
|
|
744
|
+
needed = count - #result
|
|
745
|
+
-- Loop RPOP to get items
|
|
746
|
+
for i = 1, needed do
|
|
747
|
+
local job = redis.call("RPOP", key)
|
|
748
|
+
if job then
|
|
749
|
+
table.insert(result, job)
|
|
750
|
+
else
|
|
751
|
+
break
|
|
752
|
+
end
|
|
753
|
+
end
|
|
754
|
+
end
|
|
755
|
+
end
|
|
756
|
+
|
|
757
|
+
return result
|
|
559
758
|
`;
|
|
560
759
|
constructor(config) {
|
|
561
760
|
this.client = config.client;
|
|
@@ -566,7 +765,6 @@ var init_RedisDriver = __esm({
|
|
|
566
765
|
);
|
|
567
766
|
}
|
|
568
767
|
if (typeof this.client.defineCommand === "function") {
|
|
569
|
-
;
|
|
570
768
|
this.client.defineCommand("pushGroupJob", {
|
|
571
769
|
numberOfKeys: 3,
|
|
572
770
|
lua: _RedisDriver.PUSH_SCRIPT
|
|
@@ -575,6 +773,10 @@ var init_RedisDriver = __esm({
|
|
|
575
773
|
numberOfKeys: 3,
|
|
576
774
|
lua: _RedisDriver.COMPLETE_SCRIPT
|
|
577
775
|
});
|
|
776
|
+
this.client.defineCommand("popMany", {
|
|
777
|
+
numberOfKeys: 1,
|
|
778
|
+
lua: _RedisDriver.POP_MANY_SCRIPT
|
|
779
|
+
});
|
|
578
780
|
}
|
|
579
781
|
}
|
|
580
782
|
/**
|
|
@@ -641,31 +843,70 @@ var init_RedisDriver = __esm({
|
|
|
641
843
|
}
|
|
642
844
|
}
|
|
643
845
|
/**
|
|
644
|
-
* Pop a job (
|
|
645
|
-
*
|
|
846
|
+
* Pop a job from a queue (non-blocking).
|
|
847
|
+
* Optimized with Lua script for atomic priority polling.
|
|
646
848
|
*/
|
|
647
849
|
async pop(queue) {
|
|
850
|
+
const priorities = ["critical", "high", "default", "low"];
|
|
851
|
+
const keys = [];
|
|
852
|
+
for (const p of priorities) {
|
|
853
|
+
keys.push(this.getKey(queue, p === "default" ? void 0 : p));
|
|
854
|
+
}
|
|
855
|
+
const script = `
|
|
856
|
+
local now = tonumber(ARGV[1])
|
|
857
|
+
for i, key in ipairs(KEYS) do
|
|
858
|
+
-- 1. Check delayed
|
|
859
|
+
local delayKey = key .. ":delayed"
|
|
860
|
+
local delayed = redis.call("ZRANGEBYSCORE", delayKey, 0, now, "LIMIT", 0, 1)
|
|
861
|
+
if delayed[1] then
|
|
862
|
+
redis.call("ZREM", delayKey, delayed[1])
|
|
863
|
+
return {key, delayed[1]}
|
|
864
|
+
end
|
|
865
|
+
|
|
866
|
+
-- 2. Check paused
|
|
867
|
+
local isPaused = redis.call("GET", key .. ":paused")
|
|
868
|
+
if isPaused ~= "1" then
|
|
869
|
+
-- 3. RPOP
|
|
870
|
+
local payload = redis.call("RPOP", key)
|
|
871
|
+
if payload then
|
|
872
|
+
return {key, payload}
|
|
873
|
+
end
|
|
874
|
+
end
|
|
875
|
+
end
|
|
876
|
+
return nil
|
|
877
|
+
`;
|
|
878
|
+
try {
|
|
879
|
+
const result = await this.client.eval(script, keys.length, ...keys, Date.now().toString());
|
|
880
|
+
if (result?.[1]) {
|
|
881
|
+
return this.parsePayload(result[1]);
|
|
882
|
+
}
|
|
883
|
+
} catch (err) {
|
|
884
|
+
console.error("[RedisDriver] Lua pop error:", err);
|
|
885
|
+
return this.popManualFallback(queue);
|
|
886
|
+
}
|
|
887
|
+
return null;
|
|
888
|
+
}
|
|
889
|
+
/**
|
|
890
|
+
* Manual fallback for pop if Lua fails.
|
|
891
|
+
*/
|
|
892
|
+
async popManualFallback(queue) {
|
|
648
893
|
const priorities = ["critical", "high", void 0, "low"];
|
|
649
894
|
for (const priority of priorities) {
|
|
650
895
|
const key = this.getKey(queue, priority);
|
|
651
896
|
const delayKey = `${key}:delayed`;
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
return this.parsePayload(payload2);
|
|
661
|
-
}
|
|
897
|
+
const now = Date.now();
|
|
898
|
+
const delayedJobs = await this.client.zrange?.(delayKey, 0, 0, "WITHSCORES");
|
|
899
|
+
if (delayedJobs && delayedJobs.length >= 2) {
|
|
900
|
+
const score = parseFloat(delayedJobs[1]);
|
|
901
|
+
if (score <= now) {
|
|
902
|
+
const payload2 = delayedJobs[0];
|
|
903
|
+
await this.client.zrem?.(delayKey, payload2);
|
|
904
|
+
return this.parsePayload(payload2);
|
|
662
905
|
}
|
|
663
906
|
}
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
continue;
|
|
668
|
-
}
|
|
907
|
+
const isPaused = await this.client.get?.(`${key}:paused`);
|
|
908
|
+
if (isPaused === "1") {
|
|
909
|
+
continue;
|
|
669
910
|
}
|
|
670
911
|
const payload = await this.client.rpop(key);
|
|
671
912
|
if (payload) {
|
|
@@ -674,6 +915,31 @@ var init_RedisDriver = __esm({
|
|
|
674
915
|
}
|
|
675
916
|
return null;
|
|
676
917
|
}
|
|
918
|
+
/**
|
|
919
|
+
* Pop a job from the queue (blocking).
|
|
920
|
+
* Uses BRPOP for efficiency. Supports multiple queues and priorities.
|
|
921
|
+
*/
|
|
922
|
+
async popBlocking(queues, timeout) {
|
|
923
|
+
const queueList = Array.isArray(queues) ? queues : [queues];
|
|
924
|
+
const priorities = ["critical", "high", void 0, "low"];
|
|
925
|
+
const keys = [];
|
|
926
|
+
for (const q of queueList) {
|
|
927
|
+
for (const p of priorities) {
|
|
928
|
+
keys.push(this.getKey(q, p));
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
if (typeof this.client.brpop !== "function") {
|
|
932
|
+
return this.pop(queueList[0]);
|
|
933
|
+
}
|
|
934
|
+
try {
|
|
935
|
+
const result = await this.client.brpop(...keys, timeout);
|
|
936
|
+
if (result && Array.isArray(result) && result.length >= 2) {
|
|
937
|
+
return this.parsePayload(result[1]);
|
|
938
|
+
}
|
|
939
|
+
} catch (_e) {
|
|
940
|
+
}
|
|
941
|
+
return null;
|
|
942
|
+
}
|
|
677
943
|
/**
|
|
678
944
|
* Parse Redis payload.
|
|
679
945
|
*/
|
|
@@ -723,11 +989,57 @@ var init_RedisDriver = __esm({
|
|
|
723
989
|
const delayKey = `${key}:delayed`;
|
|
724
990
|
const activeSetKey = `${this.prefix}active`;
|
|
725
991
|
await this.client.del(key);
|
|
726
|
-
if (
|
|
992
|
+
if (this.client.del) {
|
|
727
993
|
await this.client.del(delayKey);
|
|
728
994
|
await this.client.del(activeSetKey);
|
|
729
995
|
}
|
|
730
996
|
}
|
|
997
|
+
/**
|
|
998
|
+
* Get queue statistics.
|
|
999
|
+
* Optimized with Redis Pipeline to fetch all priorities and DLQ stats in one trip.
|
|
1000
|
+
*/
|
|
1001
|
+
async stats(queue) {
|
|
1002
|
+
const priorities = ["critical", "high", "default", "low"];
|
|
1003
|
+
const stats = {
|
|
1004
|
+
queue,
|
|
1005
|
+
size: 0,
|
|
1006
|
+
delayed: 0,
|
|
1007
|
+
failed: 0
|
|
1008
|
+
};
|
|
1009
|
+
const keys = [];
|
|
1010
|
+
for (const p of priorities) {
|
|
1011
|
+
keys.push(this.getKey(queue, p === "default" ? void 0 : p));
|
|
1012
|
+
}
|
|
1013
|
+
try {
|
|
1014
|
+
if (typeof this.client.pipeline === "function") {
|
|
1015
|
+
const pipe = this.client.pipeline();
|
|
1016
|
+
for (const key of keys) {
|
|
1017
|
+
pipe.llen(key);
|
|
1018
|
+
pipe.zcard(`${key}:delayed`);
|
|
1019
|
+
}
|
|
1020
|
+
pipe.llen(`${this.getKey(queue)}:failed`);
|
|
1021
|
+
const results = await pipe.exec();
|
|
1022
|
+
if (results) {
|
|
1023
|
+
let i = 0;
|
|
1024
|
+
for (const _p of priorities) {
|
|
1025
|
+
stats.size += results[i][1] || 0;
|
|
1026
|
+
stats.delayed += results[i + 1][1] || 0;
|
|
1027
|
+
i += 2;
|
|
1028
|
+
}
|
|
1029
|
+
stats.failed = results[i][1] || 0;
|
|
1030
|
+
}
|
|
1031
|
+
} else {
|
|
1032
|
+
for (const key of keys) {
|
|
1033
|
+
stats.size += await this.client.llen?.(key) || 0;
|
|
1034
|
+
stats.delayed += await this.client.zcard?.(`${key}:delayed`) || 0;
|
|
1035
|
+
}
|
|
1036
|
+
stats.failed = await this.client.llen?.(`${this.getKey(queue)}:failed`) || 0;
|
|
1037
|
+
}
|
|
1038
|
+
} catch (err) {
|
|
1039
|
+
console.error("[RedisDriver] Failed to get stats:", err);
|
|
1040
|
+
}
|
|
1041
|
+
return stats;
|
|
1042
|
+
}
|
|
731
1043
|
/**
|
|
732
1044
|
* Push multiple jobs.
|
|
733
1045
|
*/
|
|
@@ -738,6 +1050,43 @@ var init_RedisDriver = __esm({
|
|
|
738
1050
|
const hasGroup = jobs.some((j) => j.groupId);
|
|
739
1051
|
const hasPriority = jobs.some((j) => j.priority);
|
|
740
1052
|
if (hasGroup || hasPriority) {
|
|
1053
|
+
if (typeof this.client.pipeline === "function") {
|
|
1054
|
+
const pipe = this.client.pipeline();
|
|
1055
|
+
for (const job of jobs) {
|
|
1056
|
+
const priority = job.priority;
|
|
1057
|
+
const key2 = this.getKey(queue, priority);
|
|
1058
|
+
const groupId = job.groupId;
|
|
1059
|
+
const payload = JSON.stringify({
|
|
1060
|
+
id: job.id,
|
|
1061
|
+
type: job.type,
|
|
1062
|
+
data: job.data,
|
|
1063
|
+
className: job.className,
|
|
1064
|
+
createdAt: job.createdAt,
|
|
1065
|
+
delaySeconds: job.delaySeconds,
|
|
1066
|
+
attempts: job.attempts,
|
|
1067
|
+
maxAttempts: job.maxAttempts,
|
|
1068
|
+
groupId,
|
|
1069
|
+
priority,
|
|
1070
|
+
error: job.error,
|
|
1071
|
+
failedAt: job.failedAt
|
|
1072
|
+
});
|
|
1073
|
+
if (groupId) {
|
|
1074
|
+
const activeSetKey = `${this.prefix}active`;
|
|
1075
|
+
const pendingListKey = `${this.prefix}pending:${groupId}`;
|
|
1076
|
+
pipe.pushGroupJob(key2, activeSetKey, pendingListKey, groupId, payload);
|
|
1077
|
+
} else {
|
|
1078
|
+
if (job.delaySeconds && job.delaySeconds > 0) {
|
|
1079
|
+
const delayKey = `${key2}:delayed`;
|
|
1080
|
+
const score = Date.now() + job.delaySeconds * 1e3;
|
|
1081
|
+
pipe.zadd(delayKey, score, payload);
|
|
1082
|
+
} else {
|
|
1083
|
+
pipe.lpush(key2, payload);
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
await pipe.exec();
|
|
1088
|
+
return;
|
|
1089
|
+
}
|
|
741
1090
|
for (const job of jobs) {
|
|
742
1091
|
await this.push(queue, job, {
|
|
743
1092
|
groupId: job.groupId,
|
|
@@ -765,17 +1114,80 @@ var init_RedisDriver = __esm({
|
|
|
765
1114
|
}
|
|
766
1115
|
/**
|
|
767
1116
|
* Pop multiple jobs.
|
|
1117
|
+
* Atomic operation across multiple priority levels.
|
|
768
1118
|
*/
|
|
769
1119
|
async popMany(queue, count) {
|
|
770
|
-
|
|
1120
|
+
if (count <= 0) {
|
|
1121
|
+
return [];
|
|
1122
|
+
}
|
|
1123
|
+
if (count === 1) {
|
|
1124
|
+
const job = await this.pop(queue);
|
|
1125
|
+
return job ? [job] : [];
|
|
1126
|
+
}
|
|
1127
|
+
if (typeof this.client.popMany === "function") {
|
|
1128
|
+
try {
|
|
1129
|
+
const result = await this.client.popMany(queue, this.prefix, count, Date.now().toString());
|
|
1130
|
+
if (Array.isArray(result) && result.length > 0) {
|
|
1131
|
+
return result.map((p) => this.parsePayload(p));
|
|
1132
|
+
} else if (Array.isArray(result) && result.length === 0) {
|
|
1133
|
+
} else {
|
|
1134
|
+
}
|
|
1135
|
+
if (Array.isArray(result)) {
|
|
1136
|
+
return result.map((p) => this.parsePayload(p));
|
|
1137
|
+
}
|
|
1138
|
+
} catch (err) {
|
|
1139
|
+
console.error("[RedisDriver] Lua popMany error:", err);
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
const priorities = ["critical", "high", "default", "low"];
|
|
771
1143
|
const results = [];
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
if (
|
|
775
|
-
results.push(this.parsePayload(payload));
|
|
776
|
-
} else {
|
|
1144
|
+
let remaining = count;
|
|
1145
|
+
for (const priority of priorities) {
|
|
1146
|
+
if (remaining <= 0) {
|
|
777
1147
|
break;
|
|
778
1148
|
}
|
|
1149
|
+
const key = this.getKey(queue, priority === "default" ? void 0 : priority);
|
|
1150
|
+
const isPaused = await this.client.get?.(`${key}:paused`);
|
|
1151
|
+
if (isPaused === "1") {
|
|
1152
|
+
continue;
|
|
1153
|
+
}
|
|
1154
|
+
let fetched = [];
|
|
1155
|
+
try {
|
|
1156
|
+
const reply = await this.client.rpop(key, remaining);
|
|
1157
|
+
if (reply) {
|
|
1158
|
+
fetched = Array.isArray(reply) ? reply : [reply];
|
|
1159
|
+
}
|
|
1160
|
+
} catch (_e) {
|
|
1161
|
+
if (typeof this.client.pipeline === "function") {
|
|
1162
|
+
const pipeline = this.client.pipeline();
|
|
1163
|
+
for (let i = 0; i < remaining; i++) {
|
|
1164
|
+
pipeline.rpop(key);
|
|
1165
|
+
}
|
|
1166
|
+
const replies = await pipeline.exec();
|
|
1167
|
+
if (replies) {
|
|
1168
|
+
fetched = replies.map((r) => r[1]).filter((r) => r !== null);
|
|
1169
|
+
}
|
|
1170
|
+
} else {
|
|
1171
|
+
for (let i = 0; i < remaining; i++) {
|
|
1172
|
+
const res = await this.client.rpop(key);
|
|
1173
|
+
if (res) {
|
|
1174
|
+
fetched.push(res);
|
|
1175
|
+
} else {
|
|
1176
|
+
break;
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
if (fetched.length > 0) {
|
|
1182
|
+
for (const payload of fetched) {
|
|
1183
|
+
try {
|
|
1184
|
+
results.push(this.parsePayload(payload));
|
|
1185
|
+
} catch (e) {
|
|
1186
|
+
console.error("[RedisDriver] Failed to parse job payload:", e);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
remaining -= fetched.length;
|
|
1190
|
+
}
|
|
779
1191
|
}
|
|
780
1192
|
return results;
|
|
781
1193
|
}
|
|
@@ -819,7 +1231,7 @@ var init_RedisDriver = __esm({
|
|
|
819
1231
|
const client = this.client;
|
|
820
1232
|
if (typeof client.incr === "function") {
|
|
821
1233
|
const current = await client.incr(windowKey);
|
|
822
|
-
if (current === 1) {
|
|
1234
|
+
if (current === 1 && client.expire) {
|
|
823
1235
|
await client.expire(windowKey, Math.ceil(config.duration / 1e3) + 1);
|
|
824
1236
|
}
|
|
825
1237
|
return current <= config.max;
|
|
@@ -831,6 +1243,9 @@ var init_RedisDriver = __esm({
|
|
|
831
1243
|
*/
|
|
832
1244
|
async getFailed(queue, start = 0, end = -1) {
|
|
833
1245
|
const key = `${this.getKey(queue)}:failed`;
|
|
1246
|
+
if (typeof this.client.lrange !== "function") {
|
|
1247
|
+
return [];
|
|
1248
|
+
}
|
|
834
1249
|
const payloads = await this.client.lrange(key, start, end);
|
|
835
1250
|
return payloads.map((p) => this.parsePayload(p));
|
|
836
1251
|
}
|
|
@@ -842,6 +1257,9 @@ var init_RedisDriver = __esm({
|
|
|
842
1257
|
const failedKey = `${this.getKey(queue)}:failed`;
|
|
843
1258
|
let retried = 0;
|
|
844
1259
|
for (let i = 0; i < count; i++) {
|
|
1260
|
+
if (typeof this.client.rpop !== "function") {
|
|
1261
|
+
break;
|
|
1262
|
+
}
|
|
845
1263
|
const payload = await this.client.rpop(failedKey);
|
|
846
1264
|
if (!payload) {
|
|
847
1265
|
break;
|
|
@@ -964,6 +1382,40 @@ var init_SQSDriver = __esm({
|
|
|
964
1382
|
...message.ReceiptHandle && { receiptHandle: message.ReceiptHandle }
|
|
965
1383
|
};
|
|
966
1384
|
}
|
|
1385
|
+
/**
|
|
1386
|
+
* Pop multiple jobs.
|
|
1387
|
+
* Leverages SQS MaxNumberOfMessages (up to 10).
|
|
1388
|
+
*/
|
|
1389
|
+
async popMany(queue, count) {
|
|
1390
|
+
const { ReceiveMessageCommand } = await import("@aws-sdk/client-sqs");
|
|
1391
|
+
const queueUrl = await this.getQueueUrl(queue);
|
|
1392
|
+
const limit = Math.min(count, 10);
|
|
1393
|
+
const response = await this.client.send(
|
|
1394
|
+
new ReceiveMessageCommand({
|
|
1395
|
+
QueueUrl: queueUrl,
|
|
1396
|
+
MaxNumberOfMessages: limit,
|
|
1397
|
+
WaitTimeSeconds: this.waitTimeSeconds,
|
|
1398
|
+
VisibilityTimeout: this.visibilityTimeout
|
|
1399
|
+
})
|
|
1400
|
+
);
|
|
1401
|
+
if (!response.Messages || response.Messages.length === 0) {
|
|
1402
|
+
return [];
|
|
1403
|
+
}
|
|
1404
|
+
return response.Messages.map((message) => {
|
|
1405
|
+
const payload = JSON.parse(message.Body ?? "{}");
|
|
1406
|
+
return {
|
|
1407
|
+
id: payload.id ?? message.MessageId,
|
|
1408
|
+
type: payload.type,
|
|
1409
|
+
data: payload.data,
|
|
1410
|
+
className: payload.className,
|
|
1411
|
+
createdAt: payload.createdAt,
|
|
1412
|
+
delaySeconds: payload.delaySeconds,
|
|
1413
|
+
attempts: payload.attempts,
|
|
1414
|
+
maxAttempts: payload.maxAttempts,
|
|
1415
|
+
receiptHandle: message.ReceiptHandle
|
|
1416
|
+
};
|
|
1417
|
+
});
|
|
1418
|
+
}
|
|
967
1419
|
/**
|
|
968
1420
|
* Get queue size (approximate).
|
|
969
1421
|
*/
|
|
@@ -1067,6 +1519,1830 @@ var init_SQSDriver = __esm({
|
|
|
1067
1519
|
}
|
|
1068
1520
|
});
|
|
1069
1521
|
|
|
1522
|
+
// src/persistence/BufferedPersistence.ts
|
|
1523
|
+
var BufferedPersistence_exports = {};
|
|
1524
|
+
__export(BufferedPersistence_exports, {
|
|
1525
|
+
BufferedPersistence: () => BufferedPersistence
|
|
1526
|
+
});
|
|
1527
|
+
var BufferedPersistence;
|
|
1528
|
+
var init_BufferedPersistence = __esm({
|
|
1529
|
+
"src/persistence/BufferedPersistence.ts"() {
|
|
1530
|
+
"use strict";
|
|
1531
|
+
BufferedPersistence = class {
|
|
1532
|
+
constructor(adapter, options = {}) {
|
|
1533
|
+
this.adapter = adapter;
|
|
1534
|
+
this.maxBufferSize = options.maxBufferSize ?? 50;
|
|
1535
|
+
this.flushInterval = options.flushInterval ?? 5e3;
|
|
1536
|
+
}
|
|
1537
|
+
jobBuffer = [];
|
|
1538
|
+
logBuffer = [];
|
|
1539
|
+
flushTimer = null;
|
|
1540
|
+
maxBufferSize;
|
|
1541
|
+
flushInterval;
|
|
1542
|
+
async archive(queue, job, status) {
|
|
1543
|
+
this.jobBuffer.push({ queue, job, status });
|
|
1544
|
+
if (this.jobBuffer.length >= this.maxBufferSize) {
|
|
1545
|
+
this.flush().catch((err) => {
|
|
1546
|
+
console.error("[BufferedPersistence] Auto-flush failed (jobs):", err.message || err);
|
|
1547
|
+
});
|
|
1548
|
+
} else {
|
|
1549
|
+
this.ensureFlushTimer();
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
async find(queue, id) {
|
|
1553
|
+
return this.adapter.find(queue, id);
|
|
1554
|
+
}
|
|
1555
|
+
async list(queue, options) {
|
|
1556
|
+
return this.adapter.list(queue, options);
|
|
1557
|
+
}
|
|
1558
|
+
async archiveMany(jobs) {
|
|
1559
|
+
if (this.adapter.archiveMany) {
|
|
1560
|
+
return this.adapter.archiveMany(jobs);
|
|
1561
|
+
}
|
|
1562
|
+
for (const item of jobs) {
|
|
1563
|
+
await this.adapter.archive(item.queue, item.job, item.status);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
async cleanup(days) {
|
|
1567
|
+
return this.adapter.cleanup(days);
|
|
1568
|
+
}
|
|
1569
|
+
async flush() {
|
|
1570
|
+
if (this.flushTimer) {
|
|
1571
|
+
clearTimeout(this.flushTimer);
|
|
1572
|
+
this.flushTimer = null;
|
|
1573
|
+
}
|
|
1574
|
+
const jobs = [...this.jobBuffer];
|
|
1575
|
+
const logs = [...this.logBuffer];
|
|
1576
|
+
this.jobBuffer = [];
|
|
1577
|
+
this.logBuffer = [];
|
|
1578
|
+
const promises = [];
|
|
1579
|
+
if (jobs.length > 0) {
|
|
1580
|
+
if (this.adapter.archiveMany) {
|
|
1581
|
+
promises.push(this.adapter.archiveMany(jobs));
|
|
1582
|
+
} else {
|
|
1583
|
+
promises.push(
|
|
1584
|
+
(async () => {
|
|
1585
|
+
for (const item of jobs) {
|
|
1586
|
+
await this.adapter.archive(item.queue, item.job, item.status);
|
|
1587
|
+
}
|
|
1588
|
+
})()
|
|
1589
|
+
);
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
if (logs.length > 0) {
|
|
1593
|
+
if (this.adapter.archiveLogMany) {
|
|
1594
|
+
promises.push(this.adapter.archiveLogMany(logs));
|
|
1595
|
+
} else {
|
|
1596
|
+
promises.push(
|
|
1597
|
+
(async () => {
|
|
1598
|
+
for (const log of logs) {
|
|
1599
|
+
await this.adapter.archiveLog(log);
|
|
1600
|
+
}
|
|
1601
|
+
})()
|
|
1602
|
+
);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
await Promise.all(promises);
|
|
1606
|
+
}
|
|
1607
|
+
async count(queue, options) {
|
|
1608
|
+
return this.adapter.count(queue, options);
|
|
1609
|
+
}
|
|
1610
|
+
async archiveLog(log) {
|
|
1611
|
+
this.logBuffer.push(log);
|
|
1612
|
+
if (this.logBuffer.length >= this.maxBufferSize) {
|
|
1613
|
+
this.flush().catch((err) => {
|
|
1614
|
+
console.error("[BufferedPersistence] Auto-flush failed (logs):", err.message || err);
|
|
1615
|
+
});
|
|
1616
|
+
} else {
|
|
1617
|
+
this.ensureFlushTimer();
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
async archiveLogMany(logs) {
|
|
1621
|
+
if (this.adapter.archiveLogMany) {
|
|
1622
|
+
return this.adapter.archiveLogMany(logs);
|
|
1623
|
+
}
|
|
1624
|
+
for (const log of logs) {
|
|
1625
|
+
await this.adapter.archiveLog(log);
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
async listLogs(options) {
|
|
1629
|
+
return this.adapter.listLogs(options);
|
|
1630
|
+
}
|
|
1631
|
+
async countLogs(options) {
|
|
1632
|
+
return this.adapter.countLogs(options);
|
|
1633
|
+
}
|
|
1634
|
+
ensureFlushTimer() {
|
|
1635
|
+
if (this.flushTimer) {
|
|
1636
|
+
return;
|
|
1637
|
+
}
|
|
1638
|
+
this.flushTimer = setTimeout(() => {
|
|
1639
|
+
this.flush().catch((err) => {
|
|
1640
|
+
console.error("[BufferedPersistence] Interval flush failed:", err.message || err);
|
|
1641
|
+
});
|
|
1642
|
+
}, this.flushInterval);
|
|
1643
|
+
}
|
|
1644
|
+
};
|
|
1645
|
+
}
|
|
1646
|
+
});
|
|
1647
|
+
|
|
1648
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/utf8.mjs
|
|
1649
|
+
function utf8Count(str) {
|
|
1650
|
+
const strLength = str.length;
|
|
1651
|
+
let byteLength = 0;
|
|
1652
|
+
let pos = 0;
|
|
1653
|
+
while (pos < strLength) {
|
|
1654
|
+
let value = str.charCodeAt(pos++);
|
|
1655
|
+
if ((value & 4294967168) === 0) {
|
|
1656
|
+
byteLength++;
|
|
1657
|
+
continue;
|
|
1658
|
+
} else if ((value & 4294965248) === 0) {
|
|
1659
|
+
byteLength += 2;
|
|
1660
|
+
} else {
|
|
1661
|
+
if (value >= 55296 && value <= 56319) {
|
|
1662
|
+
if (pos < strLength) {
|
|
1663
|
+
const extra = str.charCodeAt(pos);
|
|
1664
|
+
if ((extra & 64512) === 56320) {
|
|
1665
|
+
++pos;
|
|
1666
|
+
value = ((value & 1023) << 10) + (extra & 1023) + 65536;
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
if ((value & 4294901760) === 0) {
|
|
1671
|
+
byteLength += 3;
|
|
1672
|
+
} else {
|
|
1673
|
+
byteLength += 4;
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
return byteLength;
|
|
1678
|
+
}
|
|
1679
|
+
function utf8EncodeJs(str, output, outputOffset) {
|
|
1680
|
+
const strLength = str.length;
|
|
1681
|
+
let offset = outputOffset;
|
|
1682
|
+
let pos = 0;
|
|
1683
|
+
while (pos < strLength) {
|
|
1684
|
+
let value = str.charCodeAt(pos++);
|
|
1685
|
+
if ((value & 4294967168) === 0) {
|
|
1686
|
+
output[offset++] = value;
|
|
1687
|
+
continue;
|
|
1688
|
+
} else if ((value & 4294965248) === 0) {
|
|
1689
|
+
output[offset++] = value >> 6 & 31 | 192;
|
|
1690
|
+
} else {
|
|
1691
|
+
if (value >= 55296 && value <= 56319) {
|
|
1692
|
+
if (pos < strLength) {
|
|
1693
|
+
const extra = str.charCodeAt(pos);
|
|
1694
|
+
if ((extra & 64512) === 56320) {
|
|
1695
|
+
++pos;
|
|
1696
|
+
value = ((value & 1023) << 10) + (extra & 1023) + 65536;
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
if ((value & 4294901760) === 0) {
|
|
1701
|
+
output[offset++] = value >> 12 & 15 | 224;
|
|
1702
|
+
output[offset++] = value >> 6 & 63 | 128;
|
|
1703
|
+
} else {
|
|
1704
|
+
output[offset++] = value >> 18 & 7 | 240;
|
|
1705
|
+
output[offset++] = value >> 12 & 63 | 128;
|
|
1706
|
+
output[offset++] = value >> 6 & 63 | 128;
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
output[offset++] = value & 63 | 128;
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
function utf8EncodeTE(str, output, outputOffset) {
|
|
1713
|
+
sharedTextEncoder.encodeInto(str, output.subarray(outputOffset));
|
|
1714
|
+
}
|
|
1715
|
+
function utf8Encode(str, output, outputOffset) {
|
|
1716
|
+
if (str.length > TEXT_ENCODER_THRESHOLD) {
|
|
1717
|
+
utf8EncodeTE(str, output, outputOffset);
|
|
1718
|
+
} else {
|
|
1719
|
+
utf8EncodeJs(str, output, outputOffset);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
function utf8DecodeJs(bytes, inputOffset, byteLength) {
|
|
1723
|
+
let offset = inputOffset;
|
|
1724
|
+
const end = offset + byteLength;
|
|
1725
|
+
const units = [];
|
|
1726
|
+
let result = "";
|
|
1727
|
+
while (offset < end) {
|
|
1728
|
+
const byte1 = bytes[offset++];
|
|
1729
|
+
if ((byte1 & 128) === 0) {
|
|
1730
|
+
units.push(byte1);
|
|
1731
|
+
} else if ((byte1 & 224) === 192) {
|
|
1732
|
+
const byte2 = bytes[offset++] & 63;
|
|
1733
|
+
units.push((byte1 & 31) << 6 | byte2);
|
|
1734
|
+
} else if ((byte1 & 240) === 224) {
|
|
1735
|
+
const byte2 = bytes[offset++] & 63;
|
|
1736
|
+
const byte3 = bytes[offset++] & 63;
|
|
1737
|
+
units.push((byte1 & 31) << 12 | byte2 << 6 | byte3);
|
|
1738
|
+
} else if ((byte1 & 248) === 240) {
|
|
1739
|
+
const byte2 = bytes[offset++] & 63;
|
|
1740
|
+
const byte3 = bytes[offset++] & 63;
|
|
1741
|
+
const byte4 = bytes[offset++] & 63;
|
|
1742
|
+
let unit = (byte1 & 7) << 18 | byte2 << 12 | byte3 << 6 | byte4;
|
|
1743
|
+
if (unit > 65535) {
|
|
1744
|
+
unit -= 65536;
|
|
1745
|
+
units.push(unit >>> 10 & 1023 | 55296);
|
|
1746
|
+
unit = 56320 | unit & 1023;
|
|
1747
|
+
}
|
|
1748
|
+
units.push(unit);
|
|
1749
|
+
} else {
|
|
1750
|
+
units.push(byte1);
|
|
1751
|
+
}
|
|
1752
|
+
if (units.length >= CHUNK_SIZE) {
|
|
1753
|
+
result += String.fromCharCode(...units);
|
|
1754
|
+
units.length = 0;
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
if (units.length > 0) {
|
|
1758
|
+
result += String.fromCharCode(...units);
|
|
1759
|
+
}
|
|
1760
|
+
return result;
|
|
1761
|
+
}
|
|
1762
|
+
function utf8DecodeTD(bytes, inputOffset, byteLength) {
|
|
1763
|
+
const stringBytes = bytes.subarray(inputOffset, inputOffset + byteLength);
|
|
1764
|
+
return sharedTextDecoder.decode(stringBytes);
|
|
1765
|
+
}
|
|
1766
|
+
function utf8Decode(bytes, inputOffset, byteLength) {
|
|
1767
|
+
if (byteLength > TEXT_DECODER_THRESHOLD) {
|
|
1768
|
+
return utf8DecodeTD(bytes, inputOffset, byteLength);
|
|
1769
|
+
} else {
|
|
1770
|
+
return utf8DecodeJs(bytes, inputOffset, byteLength);
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
var sharedTextEncoder, TEXT_ENCODER_THRESHOLD, CHUNK_SIZE, sharedTextDecoder, TEXT_DECODER_THRESHOLD;
|
|
1774
|
+
var init_utf8 = __esm({
|
|
1775
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/utf8.mjs"() {
|
|
1776
|
+
"use strict";
|
|
1777
|
+
sharedTextEncoder = new TextEncoder();
|
|
1778
|
+
TEXT_ENCODER_THRESHOLD = 50;
|
|
1779
|
+
CHUNK_SIZE = 4096;
|
|
1780
|
+
sharedTextDecoder = new TextDecoder();
|
|
1781
|
+
TEXT_DECODER_THRESHOLD = 200;
|
|
1782
|
+
}
|
|
1783
|
+
});
|
|
1784
|
+
|
|
1785
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/ExtData.mjs
|
|
1786
|
+
var ExtData;
|
|
1787
|
+
var init_ExtData = __esm({
|
|
1788
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/ExtData.mjs"() {
|
|
1789
|
+
"use strict";
|
|
1790
|
+
ExtData = class {
|
|
1791
|
+
type;
|
|
1792
|
+
data;
|
|
1793
|
+
constructor(type, data) {
|
|
1794
|
+
this.type = type;
|
|
1795
|
+
this.data = data;
|
|
1796
|
+
}
|
|
1797
|
+
};
|
|
1798
|
+
}
|
|
1799
|
+
});
|
|
1800
|
+
|
|
1801
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/DecodeError.mjs
|
|
1802
|
+
var DecodeError;
|
|
1803
|
+
var init_DecodeError = __esm({
|
|
1804
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/DecodeError.mjs"() {
|
|
1805
|
+
"use strict";
|
|
1806
|
+
DecodeError = class _DecodeError extends Error {
|
|
1807
|
+
constructor(message) {
|
|
1808
|
+
super(message);
|
|
1809
|
+
const proto = Object.create(_DecodeError.prototype);
|
|
1810
|
+
Object.setPrototypeOf(this, proto);
|
|
1811
|
+
Object.defineProperty(this, "name", {
|
|
1812
|
+
configurable: true,
|
|
1813
|
+
enumerable: false,
|
|
1814
|
+
value: _DecodeError.name
|
|
1815
|
+
});
|
|
1816
|
+
}
|
|
1817
|
+
};
|
|
1818
|
+
}
|
|
1819
|
+
});
|
|
1820
|
+
|
|
1821
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/int.mjs
|
|
1822
|
+
function setUint64(view, offset, value) {
|
|
1823
|
+
const high = value / 4294967296;
|
|
1824
|
+
const low = value;
|
|
1825
|
+
view.setUint32(offset, high);
|
|
1826
|
+
view.setUint32(offset + 4, low);
|
|
1827
|
+
}
|
|
1828
|
+
function setInt64(view, offset, value) {
|
|
1829
|
+
const high = Math.floor(value / 4294967296);
|
|
1830
|
+
const low = value;
|
|
1831
|
+
view.setUint32(offset, high);
|
|
1832
|
+
view.setUint32(offset + 4, low);
|
|
1833
|
+
}
|
|
1834
|
+
function getInt64(view, offset) {
|
|
1835
|
+
const high = view.getInt32(offset);
|
|
1836
|
+
const low = view.getUint32(offset + 4);
|
|
1837
|
+
return high * 4294967296 + low;
|
|
1838
|
+
}
|
|
1839
|
+
function getUint64(view, offset) {
|
|
1840
|
+
const high = view.getUint32(offset);
|
|
1841
|
+
const low = view.getUint32(offset + 4);
|
|
1842
|
+
return high * 4294967296 + low;
|
|
1843
|
+
}
|
|
1844
|
+
var UINT32_MAX;
|
|
1845
|
+
var init_int = __esm({
|
|
1846
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/int.mjs"() {
|
|
1847
|
+
"use strict";
|
|
1848
|
+
UINT32_MAX = 4294967295;
|
|
1849
|
+
}
|
|
1850
|
+
});
|
|
1851
|
+
|
|
1852
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/timestamp.mjs
|
|
1853
|
+
function encodeTimeSpecToTimestamp({ sec, nsec }) {
|
|
1854
|
+
if (sec >= 0 && nsec >= 0 && sec <= TIMESTAMP64_MAX_SEC) {
|
|
1855
|
+
if (nsec === 0 && sec <= TIMESTAMP32_MAX_SEC) {
|
|
1856
|
+
const rv = new Uint8Array(4);
|
|
1857
|
+
const view = new DataView(rv.buffer);
|
|
1858
|
+
view.setUint32(0, sec);
|
|
1859
|
+
return rv;
|
|
1860
|
+
} else {
|
|
1861
|
+
const secHigh = sec / 4294967296;
|
|
1862
|
+
const secLow = sec & 4294967295;
|
|
1863
|
+
const rv = new Uint8Array(8);
|
|
1864
|
+
const view = new DataView(rv.buffer);
|
|
1865
|
+
view.setUint32(0, nsec << 2 | secHigh & 3);
|
|
1866
|
+
view.setUint32(4, secLow);
|
|
1867
|
+
return rv;
|
|
1868
|
+
}
|
|
1869
|
+
} else {
|
|
1870
|
+
const rv = new Uint8Array(12);
|
|
1871
|
+
const view = new DataView(rv.buffer);
|
|
1872
|
+
view.setUint32(0, nsec);
|
|
1873
|
+
setInt64(view, 4, sec);
|
|
1874
|
+
return rv;
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
function encodeDateToTimeSpec(date) {
|
|
1878
|
+
const msec = date.getTime();
|
|
1879
|
+
const sec = Math.floor(msec / 1e3);
|
|
1880
|
+
const nsec = (msec - sec * 1e3) * 1e6;
|
|
1881
|
+
const nsecInSec = Math.floor(nsec / 1e9);
|
|
1882
|
+
return {
|
|
1883
|
+
sec: sec + nsecInSec,
|
|
1884
|
+
nsec: nsec - nsecInSec * 1e9
|
|
1885
|
+
};
|
|
1886
|
+
}
|
|
1887
|
+
function encodeTimestampExtension(object) {
|
|
1888
|
+
if (object instanceof Date) {
|
|
1889
|
+
const timeSpec = encodeDateToTimeSpec(object);
|
|
1890
|
+
return encodeTimeSpecToTimestamp(timeSpec);
|
|
1891
|
+
} else {
|
|
1892
|
+
return null;
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
function decodeTimestampToTimeSpec(data) {
|
|
1896
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
1897
|
+
switch (data.byteLength) {
|
|
1898
|
+
case 4: {
|
|
1899
|
+
const sec = view.getUint32(0);
|
|
1900
|
+
const nsec = 0;
|
|
1901
|
+
return { sec, nsec };
|
|
1902
|
+
}
|
|
1903
|
+
case 8: {
|
|
1904
|
+
const nsec30AndSecHigh2 = view.getUint32(0);
|
|
1905
|
+
const secLow32 = view.getUint32(4);
|
|
1906
|
+
const sec = (nsec30AndSecHigh2 & 3) * 4294967296 + secLow32;
|
|
1907
|
+
const nsec = nsec30AndSecHigh2 >>> 2;
|
|
1908
|
+
return { sec, nsec };
|
|
1909
|
+
}
|
|
1910
|
+
case 12: {
|
|
1911
|
+
const sec = getInt64(view, 4);
|
|
1912
|
+
const nsec = view.getUint32(0);
|
|
1913
|
+
return { sec, nsec };
|
|
1914
|
+
}
|
|
1915
|
+
default:
|
|
1916
|
+
throw new DecodeError(`Unrecognized data size for timestamp (expected 4, 8, or 12): ${data.length}`);
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
function decodeTimestampExtension(data) {
|
|
1920
|
+
const timeSpec = decodeTimestampToTimeSpec(data);
|
|
1921
|
+
return new Date(timeSpec.sec * 1e3 + timeSpec.nsec / 1e6);
|
|
1922
|
+
}
|
|
1923
|
+
var EXT_TIMESTAMP, TIMESTAMP32_MAX_SEC, TIMESTAMP64_MAX_SEC, timestampExtension;
|
|
1924
|
+
var init_timestamp = __esm({
|
|
1925
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/timestamp.mjs"() {
|
|
1926
|
+
"use strict";
|
|
1927
|
+
init_DecodeError();
|
|
1928
|
+
init_int();
|
|
1929
|
+
EXT_TIMESTAMP = -1;
|
|
1930
|
+
TIMESTAMP32_MAX_SEC = 4294967296 - 1;
|
|
1931
|
+
TIMESTAMP64_MAX_SEC = 17179869184 - 1;
|
|
1932
|
+
timestampExtension = {
|
|
1933
|
+
type: EXT_TIMESTAMP,
|
|
1934
|
+
encode: encodeTimestampExtension,
|
|
1935
|
+
decode: decodeTimestampExtension
|
|
1936
|
+
};
|
|
1937
|
+
}
|
|
1938
|
+
});
|
|
1939
|
+
|
|
1940
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/ExtensionCodec.mjs
|
|
1941
|
+
var ExtensionCodec;
|
|
1942
|
+
var init_ExtensionCodec = __esm({
|
|
1943
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/ExtensionCodec.mjs"() {
|
|
1944
|
+
"use strict";
|
|
1945
|
+
init_ExtData();
|
|
1946
|
+
init_timestamp();
|
|
1947
|
+
ExtensionCodec = class _ExtensionCodec {
|
|
1948
|
+
static defaultCodec = new _ExtensionCodec();
|
|
1949
|
+
// ensures ExtensionCodecType<X> matches ExtensionCodec<X>
|
|
1950
|
+
// this will make type errors a lot more clear
|
|
1951
|
+
// eslint-disable-next-line @typescript-eslint/naming-convention
|
|
1952
|
+
__brand;
|
|
1953
|
+
// built-in extensions
|
|
1954
|
+
builtInEncoders = [];
|
|
1955
|
+
builtInDecoders = [];
|
|
1956
|
+
// custom extensions
|
|
1957
|
+
encoders = [];
|
|
1958
|
+
decoders = [];
|
|
1959
|
+
constructor() {
|
|
1960
|
+
this.register(timestampExtension);
|
|
1961
|
+
}
|
|
1962
|
+
register({ type, encode: encode2, decode: decode2 }) {
|
|
1963
|
+
if (type >= 0) {
|
|
1964
|
+
this.encoders[type] = encode2;
|
|
1965
|
+
this.decoders[type] = decode2;
|
|
1966
|
+
} else {
|
|
1967
|
+
const index = -1 - type;
|
|
1968
|
+
this.builtInEncoders[index] = encode2;
|
|
1969
|
+
this.builtInDecoders[index] = decode2;
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
tryToEncode(object, context) {
|
|
1973
|
+
for (let i = 0; i < this.builtInEncoders.length; i++) {
|
|
1974
|
+
const encodeExt = this.builtInEncoders[i];
|
|
1975
|
+
if (encodeExt != null) {
|
|
1976
|
+
const data = encodeExt(object, context);
|
|
1977
|
+
if (data != null) {
|
|
1978
|
+
const type = -1 - i;
|
|
1979
|
+
return new ExtData(type, data);
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
for (let i = 0; i < this.encoders.length; i++) {
|
|
1984
|
+
const encodeExt = this.encoders[i];
|
|
1985
|
+
if (encodeExt != null) {
|
|
1986
|
+
const data = encodeExt(object, context);
|
|
1987
|
+
if (data != null) {
|
|
1988
|
+
const type = i;
|
|
1989
|
+
return new ExtData(type, data);
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
if (object instanceof ExtData) {
|
|
1994
|
+
return object;
|
|
1995
|
+
}
|
|
1996
|
+
return null;
|
|
1997
|
+
}
|
|
1998
|
+
decode(data, type, context) {
|
|
1999
|
+
const decodeExt = type < 0 ? this.builtInDecoders[-1 - type] : this.decoders[type];
|
|
2000
|
+
if (decodeExt) {
|
|
2001
|
+
return decodeExt(data, type, context);
|
|
2002
|
+
} else {
|
|
2003
|
+
return new ExtData(type, data);
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
};
|
|
2007
|
+
}
|
|
2008
|
+
});
|
|
2009
|
+
|
|
2010
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/typedArrays.mjs
|
|
2011
|
+
function isArrayBufferLike(buffer) {
|
|
2012
|
+
return buffer instanceof ArrayBuffer || typeof SharedArrayBuffer !== "undefined" && buffer instanceof SharedArrayBuffer;
|
|
2013
|
+
}
|
|
2014
|
+
function ensureUint8Array(buffer) {
|
|
2015
|
+
if (buffer instanceof Uint8Array) {
|
|
2016
|
+
return buffer;
|
|
2017
|
+
} else if (ArrayBuffer.isView(buffer)) {
|
|
2018
|
+
return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
|
|
2019
|
+
} else if (isArrayBufferLike(buffer)) {
|
|
2020
|
+
return new Uint8Array(buffer);
|
|
2021
|
+
} else {
|
|
2022
|
+
return Uint8Array.from(buffer);
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
var init_typedArrays = __esm({
|
|
2026
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/typedArrays.mjs"() {
|
|
2027
|
+
"use strict";
|
|
2028
|
+
}
|
|
2029
|
+
});
|
|
2030
|
+
|
|
2031
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/Encoder.mjs
|
|
2032
|
+
var DEFAULT_MAX_DEPTH, DEFAULT_INITIAL_BUFFER_SIZE, Encoder;
|
|
2033
|
+
var init_Encoder = __esm({
|
|
2034
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/Encoder.mjs"() {
|
|
2035
|
+
"use strict";
|
|
2036
|
+
init_utf8();
|
|
2037
|
+
init_ExtensionCodec();
|
|
2038
|
+
init_int();
|
|
2039
|
+
init_typedArrays();
|
|
2040
|
+
DEFAULT_MAX_DEPTH = 100;
|
|
2041
|
+
DEFAULT_INITIAL_BUFFER_SIZE = 2048;
|
|
2042
|
+
Encoder = class _Encoder {
|
|
2043
|
+
extensionCodec;
|
|
2044
|
+
context;
|
|
2045
|
+
useBigInt64;
|
|
2046
|
+
maxDepth;
|
|
2047
|
+
initialBufferSize;
|
|
2048
|
+
sortKeys;
|
|
2049
|
+
forceFloat32;
|
|
2050
|
+
ignoreUndefined;
|
|
2051
|
+
forceIntegerToFloat;
|
|
2052
|
+
pos;
|
|
2053
|
+
view;
|
|
2054
|
+
bytes;
|
|
2055
|
+
entered = false;
|
|
2056
|
+
constructor(options) {
|
|
2057
|
+
this.extensionCodec = options?.extensionCodec ?? ExtensionCodec.defaultCodec;
|
|
2058
|
+
this.context = options?.context;
|
|
2059
|
+
this.useBigInt64 = options?.useBigInt64 ?? false;
|
|
2060
|
+
this.maxDepth = options?.maxDepth ?? DEFAULT_MAX_DEPTH;
|
|
2061
|
+
this.initialBufferSize = options?.initialBufferSize ?? DEFAULT_INITIAL_BUFFER_SIZE;
|
|
2062
|
+
this.sortKeys = options?.sortKeys ?? false;
|
|
2063
|
+
this.forceFloat32 = options?.forceFloat32 ?? false;
|
|
2064
|
+
this.ignoreUndefined = options?.ignoreUndefined ?? false;
|
|
2065
|
+
this.forceIntegerToFloat = options?.forceIntegerToFloat ?? false;
|
|
2066
|
+
this.pos = 0;
|
|
2067
|
+
this.view = new DataView(new ArrayBuffer(this.initialBufferSize));
|
|
2068
|
+
this.bytes = new Uint8Array(this.view.buffer);
|
|
2069
|
+
}
|
|
2070
|
+
clone() {
|
|
2071
|
+
return new _Encoder({
|
|
2072
|
+
extensionCodec: this.extensionCodec,
|
|
2073
|
+
context: this.context,
|
|
2074
|
+
useBigInt64: this.useBigInt64,
|
|
2075
|
+
maxDepth: this.maxDepth,
|
|
2076
|
+
initialBufferSize: this.initialBufferSize,
|
|
2077
|
+
sortKeys: this.sortKeys,
|
|
2078
|
+
forceFloat32: this.forceFloat32,
|
|
2079
|
+
ignoreUndefined: this.ignoreUndefined,
|
|
2080
|
+
forceIntegerToFloat: this.forceIntegerToFloat
|
|
2081
|
+
});
|
|
2082
|
+
}
|
|
2083
|
+
reinitializeState() {
|
|
2084
|
+
this.pos = 0;
|
|
2085
|
+
}
|
|
2086
|
+
/**
|
|
2087
|
+
* This is almost equivalent to {@link Encoder#encode}, but it returns an reference of the encoder's internal buffer and thus much faster than {@link Encoder#encode}.
|
|
2088
|
+
*
|
|
2089
|
+
* @returns Encodes the object and returns a shared reference the encoder's internal buffer.
|
|
2090
|
+
*/
|
|
2091
|
+
encodeSharedRef(object) {
|
|
2092
|
+
if (this.entered) {
|
|
2093
|
+
const instance = this.clone();
|
|
2094
|
+
return instance.encodeSharedRef(object);
|
|
2095
|
+
}
|
|
2096
|
+
try {
|
|
2097
|
+
this.entered = true;
|
|
2098
|
+
this.reinitializeState();
|
|
2099
|
+
this.doEncode(object, 1);
|
|
2100
|
+
return this.bytes.subarray(0, this.pos);
|
|
2101
|
+
} finally {
|
|
2102
|
+
this.entered = false;
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
/**
|
|
2106
|
+
* @returns Encodes the object and returns a copy of the encoder's internal buffer.
|
|
2107
|
+
*/
|
|
2108
|
+
encode(object) {
|
|
2109
|
+
if (this.entered) {
|
|
2110
|
+
const instance = this.clone();
|
|
2111
|
+
return instance.encode(object);
|
|
2112
|
+
}
|
|
2113
|
+
try {
|
|
2114
|
+
this.entered = true;
|
|
2115
|
+
this.reinitializeState();
|
|
2116
|
+
this.doEncode(object, 1);
|
|
2117
|
+
return this.bytes.slice(0, this.pos);
|
|
2118
|
+
} finally {
|
|
2119
|
+
this.entered = false;
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
doEncode(object, depth) {
|
|
2123
|
+
if (depth > this.maxDepth) {
|
|
2124
|
+
throw new Error(`Too deep objects in depth ${depth}`);
|
|
2125
|
+
}
|
|
2126
|
+
if (object == null) {
|
|
2127
|
+
this.encodeNil();
|
|
2128
|
+
} else if (typeof object === "boolean") {
|
|
2129
|
+
this.encodeBoolean(object);
|
|
2130
|
+
} else if (typeof object === "number") {
|
|
2131
|
+
if (!this.forceIntegerToFloat) {
|
|
2132
|
+
this.encodeNumber(object);
|
|
2133
|
+
} else {
|
|
2134
|
+
this.encodeNumberAsFloat(object);
|
|
2135
|
+
}
|
|
2136
|
+
} else if (typeof object === "string") {
|
|
2137
|
+
this.encodeString(object);
|
|
2138
|
+
} else if (this.useBigInt64 && typeof object === "bigint") {
|
|
2139
|
+
this.encodeBigInt64(object);
|
|
2140
|
+
} else {
|
|
2141
|
+
this.encodeObject(object, depth);
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
ensureBufferSizeToWrite(sizeToWrite) {
|
|
2145
|
+
const requiredSize = this.pos + sizeToWrite;
|
|
2146
|
+
if (this.view.byteLength < requiredSize) {
|
|
2147
|
+
this.resizeBuffer(requiredSize * 2);
|
|
2148
|
+
}
|
|
2149
|
+
}
|
|
2150
|
+
resizeBuffer(newSize) {
|
|
2151
|
+
const newBuffer = new ArrayBuffer(newSize);
|
|
2152
|
+
const newBytes = new Uint8Array(newBuffer);
|
|
2153
|
+
const newView = new DataView(newBuffer);
|
|
2154
|
+
newBytes.set(this.bytes);
|
|
2155
|
+
this.view = newView;
|
|
2156
|
+
this.bytes = newBytes;
|
|
2157
|
+
}
|
|
2158
|
+
encodeNil() {
|
|
2159
|
+
this.writeU8(192);
|
|
2160
|
+
}
|
|
2161
|
+
encodeBoolean(object) {
|
|
2162
|
+
if (object === false) {
|
|
2163
|
+
this.writeU8(194);
|
|
2164
|
+
} else {
|
|
2165
|
+
this.writeU8(195);
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
encodeNumber(object) {
|
|
2169
|
+
if (!this.forceIntegerToFloat && Number.isSafeInteger(object)) {
|
|
2170
|
+
if (object >= 0) {
|
|
2171
|
+
if (object < 128) {
|
|
2172
|
+
this.writeU8(object);
|
|
2173
|
+
} else if (object < 256) {
|
|
2174
|
+
this.writeU8(204);
|
|
2175
|
+
this.writeU8(object);
|
|
2176
|
+
} else if (object < 65536) {
|
|
2177
|
+
this.writeU8(205);
|
|
2178
|
+
this.writeU16(object);
|
|
2179
|
+
} else if (object < 4294967296) {
|
|
2180
|
+
this.writeU8(206);
|
|
2181
|
+
this.writeU32(object);
|
|
2182
|
+
} else if (!this.useBigInt64) {
|
|
2183
|
+
this.writeU8(207);
|
|
2184
|
+
this.writeU64(object);
|
|
2185
|
+
} else {
|
|
2186
|
+
this.encodeNumberAsFloat(object);
|
|
2187
|
+
}
|
|
2188
|
+
} else {
|
|
2189
|
+
if (object >= -32) {
|
|
2190
|
+
this.writeU8(224 | object + 32);
|
|
2191
|
+
} else if (object >= -128) {
|
|
2192
|
+
this.writeU8(208);
|
|
2193
|
+
this.writeI8(object);
|
|
2194
|
+
} else if (object >= -32768) {
|
|
2195
|
+
this.writeU8(209);
|
|
2196
|
+
this.writeI16(object);
|
|
2197
|
+
} else if (object >= -2147483648) {
|
|
2198
|
+
this.writeU8(210);
|
|
2199
|
+
this.writeI32(object);
|
|
2200
|
+
} else if (!this.useBigInt64) {
|
|
2201
|
+
this.writeU8(211);
|
|
2202
|
+
this.writeI64(object);
|
|
2203
|
+
} else {
|
|
2204
|
+
this.encodeNumberAsFloat(object);
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
} else {
|
|
2208
|
+
this.encodeNumberAsFloat(object);
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
encodeNumberAsFloat(object) {
|
|
2212
|
+
if (this.forceFloat32) {
|
|
2213
|
+
this.writeU8(202);
|
|
2214
|
+
this.writeF32(object);
|
|
2215
|
+
} else {
|
|
2216
|
+
this.writeU8(203);
|
|
2217
|
+
this.writeF64(object);
|
|
2218
|
+
}
|
|
2219
|
+
}
|
|
2220
|
+
encodeBigInt64(object) {
|
|
2221
|
+
if (object >= BigInt(0)) {
|
|
2222
|
+
this.writeU8(207);
|
|
2223
|
+
this.writeBigUint64(object);
|
|
2224
|
+
} else {
|
|
2225
|
+
this.writeU8(211);
|
|
2226
|
+
this.writeBigInt64(object);
|
|
2227
|
+
}
|
|
2228
|
+
}
|
|
2229
|
+
writeStringHeader(byteLength) {
|
|
2230
|
+
if (byteLength < 32) {
|
|
2231
|
+
this.writeU8(160 + byteLength);
|
|
2232
|
+
} else if (byteLength < 256) {
|
|
2233
|
+
this.writeU8(217);
|
|
2234
|
+
this.writeU8(byteLength);
|
|
2235
|
+
} else if (byteLength < 65536) {
|
|
2236
|
+
this.writeU8(218);
|
|
2237
|
+
this.writeU16(byteLength);
|
|
2238
|
+
} else if (byteLength < 4294967296) {
|
|
2239
|
+
this.writeU8(219);
|
|
2240
|
+
this.writeU32(byteLength);
|
|
2241
|
+
} else {
|
|
2242
|
+
throw new Error(`Too long string: ${byteLength} bytes in UTF-8`);
|
|
2243
|
+
}
|
|
2244
|
+
}
|
|
2245
|
+
encodeString(object) {
|
|
2246
|
+
const maxHeaderSize = 1 + 4;
|
|
2247
|
+
const byteLength = utf8Count(object);
|
|
2248
|
+
this.ensureBufferSizeToWrite(maxHeaderSize + byteLength);
|
|
2249
|
+
this.writeStringHeader(byteLength);
|
|
2250
|
+
utf8Encode(object, this.bytes, this.pos);
|
|
2251
|
+
this.pos += byteLength;
|
|
2252
|
+
}
|
|
2253
|
+
encodeObject(object, depth) {
|
|
2254
|
+
const ext = this.extensionCodec.tryToEncode(object, this.context);
|
|
2255
|
+
if (ext != null) {
|
|
2256
|
+
this.encodeExtension(ext);
|
|
2257
|
+
} else if (Array.isArray(object)) {
|
|
2258
|
+
this.encodeArray(object, depth);
|
|
2259
|
+
} else if (ArrayBuffer.isView(object)) {
|
|
2260
|
+
this.encodeBinary(object);
|
|
2261
|
+
} else if (typeof object === "object") {
|
|
2262
|
+
this.encodeMap(object, depth);
|
|
2263
|
+
} else {
|
|
2264
|
+
throw new Error(`Unrecognized object: ${Object.prototype.toString.apply(object)}`);
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2267
|
+
encodeBinary(object) {
|
|
2268
|
+
const size = object.byteLength;
|
|
2269
|
+
if (size < 256) {
|
|
2270
|
+
this.writeU8(196);
|
|
2271
|
+
this.writeU8(size);
|
|
2272
|
+
} else if (size < 65536) {
|
|
2273
|
+
this.writeU8(197);
|
|
2274
|
+
this.writeU16(size);
|
|
2275
|
+
} else if (size < 4294967296) {
|
|
2276
|
+
this.writeU8(198);
|
|
2277
|
+
this.writeU32(size);
|
|
2278
|
+
} else {
|
|
2279
|
+
throw new Error(`Too large binary: ${size}`);
|
|
2280
|
+
}
|
|
2281
|
+
const bytes = ensureUint8Array(object);
|
|
2282
|
+
this.writeU8a(bytes);
|
|
2283
|
+
}
|
|
2284
|
+
encodeArray(object, depth) {
|
|
2285
|
+
const size = object.length;
|
|
2286
|
+
if (size < 16) {
|
|
2287
|
+
this.writeU8(144 + size);
|
|
2288
|
+
} else if (size < 65536) {
|
|
2289
|
+
this.writeU8(220);
|
|
2290
|
+
this.writeU16(size);
|
|
2291
|
+
} else if (size < 4294967296) {
|
|
2292
|
+
this.writeU8(221);
|
|
2293
|
+
this.writeU32(size);
|
|
2294
|
+
} else {
|
|
2295
|
+
throw new Error(`Too large array: ${size}`);
|
|
2296
|
+
}
|
|
2297
|
+
for (const item of object) {
|
|
2298
|
+
this.doEncode(item, depth + 1);
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
countWithoutUndefined(object, keys) {
|
|
2302
|
+
let count = 0;
|
|
2303
|
+
for (const key of keys) {
|
|
2304
|
+
if (object[key] !== void 0) {
|
|
2305
|
+
count++;
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
2308
|
+
return count;
|
|
2309
|
+
}
|
|
2310
|
+
encodeMap(object, depth) {
|
|
2311
|
+
const keys = Object.keys(object);
|
|
2312
|
+
if (this.sortKeys) {
|
|
2313
|
+
keys.sort();
|
|
2314
|
+
}
|
|
2315
|
+
const size = this.ignoreUndefined ? this.countWithoutUndefined(object, keys) : keys.length;
|
|
2316
|
+
if (size < 16) {
|
|
2317
|
+
this.writeU8(128 + size);
|
|
2318
|
+
} else if (size < 65536) {
|
|
2319
|
+
this.writeU8(222);
|
|
2320
|
+
this.writeU16(size);
|
|
2321
|
+
} else if (size < 4294967296) {
|
|
2322
|
+
this.writeU8(223);
|
|
2323
|
+
this.writeU32(size);
|
|
2324
|
+
} else {
|
|
2325
|
+
throw new Error(`Too large map object: ${size}`);
|
|
2326
|
+
}
|
|
2327
|
+
for (const key of keys) {
|
|
2328
|
+
const value = object[key];
|
|
2329
|
+
if (!(this.ignoreUndefined && value === void 0)) {
|
|
2330
|
+
this.encodeString(key);
|
|
2331
|
+
this.doEncode(value, depth + 1);
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
encodeExtension(ext) {
|
|
2336
|
+
if (typeof ext.data === "function") {
|
|
2337
|
+
const data = ext.data(this.pos + 6);
|
|
2338
|
+
const size2 = data.length;
|
|
2339
|
+
if (size2 >= 4294967296) {
|
|
2340
|
+
throw new Error(`Too large extension object: ${size2}`);
|
|
2341
|
+
}
|
|
2342
|
+
this.writeU8(201);
|
|
2343
|
+
this.writeU32(size2);
|
|
2344
|
+
this.writeI8(ext.type);
|
|
2345
|
+
this.writeU8a(data);
|
|
2346
|
+
return;
|
|
2347
|
+
}
|
|
2348
|
+
const size = ext.data.length;
|
|
2349
|
+
if (size === 1) {
|
|
2350
|
+
this.writeU8(212);
|
|
2351
|
+
} else if (size === 2) {
|
|
2352
|
+
this.writeU8(213);
|
|
2353
|
+
} else if (size === 4) {
|
|
2354
|
+
this.writeU8(214);
|
|
2355
|
+
} else if (size === 8) {
|
|
2356
|
+
this.writeU8(215);
|
|
2357
|
+
} else if (size === 16) {
|
|
2358
|
+
this.writeU8(216);
|
|
2359
|
+
} else if (size < 256) {
|
|
2360
|
+
this.writeU8(199);
|
|
2361
|
+
this.writeU8(size);
|
|
2362
|
+
} else if (size < 65536) {
|
|
2363
|
+
this.writeU8(200);
|
|
2364
|
+
this.writeU16(size);
|
|
2365
|
+
} else if (size < 4294967296) {
|
|
2366
|
+
this.writeU8(201);
|
|
2367
|
+
this.writeU32(size);
|
|
2368
|
+
} else {
|
|
2369
|
+
throw new Error(`Too large extension object: ${size}`);
|
|
2370
|
+
}
|
|
2371
|
+
this.writeI8(ext.type);
|
|
2372
|
+
this.writeU8a(ext.data);
|
|
2373
|
+
}
|
|
2374
|
+
writeU8(value) {
|
|
2375
|
+
this.ensureBufferSizeToWrite(1);
|
|
2376
|
+
this.view.setUint8(this.pos, value);
|
|
2377
|
+
this.pos++;
|
|
2378
|
+
}
|
|
2379
|
+
writeU8a(values) {
|
|
2380
|
+
const size = values.length;
|
|
2381
|
+
this.ensureBufferSizeToWrite(size);
|
|
2382
|
+
this.bytes.set(values, this.pos);
|
|
2383
|
+
this.pos += size;
|
|
2384
|
+
}
|
|
2385
|
+
writeI8(value) {
|
|
2386
|
+
this.ensureBufferSizeToWrite(1);
|
|
2387
|
+
this.view.setInt8(this.pos, value);
|
|
2388
|
+
this.pos++;
|
|
2389
|
+
}
|
|
2390
|
+
writeU16(value) {
|
|
2391
|
+
this.ensureBufferSizeToWrite(2);
|
|
2392
|
+
this.view.setUint16(this.pos, value);
|
|
2393
|
+
this.pos += 2;
|
|
2394
|
+
}
|
|
2395
|
+
writeI16(value) {
|
|
2396
|
+
this.ensureBufferSizeToWrite(2);
|
|
2397
|
+
this.view.setInt16(this.pos, value);
|
|
2398
|
+
this.pos += 2;
|
|
2399
|
+
}
|
|
2400
|
+
writeU32(value) {
|
|
2401
|
+
this.ensureBufferSizeToWrite(4);
|
|
2402
|
+
this.view.setUint32(this.pos, value);
|
|
2403
|
+
this.pos += 4;
|
|
2404
|
+
}
|
|
2405
|
+
writeI32(value) {
|
|
2406
|
+
this.ensureBufferSizeToWrite(4);
|
|
2407
|
+
this.view.setInt32(this.pos, value);
|
|
2408
|
+
this.pos += 4;
|
|
2409
|
+
}
|
|
2410
|
+
writeF32(value) {
|
|
2411
|
+
this.ensureBufferSizeToWrite(4);
|
|
2412
|
+
this.view.setFloat32(this.pos, value);
|
|
2413
|
+
this.pos += 4;
|
|
2414
|
+
}
|
|
2415
|
+
writeF64(value) {
|
|
2416
|
+
this.ensureBufferSizeToWrite(8);
|
|
2417
|
+
this.view.setFloat64(this.pos, value);
|
|
2418
|
+
this.pos += 8;
|
|
2419
|
+
}
|
|
2420
|
+
writeU64(value) {
|
|
2421
|
+
this.ensureBufferSizeToWrite(8);
|
|
2422
|
+
setUint64(this.view, this.pos, value);
|
|
2423
|
+
this.pos += 8;
|
|
2424
|
+
}
|
|
2425
|
+
writeI64(value) {
|
|
2426
|
+
this.ensureBufferSizeToWrite(8);
|
|
2427
|
+
setInt64(this.view, this.pos, value);
|
|
2428
|
+
this.pos += 8;
|
|
2429
|
+
}
|
|
2430
|
+
writeBigUint64(value) {
|
|
2431
|
+
this.ensureBufferSizeToWrite(8);
|
|
2432
|
+
this.view.setBigUint64(this.pos, value);
|
|
2433
|
+
this.pos += 8;
|
|
2434
|
+
}
|
|
2435
|
+
writeBigInt64(value) {
|
|
2436
|
+
this.ensureBufferSizeToWrite(8);
|
|
2437
|
+
this.view.setBigInt64(this.pos, value);
|
|
2438
|
+
this.pos += 8;
|
|
2439
|
+
}
|
|
2440
|
+
};
|
|
2441
|
+
}
|
|
2442
|
+
});
|
|
2443
|
+
|
|
2444
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/encode.mjs
|
|
2445
|
+
function encode(value, options) {
|
|
2446
|
+
const encoder = new Encoder(options);
|
|
2447
|
+
return encoder.encodeSharedRef(value);
|
|
2448
|
+
}
|
|
2449
|
+
var init_encode = __esm({
|
|
2450
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/encode.mjs"() {
|
|
2451
|
+
"use strict";
|
|
2452
|
+
init_Encoder();
|
|
2453
|
+
}
|
|
2454
|
+
});
|
|
2455
|
+
|
|
2456
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/prettyByte.mjs
|
|
2457
|
+
function prettyByte(byte) {
|
|
2458
|
+
return `${byte < 0 ? "-" : ""}0x${Math.abs(byte).toString(16).padStart(2, "0")}`;
|
|
2459
|
+
}
|
|
2460
|
+
var init_prettyByte = __esm({
|
|
2461
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/prettyByte.mjs"() {
|
|
2462
|
+
"use strict";
|
|
2463
|
+
}
|
|
2464
|
+
});
|
|
2465
|
+
|
|
2466
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/CachedKeyDecoder.mjs
|
|
2467
|
+
var DEFAULT_MAX_KEY_LENGTH, DEFAULT_MAX_LENGTH_PER_KEY, CachedKeyDecoder;
|
|
2468
|
+
var init_CachedKeyDecoder = __esm({
|
|
2469
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/CachedKeyDecoder.mjs"() {
|
|
2470
|
+
"use strict";
|
|
2471
|
+
init_utf8();
|
|
2472
|
+
DEFAULT_MAX_KEY_LENGTH = 16;
|
|
2473
|
+
DEFAULT_MAX_LENGTH_PER_KEY = 16;
|
|
2474
|
+
CachedKeyDecoder = class {
|
|
2475
|
+
hit = 0;
|
|
2476
|
+
miss = 0;
|
|
2477
|
+
caches;
|
|
2478
|
+
maxKeyLength;
|
|
2479
|
+
maxLengthPerKey;
|
|
2480
|
+
constructor(maxKeyLength = DEFAULT_MAX_KEY_LENGTH, maxLengthPerKey = DEFAULT_MAX_LENGTH_PER_KEY) {
|
|
2481
|
+
this.maxKeyLength = maxKeyLength;
|
|
2482
|
+
this.maxLengthPerKey = maxLengthPerKey;
|
|
2483
|
+
this.caches = [];
|
|
2484
|
+
for (let i = 0; i < this.maxKeyLength; i++) {
|
|
2485
|
+
this.caches.push([]);
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
canBeCached(byteLength) {
|
|
2489
|
+
return byteLength > 0 && byteLength <= this.maxKeyLength;
|
|
2490
|
+
}
|
|
2491
|
+
find(bytes, inputOffset, byteLength) {
|
|
2492
|
+
const records = this.caches[byteLength - 1];
|
|
2493
|
+
FIND_CHUNK: for (const record of records) {
|
|
2494
|
+
const recordBytes = record.bytes;
|
|
2495
|
+
for (let j = 0; j < byteLength; j++) {
|
|
2496
|
+
if (recordBytes[j] !== bytes[inputOffset + j]) {
|
|
2497
|
+
continue FIND_CHUNK;
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
return record.str;
|
|
2501
|
+
}
|
|
2502
|
+
return null;
|
|
2503
|
+
}
|
|
2504
|
+
store(bytes, value) {
|
|
2505
|
+
const records = this.caches[bytes.length - 1];
|
|
2506
|
+
const record = { bytes, str: value };
|
|
2507
|
+
if (records.length >= this.maxLengthPerKey) {
|
|
2508
|
+
records[Math.random() * records.length | 0] = record;
|
|
2509
|
+
} else {
|
|
2510
|
+
records.push(record);
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
decode(bytes, inputOffset, byteLength) {
|
|
2514
|
+
const cachedValue = this.find(bytes, inputOffset, byteLength);
|
|
2515
|
+
if (cachedValue != null) {
|
|
2516
|
+
this.hit++;
|
|
2517
|
+
return cachedValue;
|
|
2518
|
+
}
|
|
2519
|
+
this.miss++;
|
|
2520
|
+
const str = utf8DecodeJs(bytes, inputOffset, byteLength);
|
|
2521
|
+
const slicedCopyOfBytes = Uint8Array.prototype.slice.call(bytes, inputOffset, inputOffset + byteLength);
|
|
2522
|
+
this.store(slicedCopyOfBytes, str);
|
|
2523
|
+
return str;
|
|
2524
|
+
}
|
|
2525
|
+
};
|
|
2526
|
+
}
|
|
2527
|
+
});
|
|
2528
|
+
|
|
2529
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/Decoder.mjs
|
|
2530
|
+
var STATE_ARRAY, STATE_MAP_KEY, STATE_MAP_VALUE, mapKeyConverter, StackPool, HEAD_BYTE_REQUIRED, EMPTY_VIEW, EMPTY_BYTES, MORE_DATA, sharedCachedKeyDecoder, Decoder;
|
|
2531
|
+
var init_Decoder = __esm({
|
|
2532
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/Decoder.mjs"() {
|
|
2533
|
+
"use strict";
|
|
2534
|
+
init_prettyByte();
|
|
2535
|
+
init_ExtensionCodec();
|
|
2536
|
+
init_int();
|
|
2537
|
+
init_utf8();
|
|
2538
|
+
init_typedArrays();
|
|
2539
|
+
init_CachedKeyDecoder();
|
|
2540
|
+
init_DecodeError();
|
|
2541
|
+
STATE_ARRAY = "array";
|
|
2542
|
+
STATE_MAP_KEY = "map_key";
|
|
2543
|
+
STATE_MAP_VALUE = "map_value";
|
|
2544
|
+
mapKeyConverter = (key) => {
|
|
2545
|
+
if (typeof key === "string" || typeof key === "number") {
|
|
2546
|
+
return key;
|
|
2547
|
+
}
|
|
2548
|
+
throw new DecodeError("The type of key must be string or number but " + typeof key);
|
|
2549
|
+
};
|
|
2550
|
+
StackPool = class {
|
|
2551
|
+
stack = [];
|
|
2552
|
+
stackHeadPosition = -1;
|
|
2553
|
+
get length() {
|
|
2554
|
+
return this.stackHeadPosition + 1;
|
|
2555
|
+
}
|
|
2556
|
+
top() {
|
|
2557
|
+
return this.stack[this.stackHeadPosition];
|
|
2558
|
+
}
|
|
2559
|
+
pushArrayState(size) {
|
|
2560
|
+
const state = this.getUninitializedStateFromPool();
|
|
2561
|
+
state.type = STATE_ARRAY;
|
|
2562
|
+
state.position = 0;
|
|
2563
|
+
state.size = size;
|
|
2564
|
+
state.array = new Array(size);
|
|
2565
|
+
}
|
|
2566
|
+
pushMapState(size) {
|
|
2567
|
+
const state = this.getUninitializedStateFromPool();
|
|
2568
|
+
state.type = STATE_MAP_KEY;
|
|
2569
|
+
state.readCount = 0;
|
|
2570
|
+
state.size = size;
|
|
2571
|
+
state.map = {};
|
|
2572
|
+
}
|
|
2573
|
+
getUninitializedStateFromPool() {
|
|
2574
|
+
this.stackHeadPosition++;
|
|
2575
|
+
if (this.stackHeadPosition === this.stack.length) {
|
|
2576
|
+
const partialState = {
|
|
2577
|
+
type: void 0,
|
|
2578
|
+
size: 0,
|
|
2579
|
+
array: void 0,
|
|
2580
|
+
position: 0,
|
|
2581
|
+
readCount: 0,
|
|
2582
|
+
map: void 0,
|
|
2583
|
+
key: null
|
|
2584
|
+
};
|
|
2585
|
+
this.stack.push(partialState);
|
|
2586
|
+
}
|
|
2587
|
+
return this.stack[this.stackHeadPosition];
|
|
2588
|
+
}
|
|
2589
|
+
release(state) {
|
|
2590
|
+
const topStackState = this.stack[this.stackHeadPosition];
|
|
2591
|
+
if (topStackState !== state) {
|
|
2592
|
+
throw new Error("Invalid stack state. Released state is not on top of the stack.");
|
|
2593
|
+
}
|
|
2594
|
+
if (state.type === STATE_ARRAY) {
|
|
2595
|
+
const partialState = state;
|
|
2596
|
+
partialState.size = 0;
|
|
2597
|
+
partialState.array = void 0;
|
|
2598
|
+
partialState.position = 0;
|
|
2599
|
+
partialState.type = void 0;
|
|
2600
|
+
}
|
|
2601
|
+
if (state.type === STATE_MAP_KEY || state.type === STATE_MAP_VALUE) {
|
|
2602
|
+
const partialState = state;
|
|
2603
|
+
partialState.size = 0;
|
|
2604
|
+
partialState.map = void 0;
|
|
2605
|
+
partialState.readCount = 0;
|
|
2606
|
+
partialState.type = void 0;
|
|
2607
|
+
}
|
|
2608
|
+
this.stackHeadPosition--;
|
|
2609
|
+
}
|
|
2610
|
+
reset() {
|
|
2611
|
+
this.stack.length = 0;
|
|
2612
|
+
this.stackHeadPosition = -1;
|
|
2613
|
+
}
|
|
2614
|
+
};
|
|
2615
|
+
HEAD_BYTE_REQUIRED = -1;
|
|
2616
|
+
EMPTY_VIEW = new DataView(new ArrayBuffer(0));
|
|
2617
|
+
EMPTY_BYTES = new Uint8Array(EMPTY_VIEW.buffer);
|
|
2618
|
+
try {
|
|
2619
|
+
EMPTY_VIEW.getInt8(0);
|
|
2620
|
+
} catch (e) {
|
|
2621
|
+
if (!(e instanceof RangeError)) {
|
|
2622
|
+
throw new Error("This module is not supported in the current JavaScript engine because DataView does not throw RangeError on out-of-bounds access");
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
MORE_DATA = new RangeError("Insufficient data");
|
|
2626
|
+
sharedCachedKeyDecoder = new CachedKeyDecoder();
|
|
2627
|
+
Decoder = class _Decoder {
|
|
2628
|
+
extensionCodec;
|
|
2629
|
+
context;
|
|
2630
|
+
useBigInt64;
|
|
2631
|
+
rawStrings;
|
|
2632
|
+
maxStrLength;
|
|
2633
|
+
maxBinLength;
|
|
2634
|
+
maxArrayLength;
|
|
2635
|
+
maxMapLength;
|
|
2636
|
+
maxExtLength;
|
|
2637
|
+
keyDecoder;
|
|
2638
|
+
mapKeyConverter;
|
|
2639
|
+
totalPos = 0;
|
|
2640
|
+
pos = 0;
|
|
2641
|
+
view = EMPTY_VIEW;
|
|
2642
|
+
bytes = EMPTY_BYTES;
|
|
2643
|
+
headByte = HEAD_BYTE_REQUIRED;
|
|
2644
|
+
stack = new StackPool();
|
|
2645
|
+
entered = false;
|
|
2646
|
+
constructor(options) {
|
|
2647
|
+
this.extensionCodec = options?.extensionCodec ?? ExtensionCodec.defaultCodec;
|
|
2648
|
+
this.context = options?.context;
|
|
2649
|
+
this.useBigInt64 = options?.useBigInt64 ?? false;
|
|
2650
|
+
this.rawStrings = options?.rawStrings ?? false;
|
|
2651
|
+
this.maxStrLength = options?.maxStrLength ?? UINT32_MAX;
|
|
2652
|
+
this.maxBinLength = options?.maxBinLength ?? UINT32_MAX;
|
|
2653
|
+
this.maxArrayLength = options?.maxArrayLength ?? UINT32_MAX;
|
|
2654
|
+
this.maxMapLength = options?.maxMapLength ?? UINT32_MAX;
|
|
2655
|
+
this.maxExtLength = options?.maxExtLength ?? UINT32_MAX;
|
|
2656
|
+
this.keyDecoder = options?.keyDecoder !== void 0 ? options.keyDecoder : sharedCachedKeyDecoder;
|
|
2657
|
+
this.mapKeyConverter = options?.mapKeyConverter ?? mapKeyConverter;
|
|
2658
|
+
}
|
|
2659
|
+
clone() {
|
|
2660
|
+
return new _Decoder({
|
|
2661
|
+
extensionCodec: this.extensionCodec,
|
|
2662
|
+
context: this.context,
|
|
2663
|
+
useBigInt64: this.useBigInt64,
|
|
2664
|
+
rawStrings: this.rawStrings,
|
|
2665
|
+
maxStrLength: this.maxStrLength,
|
|
2666
|
+
maxBinLength: this.maxBinLength,
|
|
2667
|
+
maxArrayLength: this.maxArrayLength,
|
|
2668
|
+
maxMapLength: this.maxMapLength,
|
|
2669
|
+
maxExtLength: this.maxExtLength,
|
|
2670
|
+
keyDecoder: this.keyDecoder
|
|
2671
|
+
});
|
|
2672
|
+
}
|
|
2673
|
+
reinitializeState() {
|
|
2674
|
+
this.totalPos = 0;
|
|
2675
|
+
this.headByte = HEAD_BYTE_REQUIRED;
|
|
2676
|
+
this.stack.reset();
|
|
2677
|
+
}
|
|
2678
|
+
setBuffer(buffer) {
|
|
2679
|
+
const bytes = ensureUint8Array(buffer);
|
|
2680
|
+
this.bytes = bytes;
|
|
2681
|
+
this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2682
|
+
this.pos = 0;
|
|
2683
|
+
}
|
|
2684
|
+
appendBuffer(buffer) {
|
|
2685
|
+
if (this.headByte === HEAD_BYTE_REQUIRED && !this.hasRemaining(1)) {
|
|
2686
|
+
this.setBuffer(buffer);
|
|
2687
|
+
} else {
|
|
2688
|
+
const remainingData = this.bytes.subarray(this.pos);
|
|
2689
|
+
const newData = ensureUint8Array(buffer);
|
|
2690
|
+
const newBuffer = new Uint8Array(remainingData.length + newData.length);
|
|
2691
|
+
newBuffer.set(remainingData);
|
|
2692
|
+
newBuffer.set(newData, remainingData.length);
|
|
2693
|
+
this.setBuffer(newBuffer);
|
|
2694
|
+
}
|
|
2695
|
+
}
|
|
2696
|
+
hasRemaining(size) {
|
|
2697
|
+
return this.view.byteLength - this.pos >= size;
|
|
2698
|
+
}
|
|
2699
|
+
createExtraByteError(posToShow) {
|
|
2700
|
+
const { view, pos } = this;
|
|
2701
|
+
return new RangeError(`Extra ${view.byteLength - pos} of ${view.byteLength} byte(s) found at buffer[${posToShow}]`);
|
|
2702
|
+
}
|
|
2703
|
+
/**
|
|
2704
|
+
* @throws {@link DecodeError}
|
|
2705
|
+
* @throws {@link RangeError}
|
|
2706
|
+
*/
|
|
2707
|
+
decode(buffer) {
|
|
2708
|
+
if (this.entered) {
|
|
2709
|
+
const instance = this.clone();
|
|
2710
|
+
return instance.decode(buffer);
|
|
2711
|
+
}
|
|
2712
|
+
try {
|
|
2713
|
+
this.entered = true;
|
|
2714
|
+
this.reinitializeState();
|
|
2715
|
+
this.setBuffer(buffer);
|
|
2716
|
+
const object = this.doDecodeSync();
|
|
2717
|
+
if (this.hasRemaining(1)) {
|
|
2718
|
+
throw this.createExtraByteError(this.pos);
|
|
2719
|
+
}
|
|
2720
|
+
return object;
|
|
2721
|
+
} finally {
|
|
2722
|
+
this.entered = false;
|
|
2723
|
+
}
|
|
2724
|
+
}
|
|
2725
|
+
*decodeMulti(buffer) {
|
|
2726
|
+
if (this.entered) {
|
|
2727
|
+
const instance = this.clone();
|
|
2728
|
+
yield* instance.decodeMulti(buffer);
|
|
2729
|
+
return;
|
|
2730
|
+
}
|
|
2731
|
+
try {
|
|
2732
|
+
this.entered = true;
|
|
2733
|
+
this.reinitializeState();
|
|
2734
|
+
this.setBuffer(buffer);
|
|
2735
|
+
while (this.hasRemaining(1)) {
|
|
2736
|
+
yield this.doDecodeSync();
|
|
2737
|
+
}
|
|
2738
|
+
} finally {
|
|
2739
|
+
this.entered = false;
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2742
|
+
async decodeAsync(stream) {
|
|
2743
|
+
if (this.entered) {
|
|
2744
|
+
const instance = this.clone();
|
|
2745
|
+
return instance.decodeAsync(stream);
|
|
2746
|
+
}
|
|
2747
|
+
try {
|
|
2748
|
+
this.entered = true;
|
|
2749
|
+
let decoded = false;
|
|
2750
|
+
let object;
|
|
2751
|
+
for await (const buffer of stream) {
|
|
2752
|
+
if (decoded) {
|
|
2753
|
+
this.entered = false;
|
|
2754
|
+
throw this.createExtraByteError(this.totalPos);
|
|
2755
|
+
}
|
|
2756
|
+
this.appendBuffer(buffer);
|
|
2757
|
+
try {
|
|
2758
|
+
object = this.doDecodeSync();
|
|
2759
|
+
decoded = true;
|
|
2760
|
+
} catch (e) {
|
|
2761
|
+
if (!(e instanceof RangeError)) {
|
|
2762
|
+
throw e;
|
|
2763
|
+
}
|
|
2764
|
+
}
|
|
2765
|
+
this.totalPos += this.pos;
|
|
2766
|
+
}
|
|
2767
|
+
if (decoded) {
|
|
2768
|
+
if (this.hasRemaining(1)) {
|
|
2769
|
+
throw this.createExtraByteError(this.totalPos);
|
|
2770
|
+
}
|
|
2771
|
+
return object;
|
|
2772
|
+
}
|
|
2773
|
+
const { headByte, pos, totalPos } = this;
|
|
2774
|
+
throw new RangeError(`Insufficient data in parsing ${prettyByte(headByte)} at ${totalPos} (${pos} in the current buffer)`);
|
|
2775
|
+
} finally {
|
|
2776
|
+
this.entered = false;
|
|
2777
|
+
}
|
|
2778
|
+
}
|
|
2779
|
+
decodeArrayStream(stream) {
|
|
2780
|
+
return this.decodeMultiAsync(stream, true);
|
|
2781
|
+
}
|
|
2782
|
+
decodeStream(stream) {
|
|
2783
|
+
return this.decodeMultiAsync(stream, false);
|
|
2784
|
+
}
|
|
2785
|
+
async *decodeMultiAsync(stream, isArray) {
|
|
2786
|
+
if (this.entered) {
|
|
2787
|
+
const instance = this.clone();
|
|
2788
|
+
yield* instance.decodeMultiAsync(stream, isArray);
|
|
2789
|
+
return;
|
|
2790
|
+
}
|
|
2791
|
+
try {
|
|
2792
|
+
this.entered = true;
|
|
2793
|
+
let isArrayHeaderRequired = isArray;
|
|
2794
|
+
let arrayItemsLeft = -1;
|
|
2795
|
+
for await (const buffer of stream) {
|
|
2796
|
+
if (isArray && arrayItemsLeft === 0) {
|
|
2797
|
+
throw this.createExtraByteError(this.totalPos);
|
|
2798
|
+
}
|
|
2799
|
+
this.appendBuffer(buffer);
|
|
2800
|
+
if (isArrayHeaderRequired) {
|
|
2801
|
+
arrayItemsLeft = this.readArraySize();
|
|
2802
|
+
isArrayHeaderRequired = false;
|
|
2803
|
+
this.complete();
|
|
2804
|
+
}
|
|
2805
|
+
try {
|
|
2806
|
+
while (true) {
|
|
2807
|
+
yield this.doDecodeSync();
|
|
2808
|
+
if (--arrayItemsLeft === 0) {
|
|
2809
|
+
break;
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2812
|
+
} catch (e) {
|
|
2813
|
+
if (!(e instanceof RangeError)) {
|
|
2814
|
+
throw e;
|
|
2815
|
+
}
|
|
2816
|
+
}
|
|
2817
|
+
this.totalPos += this.pos;
|
|
2818
|
+
}
|
|
2819
|
+
} finally {
|
|
2820
|
+
this.entered = false;
|
|
2821
|
+
}
|
|
2822
|
+
}
|
|
2823
|
+
doDecodeSync() {
|
|
2824
|
+
DECODE: while (true) {
|
|
2825
|
+
const headByte = this.readHeadByte();
|
|
2826
|
+
let object;
|
|
2827
|
+
if (headByte >= 224) {
|
|
2828
|
+
object = headByte - 256;
|
|
2829
|
+
} else if (headByte < 192) {
|
|
2830
|
+
if (headByte < 128) {
|
|
2831
|
+
object = headByte;
|
|
2832
|
+
} else if (headByte < 144) {
|
|
2833
|
+
const size = headByte - 128;
|
|
2834
|
+
if (size !== 0) {
|
|
2835
|
+
this.pushMapState(size);
|
|
2836
|
+
this.complete();
|
|
2837
|
+
continue DECODE;
|
|
2838
|
+
} else {
|
|
2839
|
+
object = {};
|
|
2840
|
+
}
|
|
2841
|
+
} else if (headByte < 160) {
|
|
2842
|
+
const size = headByte - 144;
|
|
2843
|
+
if (size !== 0) {
|
|
2844
|
+
this.pushArrayState(size);
|
|
2845
|
+
this.complete();
|
|
2846
|
+
continue DECODE;
|
|
2847
|
+
} else {
|
|
2848
|
+
object = [];
|
|
2849
|
+
}
|
|
2850
|
+
} else {
|
|
2851
|
+
const byteLength = headByte - 160;
|
|
2852
|
+
object = this.decodeString(byteLength, 0);
|
|
2853
|
+
}
|
|
2854
|
+
} else if (headByte === 192) {
|
|
2855
|
+
object = null;
|
|
2856
|
+
} else if (headByte === 194) {
|
|
2857
|
+
object = false;
|
|
2858
|
+
} else if (headByte === 195) {
|
|
2859
|
+
object = true;
|
|
2860
|
+
} else if (headByte === 202) {
|
|
2861
|
+
object = this.readF32();
|
|
2862
|
+
} else if (headByte === 203) {
|
|
2863
|
+
object = this.readF64();
|
|
2864
|
+
} else if (headByte === 204) {
|
|
2865
|
+
object = this.readU8();
|
|
2866
|
+
} else if (headByte === 205) {
|
|
2867
|
+
object = this.readU16();
|
|
2868
|
+
} else if (headByte === 206) {
|
|
2869
|
+
object = this.readU32();
|
|
2870
|
+
} else if (headByte === 207) {
|
|
2871
|
+
if (this.useBigInt64) {
|
|
2872
|
+
object = this.readU64AsBigInt();
|
|
2873
|
+
} else {
|
|
2874
|
+
object = this.readU64();
|
|
2875
|
+
}
|
|
2876
|
+
} else if (headByte === 208) {
|
|
2877
|
+
object = this.readI8();
|
|
2878
|
+
} else if (headByte === 209) {
|
|
2879
|
+
object = this.readI16();
|
|
2880
|
+
} else if (headByte === 210) {
|
|
2881
|
+
object = this.readI32();
|
|
2882
|
+
} else if (headByte === 211) {
|
|
2883
|
+
if (this.useBigInt64) {
|
|
2884
|
+
object = this.readI64AsBigInt();
|
|
2885
|
+
} else {
|
|
2886
|
+
object = this.readI64();
|
|
2887
|
+
}
|
|
2888
|
+
} else if (headByte === 217) {
|
|
2889
|
+
const byteLength = this.lookU8();
|
|
2890
|
+
object = this.decodeString(byteLength, 1);
|
|
2891
|
+
} else if (headByte === 218) {
|
|
2892
|
+
const byteLength = this.lookU16();
|
|
2893
|
+
object = this.decodeString(byteLength, 2);
|
|
2894
|
+
} else if (headByte === 219) {
|
|
2895
|
+
const byteLength = this.lookU32();
|
|
2896
|
+
object = this.decodeString(byteLength, 4);
|
|
2897
|
+
} else if (headByte === 220) {
|
|
2898
|
+
const size = this.readU16();
|
|
2899
|
+
if (size !== 0) {
|
|
2900
|
+
this.pushArrayState(size);
|
|
2901
|
+
this.complete();
|
|
2902
|
+
continue DECODE;
|
|
2903
|
+
} else {
|
|
2904
|
+
object = [];
|
|
2905
|
+
}
|
|
2906
|
+
} else if (headByte === 221) {
|
|
2907
|
+
const size = this.readU32();
|
|
2908
|
+
if (size !== 0) {
|
|
2909
|
+
this.pushArrayState(size);
|
|
2910
|
+
this.complete();
|
|
2911
|
+
continue DECODE;
|
|
2912
|
+
} else {
|
|
2913
|
+
object = [];
|
|
2914
|
+
}
|
|
2915
|
+
} else if (headByte === 222) {
|
|
2916
|
+
const size = this.readU16();
|
|
2917
|
+
if (size !== 0) {
|
|
2918
|
+
this.pushMapState(size);
|
|
2919
|
+
this.complete();
|
|
2920
|
+
continue DECODE;
|
|
2921
|
+
} else {
|
|
2922
|
+
object = {};
|
|
2923
|
+
}
|
|
2924
|
+
} else if (headByte === 223) {
|
|
2925
|
+
const size = this.readU32();
|
|
2926
|
+
if (size !== 0) {
|
|
2927
|
+
this.pushMapState(size);
|
|
2928
|
+
this.complete();
|
|
2929
|
+
continue DECODE;
|
|
2930
|
+
} else {
|
|
2931
|
+
object = {};
|
|
2932
|
+
}
|
|
2933
|
+
} else if (headByte === 196) {
|
|
2934
|
+
const size = this.lookU8();
|
|
2935
|
+
object = this.decodeBinary(size, 1);
|
|
2936
|
+
} else if (headByte === 197) {
|
|
2937
|
+
const size = this.lookU16();
|
|
2938
|
+
object = this.decodeBinary(size, 2);
|
|
2939
|
+
} else if (headByte === 198) {
|
|
2940
|
+
const size = this.lookU32();
|
|
2941
|
+
object = this.decodeBinary(size, 4);
|
|
2942
|
+
} else if (headByte === 212) {
|
|
2943
|
+
object = this.decodeExtension(1, 0);
|
|
2944
|
+
} else if (headByte === 213) {
|
|
2945
|
+
object = this.decodeExtension(2, 0);
|
|
2946
|
+
} else if (headByte === 214) {
|
|
2947
|
+
object = this.decodeExtension(4, 0);
|
|
2948
|
+
} else if (headByte === 215) {
|
|
2949
|
+
object = this.decodeExtension(8, 0);
|
|
2950
|
+
} else if (headByte === 216) {
|
|
2951
|
+
object = this.decodeExtension(16, 0);
|
|
2952
|
+
} else if (headByte === 199) {
|
|
2953
|
+
const size = this.lookU8();
|
|
2954
|
+
object = this.decodeExtension(size, 1);
|
|
2955
|
+
} else if (headByte === 200) {
|
|
2956
|
+
const size = this.lookU16();
|
|
2957
|
+
object = this.decodeExtension(size, 2);
|
|
2958
|
+
} else if (headByte === 201) {
|
|
2959
|
+
const size = this.lookU32();
|
|
2960
|
+
object = this.decodeExtension(size, 4);
|
|
2961
|
+
} else {
|
|
2962
|
+
throw new DecodeError(`Unrecognized type byte: ${prettyByte(headByte)}`);
|
|
2963
|
+
}
|
|
2964
|
+
this.complete();
|
|
2965
|
+
const stack = this.stack;
|
|
2966
|
+
while (stack.length > 0) {
|
|
2967
|
+
const state = stack.top();
|
|
2968
|
+
if (state.type === STATE_ARRAY) {
|
|
2969
|
+
state.array[state.position] = object;
|
|
2970
|
+
state.position++;
|
|
2971
|
+
if (state.position === state.size) {
|
|
2972
|
+
object = state.array;
|
|
2973
|
+
stack.release(state);
|
|
2974
|
+
} else {
|
|
2975
|
+
continue DECODE;
|
|
2976
|
+
}
|
|
2977
|
+
} else if (state.type === STATE_MAP_KEY) {
|
|
2978
|
+
if (object === "__proto__") {
|
|
2979
|
+
throw new DecodeError("The key __proto__ is not allowed");
|
|
2980
|
+
}
|
|
2981
|
+
state.key = this.mapKeyConverter(object);
|
|
2982
|
+
state.type = STATE_MAP_VALUE;
|
|
2983
|
+
continue DECODE;
|
|
2984
|
+
} else {
|
|
2985
|
+
state.map[state.key] = object;
|
|
2986
|
+
state.readCount++;
|
|
2987
|
+
if (state.readCount === state.size) {
|
|
2988
|
+
object = state.map;
|
|
2989
|
+
stack.release(state);
|
|
2990
|
+
} else {
|
|
2991
|
+
state.key = null;
|
|
2992
|
+
state.type = STATE_MAP_KEY;
|
|
2993
|
+
continue DECODE;
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
}
|
|
2997
|
+
return object;
|
|
2998
|
+
}
|
|
2999
|
+
}
|
|
3000
|
+
readHeadByte() {
|
|
3001
|
+
if (this.headByte === HEAD_BYTE_REQUIRED) {
|
|
3002
|
+
this.headByte = this.readU8();
|
|
3003
|
+
}
|
|
3004
|
+
return this.headByte;
|
|
3005
|
+
}
|
|
3006
|
+
complete() {
|
|
3007
|
+
this.headByte = HEAD_BYTE_REQUIRED;
|
|
3008
|
+
}
|
|
3009
|
+
readArraySize() {
|
|
3010
|
+
const headByte = this.readHeadByte();
|
|
3011
|
+
switch (headByte) {
|
|
3012
|
+
case 220:
|
|
3013
|
+
return this.readU16();
|
|
3014
|
+
case 221:
|
|
3015
|
+
return this.readU32();
|
|
3016
|
+
default: {
|
|
3017
|
+
if (headByte < 160) {
|
|
3018
|
+
return headByte - 144;
|
|
3019
|
+
} else {
|
|
3020
|
+
throw new DecodeError(`Unrecognized array type byte: ${prettyByte(headByte)}`);
|
|
3021
|
+
}
|
|
3022
|
+
}
|
|
3023
|
+
}
|
|
3024
|
+
}
|
|
3025
|
+
pushMapState(size) {
|
|
3026
|
+
if (size > this.maxMapLength) {
|
|
3027
|
+
throw new DecodeError(`Max length exceeded: map length (${size}) > maxMapLengthLength (${this.maxMapLength})`);
|
|
3028
|
+
}
|
|
3029
|
+
this.stack.pushMapState(size);
|
|
3030
|
+
}
|
|
3031
|
+
pushArrayState(size) {
|
|
3032
|
+
if (size > this.maxArrayLength) {
|
|
3033
|
+
throw new DecodeError(`Max length exceeded: array length (${size}) > maxArrayLength (${this.maxArrayLength})`);
|
|
3034
|
+
}
|
|
3035
|
+
this.stack.pushArrayState(size);
|
|
3036
|
+
}
|
|
3037
|
+
decodeString(byteLength, headerOffset) {
|
|
3038
|
+
if (!this.rawStrings || this.stateIsMapKey()) {
|
|
3039
|
+
return this.decodeUtf8String(byteLength, headerOffset);
|
|
3040
|
+
}
|
|
3041
|
+
return this.decodeBinary(byteLength, headerOffset);
|
|
3042
|
+
}
|
|
3043
|
+
/**
|
|
3044
|
+
* @throws {@link RangeError}
|
|
3045
|
+
*/
|
|
3046
|
+
decodeUtf8String(byteLength, headerOffset) {
|
|
3047
|
+
if (byteLength > this.maxStrLength) {
|
|
3048
|
+
throw new DecodeError(`Max length exceeded: UTF-8 byte length (${byteLength}) > maxStrLength (${this.maxStrLength})`);
|
|
3049
|
+
}
|
|
3050
|
+
if (this.bytes.byteLength < this.pos + headerOffset + byteLength) {
|
|
3051
|
+
throw MORE_DATA;
|
|
3052
|
+
}
|
|
3053
|
+
const offset = this.pos + headerOffset;
|
|
3054
|
+
let object;
|
|
3055
|
+
if (this.stateIsMapKey() && this.keyDecoder?.canBeCached(byteLength)) {
|
|
3056
|
+
object = this.keyDecoder.decode(this.bytes, offset, byteLength);
|
|
3057
|
+
} else {
|
|
3058
|
+
object = utf8Decode(this.bytes, offset, byteLength);
|
|
3059
|
+
}
|
|
3060
|
+
this.pos += headerOffset + byteLength;
|
|
3061
|
+
return object;
|
|
3062
|
+
}
|
|
3063
|
+
stateIsMapKey() {
|
|
3064
|
+
if (this.stack.length > 0) {
|
|
3065
|
+
const state = this.stack.top();
|
|
3066
|
+
return state.type === STATE_MAP_KEY;
|
|
3067
|
+
}
|
|
3068
|
+
return false;
|
|
3069
|
+
}
|
|
3070
|
+
/**
|
|
3071
|
+
* @throws {@link RangeError}
|
|
3072
|
+
*/
|
|
3073
|
+
decodeBinary(byteLength, headOffset) {
|
|
3074
|
+
if (byteLength > this.maxBinLength) {
|
|
3075
|
+
throw new DecodeError(`Max length exceeded: bin length (${byteLength}) > maxBinLength (${this.maxBinLength})`);
|
|
3076
|
+
}
|
|
3077
|
+
if (!this.hasRemaining(byteLength + headOffset)) {
|
|
3078
|
+
throw MORE_DATA;
|
|
3079
|
+
}
|
|
3080
|
+
const offset = this.pos + headOffset;
|
|
3081
|
+
const object = this.bytes.subarray(offset, offset + byteLength);
|
|
3082
|
+
this.pos += headOffset + byteLength;
|
|
3083
|
+
return object;
|
|
3084
|
+
}
|
|
3085
|
+
decodeExtension(size, headOffset) {
|
|
3086
|
+
if (size > this.maxExtLength) {
|
|
3087
|
+
throw new DecodeError(`Max length exceeded: ext length (${size}) > maxExtLength (${this.maxExtLength})`);
|
|
3088
|
+
}
|
|
3089
|
+
const extType = this.view.getInt8(this.pos + headOffset);
|
|
3090
|
+
const data = this.decodeBinary(
|
|
3091
|
+
size,
|
|
3092
|
+
headOffset + 1
|
|
3093
|
+
/* extType */
|
|
3094
|
+
);
|
|
3095
|
+
return this.extensionCodec.decode(data, extType, this.context);
|
|
3096
|
+
}
|
|
3097
|
+
lookU8() {
|
|
3098
|
+
return this.view.getUint8(this.pos);
|
|
3099
|
+
}
|
|
3100
|
+
lookU16() {
|
|
3101
|
+
return this.view.getUint16(this.pos);
|
|
3102
|
+
}
|
|
3103
|
+
lookU32() {
|
|
3104
|
+
return this.view.getUint32(this.pos);
|
|
3105
|
+
}
|
|
3106
|
+
readU8() {
|
|
3107
|
+
const value = this.view.getUint8(this.pos);
|
|
3108
|
+
this.pos++;
|
|
3109
|
+
return value;
|
|
3110
|
+
}
|
|
3111
|
+
readI8() {
|
|
3112
|
+
const value = this.view.getInt8(this.pos);
|
|
3113
|
+
this.pos++;
|
|
3114
|
+
return value;
|
|
3115
|
+
}
|
|
3116
|
+
readU16() {
|
|
3117
|
+
const value = this.view.getUint16(this.pos);
|
|
3118
|
+
this.pos += 2;
|
|
3119
|
+
return value;
|
|
3120
|
+
}
|
|
3121
|
+
readI16() {
|
|
3122
|
+
const value = this.view.getInt16(this.pos);
|
|
3123
|
+
this.pos += 2;
|
|
3124
|
+
return value;
|
|
3125
|
+
}
|
|
3126
|
+
readU32() {
|
|
3127
|
+
const value = this.view.getUint32(this.pos);
|
|
3128
|
+
this.pos += 4;
|
|
3129
|
+
return value;
|
|
3130
|
+
}
|
|
3131
|
+
readI32() {
|
|
3132
|
+
const value = this.view.getInt32(this.pos);
|
|
3133
|
+
this.pos += 4;
|
|
3134
|
+
return value;
|
|
3135
|
+
}
|
|
3136
|
+
readU64() {
|
|
3137
|
+
const value = getUint64(this.view, this.pos);
|
|
3138
|
+
this.pos += 8;
|
|
3139
|
+
return value;
|
|
3140
|
+
}
|
|
3141
|
+
readI64() {
|
|
3142
|
+
const value = getInt64(this.view, this.pos);
|
|
3143
|
+
this.pos += 8;
|
|
3144
|
+
return value;
|
|
3145
|
+
}
|
|
3146
|
+
readU64AsBigInt() {
|
|
3147
|
+
const value = this.view.getBigUint64(this.pos);
|
|
3148
|
+
this.pos += 8;
|
|
3149
|
+
return value;
|
|
3150
|
+
}
|
|
3151
|
+
readI64AsBigInt() {
|
|
3152
|
+
const value = this.view.getBigInt64(this.pos);
|
|
3153
|
+
this.pos += 8;
|
|
3154
|
+
return value;
|
|
3155
|
+
}
|
|
3156
|
+
readF32() {
|
|
3157
|
+
const value = this.view.getFloat32(this.pos);
|
|
3158
|
+
this.pos += 4;
|
|
3159
|
+
return value;
|
|
3160
|
+
}
|
|
3161
|
+
readF64() {
|
|
3162
|
+
const value = this.view.getFloat64(this.pos);
|
|
3163
|
+
this.pos += 8;
|
|
3164
|
+
return value;
|
|
3165
|
+
}
|
|
3166
|
+
};
|
|
3167
|
+
}
|
|
3168
|
+
});
|
|
3169
|
+
|
|
3170
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/decode.mjs
|
|
3171
|
+
function decode(buffer, options) {
|
|
3172
|
+
const decoder = new Decoder(options);
|
|
3173
|
+
return decoder.decode(buffer);
|
|
3174
|
+
}
|
|
3175
|
+
function decodeMulti(buffer, options) {
|
|
3176
|
+
const decoder = new Decoder(options);
|
|
3177
|
+
return decoder.decodeMulti(buffer);
|
|
3178
|
+
}
|
|
3179
|
+
var init_decode = __esm({
|
|
3180
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/decode.mjs"() {
|
|
3181
|
+
"use strict";
|
|
3182
|
+
init_Decoder();
|
|
3183
|
+
}
|
|
3184
|
+
});
|
|
3185
|
+
|
|
3186
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/stream.mjs
|
|
3187
|
+
function isAsyncIterable(object) {
|
|
3188
|
+
return object[Symbol.asyncIterator] != null;
|
|
3189
|
+
}
|
|
3190
|
+
async function* asyncIterableFromStream(stream) {
|
|
3191
|
+
const reader = stream.getReader();
|
|
3192
|
+
try {
|
|
3193
|
+
while (true) {
|
|
3194
|
+
const { done, value } = await reader.read();
|
|
3195
|
+
if (done) {
|
|
3196
|
+
return;
|
|
3197
|
+
}
|
|
3198
|
+
yield value;
|
|
3199
|
+
}
|
|
3200
|
+
} finally {
|
|
3201
|
+
reader.releaseLock();
|
|
3202
|
+
}
|
|
3203
|
+
}
|
|
3204
|
+
function ensureAsyncIterable(streamLike) {
|
|
3205
|
+
if (isAsyncIterable(streamLike)) {
|
|
3206
|
+
return streamLike;
|
|
3207
|
+
} else {
|
|
3208
|
+
return asyncIterableFromStream(streamLike);
|
|
3209
|
+
}
|
|
3210
|
+
}
|
|
3211
|
+
var init_stream = __esm({
|
|
3212
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/utils/stream.mjs"() {
|
|
3213
|
+
"use strict";
|
|
3214
|
+
}
|
|
3215
|
+
});
|
|
3216
|
+
|
|
3217
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/decodeAsync.mjs
|
|
3218
|
+
async function decodeAsync(streamLike, options) {
|
|
3219
|
+
const stream = ensureAsyncIterable(streamLike);
|
|
3220
|
+
const decoder = new Decoder(options);
|
|
3221
|
+
return decoder.decodeAsync(stream);
|
|
3222
|
+
}
|
|
3223
|
+
function decodeArrayStream(streamLike, options) {
|
|
3224
|
+
const stream = ensureAsyncIterable(streamLike);
|
|
3225
|
+
const decoder = new Decoder(options);
|
|
3226
|
+
return decoder.decodeArrayStream(stream);
|
|
3227
|
+
}
|
|
3228
|
+
function decodeMultiStream(streamLike, options) {
|
|
3229
|
+
const stream = ensureAsyncIterable(streamLike);
|
|
3230
|
+
const decoder = new Decoder(options);
|
|
3231
|
+
return decoder.decodeStream(stream);
|
|
3232
|
+
}
|
|
3233
|
+
var init_decodeAsync = __esm({
|
|
3234
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/decodeAsync.mjs"() {
|
|
3235
|
+
"use strict";
|
|
3236
|
+
init_Decoder();
|
|
3237
|
+
init_stream();
|
|
3238
|
+
}
|
|
3239
|
+
});
|
|
3240
|
+
|
|
3241
|
+
// ../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/index.mjs
|
|
3242
|
+
var dist_exports = {};
|
|
3243
|
+
__export(dist_exports, {
|
|
3244
|
+
DecodeError: () => DecodeError,
|
|
3245
|
+
Decoder: () => Decoder,
|
|
3246
|
+
EXT_TIMESTAMP: () => EXT_TIMESTAMP,
|
|
3247
|
+
Encoder: () => Encoder,
|
|
3248
|
+
ExtData: () => ExtData,
|
|
3249
|
+
ExtensionCodec: () => ExtensionCodec,
|
|
3250
|
+
decode: () => decode,
|
|
3251
|
+
decodeArrayStream: () => decodeArrayStream,
|
|
3252
|
+
decodeAsync: () => decodeAsync,
|
|
3253
|
+
decodeMulti: () => decodeMulti,
|
|
3254
|
+
decodeMultiStream: () => decodeMultiStream,
|
|
3255
|
+
decodeTimestampExtension: () => decodeTimestampExtension,
|
|
3256
|
+
decodeTimestampToTimeSpec: () => decodeTimestampToTimeSpec,
|
|
3257
|
+
encode: () => encode,
|
|
3258
|
+
encodeDateToTimeSpec: () => encodeDateToTimeSpec,
|
|
3259
|
+
encodeTimeSpecToTimestamp: () => encodeTimeSpecToTimestamp,
|
|
3260
|
+
encodeTimestampExtension: () => encodeTimestampExtension
|
|
3261
|
+
});
|
|
3262
|
+
var init_dist = __esm({
|
|
3263
|
+
"../../node_modules/.bun/@msgpack+msgpack@3.1.3/node_modules/@msgpack/msgpack/dist.esm/index.mjs"() {
|
|
3264
|
+
"use strict";
|
|
3265
|
+
init_encode();
|
|
3266
|
+
init_decode();
|
|
3267
|
+
init_decodeAsync();
|
|
3268
|
+
init_Decoder();
|
|
3269
|
+
init_DecodeError();
|
|
3270
|
+
init_Encoder();
|
|
3271
|
+
init_ExtensionCodec();
|
|
3272
|
+
init_ExtData();
|
|
3273
|
+
init_timestamp();
|
|
3274
|
+
}
|
|
3275
|
+
});
|
|
3276
|
+
|
|
3277
|
+
// src/serializers/MessagePackSerializer.ts
|
|
3278
|
+
var MessagePackSerializer_exports = {};
|
|
3279
|
+
__export(MessagePackSerializer_exports, {
|
|
3280
|
+
MessagePackSerializer: () => MessagePackSerializer
|
|
3281
|
+
});
|
|
3282
|
+
var MessagePackSerializer;
|
|
3283
|
+
var init_MessagePackSerializer = __esm({
|
|
3284
|
+
"src/serializers/MessagePackSerializer.ts"() {
|
|
3285
|
+
"use strict";
|
|
3286
|
+
MessagePackSerializer = class {
|
|
3287
|
+
msgpack;
|
|
3288
|
+
constructor() {
|
|
3289
|
+
try {
|
|
3290
|
+
this.msgpack = (init_dist(), __toCommonJS(dist_exports));
|
|
3291
|
+
} catch (_e) {
|
|
3292
|
+
throw new Error(
|
|
3293
|
+
"MessagePackSerializer requires @msgpack/msgpack. Please install it: bun add @msgpack/msgpack"
|
|
3294
|
+
);
|
|
3295
|
+
}
|
|
3296
|
+
}
|
|
3297
|
+
serialize(job) {
|
|
3298
|
+
const id = job.id || `${Date.now()}-${crypto.randomUUID()}`;
|
|
3299
|
+
const properties = {};
|
|
3300
|
+
for (const key in job) {
|
|
3301
|
+
if (Object.hasOwn(job, key) && typeof job[key] !== "function") {
|
|
3302
|
+
properties[key] = job[key];
|
|
3303
|
+
}
|
|
3304
|
+
}
|
|
3305
|
+
const encoded = this.msgpack.encode(properties);
|
|
3306
|
+
const data = Buffer.from(encoded).toString("base64");
|
|
3307
|
+
return {
|
|
3308
|
+
id,
|
|
3309
|
+
type: "msgpack",
|
|
3310
|
+
data,
|
|
3311
|
+
createdAt: Date.now(),
|
|
3312
|
+
...job.delaySeconds !== void 0 ? { delaySeconds: job.delaySeconds } : {},
|
|
3313
|
+
attempts: job.attempts ?? 0,
|
|
3314
|
+
...job.maxAttempts !== void 0 ? { maxAttempts: job.maxAttempts } : {},
|
|
3315
|
+
...job.groupId ? { groupId: job.groupId } : {},
|
|
3316
|
+
...job.priority ? { priority: job.priority } : {}
|
|
3317
|
+
};
|
|
3318
|
+
}
|
|
3319
|
+
deserialize(serialized) {
|
|
3320
|
+
if (serialized.type !== "msgpack") {
|
|
3321
|
+
throw new Error('Invalid serialization type: expected "msgpack"');
|
|
3322
|
+
}
|
|
3323
|
+
const buffer = Buffer.from(serialized.data, "base64");
|
|
3324
|
+
const properties = this.msgpack.decode(buffer);
|
|
3325
|
+
const job = /* @__PURE__ */ Object.create({});
|
|
3326
|
+
Object.assign(job, properties);
|
|
3327
|
+
job.id = serialized.id;
|
|
3328
|
+
if (serialized.groupId) {
|
|
3329
|
+
job.groupId = serialized.groupId;
|
|
3330
|
+
}
|
|
3331
|
+
if (serialized.priority) {
|
|
3332
|
+
job.priority = serialized.priority;
|
|
3333
|
+
}
|
|
3334
|
+
if (serialized.delaySeconds !== void 0) {
|
|
3335
|
+
job.delaySeconds = serialized.delaySeconds;
|
|
3336
|
+
}
|
|
3337
|
+
if (serialized.attempts !== void 0) {
|
|
3338
|
+
job.attempts = serialized.attempts;
|
|
3339
|
+
}
|
|
3340
|
+
return job;
|
|
3341
|
+
}
|
|
3342
|
+
};
|
|
3343
|
+
}
|
|
3344
|
+
});
|
|
3345
|
+
|
|
1070
3346
|
// src/Scheduler.ts
|
|
1071
3347
|
var Scheduler_exports = {};
|
|
1072
3348
|
__export(Scheduler_exports, {
|
|
@@ -1085,6 +3361,9 @@ var init_Scheduler = __esm({
|
|
|
1085
3361
|
prefix;
|
|
1086
3362
|
get client() {
|
|
1087
3363
|
const driver = this.manager.getDriver(this.manager.getDefaultConnection());
|
|
3364
|
+
if (!driver || !("client" in driver)) {
|
|
3365
|
+
throw new Error("[Scheduler] Driver does not support Redis client access");
|
|
3366
|
+
}
|
|
1088
3367
|
return driver.client;
|
|
1089
3368
|
}
|
|
1090
3369
|
/**
|
|
@@ -1097,7 +3376,11 @@ var init_Scheduler = __esm({
|
|
|
1097
3376
|
nextRun,
|
|
1098
3377
|
enabled: true
|
|
1099
3378
|
};
|
|
1100
|
-
const
|
|
3379
|
+
const client = this.client;
|
|
3380
|
+
if (typeof client.pipeline !== "function") {
|
|
3381
|
+
throw new Error("[Scheduler] Redis client does not support pipeline");
|
|
3382
|
+
}
|
|
3383
|
+
const pipe = client.pipeline();
|
|
1101
3384
|
pipe.hset(`${this.prefix}schedule:${config.id}`, {
|
|
1102
3385
|
...fullConfig,
|
|
1103
3386
|
job: JSON.stringify(fullConfig.job)
|
|
@@ -1109,7 +3392,11 @@ var init_Scheduler = __esm({
|
|
|
1109
3392
|
* Remove a scheduled job.
|
|
1110
3393
|
*/
|
|
1111
3394
|
async remove(id) {
|
|
1112
|
-
const
|
|
3395
|
+
const client = this.client;
|
|
3396
|
+
if (typeof client.pipeline !== "function") {
|
|
3397
|
+
throw new Error("[Scheduler] Redis client does not support pipeline");
|
|
3398
|
+
}
|
|
3399
|
+
const pipe = client.pipeline();
|
|
1113
3400
|
pipe.del(`${this.prefix}schedule:${id}`);
|
|
1114
3401
|
pipe.zrem(`${this.prefix}schedules`, id);
|
|
1115
3402
|
await pipe.exec();
|
|
@@ -1118,10 +3405,14 @@ var init_Scheduler = __esm({
|
|
|
1118
3405
|
* List all scheduled jobs.
|
|
1119
3406
|
*/
|
|
1120
3407
|
async list() {
|
|
1121
|
-
const
|
|
3408
|
+
const client = this.client;
|
|
3409
|
+
if (typeof client.zrange !== "function") {
|
|
3410
|
+
throw new Error("[Scheduler] Redis client does not support zrange");
|
|
3411
|
+
}
|
|
3412
|
+
const ids = await client.zrange(`${this.prefix}schedules`, 0, -1);
|
|
1122
3413
|
const configs = [];
|
|
1123
3414
|
for (const id of ids) {
|
|
1124
|
-
const data = await
|
|
3415
|
+
const data = await client.hgetall?.(`${this.prefix}schedule:${id}`);
|
|
1125
3416
|
if (data?.id) {
|
|
1126
3417
|
configs.push({
|
|
1127
3418
|
...data,
|
|
@@ -1138,7 +3429,8 @@ var init_Scheduler = __esm({
|
|
|
1138
3429
|
* Run a scheduled job immediately (out of schedule).
|
|
1139
3430
|
*/
|
|
1140
3431
|
async runNow(id) {
|
|
1141
|
-
const
|
|
3432
|
+
const client = this.client;
|
|
3433
|
+
const data = await client.hgetall?.(`${this.prefix}schedule:${id}`);
|
|
1142
3434
|
if (data?.id) {
|
|
1143
3435
|
const serialized = JSON.parse(data.job);
|
|
1144
3436
|
const serializer = this.manager.getSerializer();
|
|
@@ -1151,14 +3443,21 @@ var init_Scheduler = __esm({
|
|
|
1151
3443
|
* This should be called periodically (e.g. every minute).
|
|
1152
3444
|
*/
|
|
1153
3445
|
async tick() {
|
|
3446
|
+
const client = this.client;
|
|
3447
|
+
if (typeof client.zrangebyscore !== "function") {
|
|
3448
|
+
throw new Error("[Scheduler] Redis client does not support zrangebyscore");
|
|
3449
|
+
}
|
|
1154
3450
|
const now = Date.now();
|
|
1155
|
-
const dueIds = await
|
|
3451
|
+
const dueIds = await client.zrangebyscore(`${this.prefix}schedules`, 0, now);
|
|
1156
3452
|
let fired = 0;
|
|
1157
3453
|
for (const id of dueIds) {
|
|
1158
3454
|
const lockKey = `${this.prefix}lock:schedule:${id}:${Math.floor(now / 1e3)}`;
|
|
1159
|
-
|
|
3455
|
+
if (typeof client.set !== "function") {
|
|
3456
|
+
continue;
|
|
3457
|
+
}
|
|
3458
|
+
const lock = await client.set(lockKey, "1", "EX", 10, "NX");
|
|
1160
3459
|
if (lock === "OK") {
|
|
1161
|
-
const data = await
|
|
3460
|
+
const data = await client.hgetall?.(`${this.prefix}schedule:${id}`);
|
|
1162
3461
|
if (data?.id && data.enabled === "true") {
|
|
1163
3462
|
try {
|
|
1164
3463
|
const serializedJob = JSON.parse(data.job);
|
|
@@ -1166,16 +3465,19 @@ var init_Scheduler = __esm({
|
|
|
1166
3465
|
const driver = this.manager.getDriver(connection);
|
|
1167
3466
|
await driver.push(data.queue, serializedJob);
|
|
1168
3467
|
const nextRun = import_cron_parser.default.parse(data.cron).next().getTime();
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
3468
|
+
if (typeof client.pipeline === "function") {
|
|
3469
|
+
const pipe = client.pipeline();
|
|
3470
|
+
pipe.hset(`${this.prefix}schedule:${id}`, {
|
|
3471
|
+
lastRun: now,
|
|
3472
|
+
nextRun
|
|
3473
|
+
});
|
|
3474
|
+
pipe.zadd(`${this.prefix}schedules`, nextRun, id);
|
|
3475
|
+
await pipe.exec();
|
|
3476
|
+
}
|
|
1176
3477
|
fired++;
|
|
1177
3478
|
} catch (err) {
|
|
1178
|
-
|
|
3479
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
3480
|
+
console.error(`[Scheduler] Failed to process schedule ${id}:`, error.message);
|
|
1179
3481
|
}
|
|
1180
3482
|
}
|
|
1181
3483
|
}
|
|
@@ -1189,6 +3491,7 @@ var init_Scheduler = __esm({
|
|
|
1189
3491
|
// src/index.ts
|
|
1190
3492
|
var index_exports = {};
|
|
1191
3493
|
__export(index_exports, {
|
|
3494
|
+
BufferedPersistence: () => BufferedPersistence,
|
|
1192
3495
|
ClassNameSerializer: () => ClassNameSerializer,
|
|
1193
3496
|
Consumer: () => Consumer,
|
|
1194
3497
|
DatabaseDriver: () => DatabaseDriver,
|
|
@@ -1208,6 +3511,10 @@ __export(index_exports, {
|
|
|
1208
3511
|
});
|
|
1209
3512
|
module.exports = __toCommonJS(index_exports);
|
|
1210
3513
|
|
|
3514
|
+
// src/Consumer.ts
|
|
3515
|
+
var import_node_events = require("events");
|
|
3516
|
+
var import_p_limit = __toESM(require("p-limit"), 1);
|
|
3517
|
+
|
|
1211
3518
|
// src/Worker.ts
|
|
1212
3519
|
var Worker = class {
|
|
1213
3520
|
constructor(options = {}) {
|
|
@@ -1265,18 +3572,40 @@ var Worker = class {
|
|
|
1265
3572
|
};
|
|
1266
3573
|
|
|
1267
3574
|
// src/Consumer.ts
|
|
1268
|
-
var Consumer = class {
|
|
3575
|
+
var Consumer = class extends import_node_events.EventEmitter {
|
|
1269
3576
|
constructor(queueManager, options) {
|
|
3577
|
+
super();
|
|
1270
3578
|
this.queueManager = queueManager;
|
|
1271
3579
|
this.options = options;
|
|
1272
3580
|
}
|
|
1273
3581
|
running = false;
|
|
1274
3582
|
stopRequested = false;
|
|
1275
|
-
workerId = `worker-${
|
|
3583
|
+
workerId = `worker-${crypto.randomUUID()}`;
|
|
1276
3584
|
heartbeatTimer = null;
|
|
3585
|
+
groupLimiters = /* @__PURE__ */ new Map();
|
|
3586
|
+
stats = {
|
|
3587
|
+
processed: 0,
|
|
3588
|
+
failed: 0,
|
|
3589
|
+
retried: 0,
|
|
3590
|
+
active: 0
|
|
3591
|
+
};
|
|
1277
3592
|
get connectionName() {
|
|
1278
3593
|
return this.options.connection ?? this.queueManager.getDefaultConnection();
|
|
1279
3594
|
}
|
|
3595
|
+
/**
|
|
3596
|
+
* Log debug message.
|
|
3597
|
+
*/
|
|
3598
|
+
log(message, data) {
|
|
3599
|
+
if (this.options.debug) {
|
|
3600
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
3601
|
+
const prefix = `[Consumer:${this.workerId}] [${timestamp}]`;
|
|
3602
|
+
if (data) {
|
|
3603
|
+
console.log(prefix, message, data);
|
|
3604
|
+
} else {
|
|
3605
|
+
console.log(prefix, message);
|
|
3606
|
+
}
|
|
3607
|
+
}
|
|
3608
|
+
}
|
|
1280
3609
|
/**
|
|
1281
3610
|
* Start the consumer loop.
|
|
1282
3611
|
*/
|
|
@@ -1287,19 +3616,36 @@ var Consumer = class {
|
|
|
1287
3616
|
this.running = true;
|
|
1288
3617
|
this.stopRequested = false;
|
|
1289
3618
|
const worker = new Worker(this.options.workerOptions);
|
|
1290
|
-
|
|
3619
|
+
let currentPollInterval = this.options.pollInterval ?? 1e3;
|
|
3620
|
+
const minPollInterval = this.options.minPollInterval ?? 100;
|
|
3621
|
+
const maxPollInterval = this.options.maxPollInterval ?? 5e3;
|
|
3622
|
+
const backoffMultiplier = this.options.backoffMultiplier ?? 1.5;
|
|
1291
3623
|
const keepAlive = this.options.keepAlive ?? true;
|
|
1292
|
-
|
|
3624
|
+
const concurrency = this.options.concurrency ?? 1;
|
|
3625
|
+
const batchSize = this.options.batchSize ?? 1;
|
|
3626
|
+
const useBlocking = this.options.useBlocking ?? true;
|
|
3627
|
+
const blockingTimeout = this.options.blockingTimeout ?? 5;
|
|
3628
|
+
this.log("Started", {
|
|
1293
3629
|
queues: this.options.queues,
|
|
1294
3630
|
connection: this.options.connection,
|
|
1295
|
-
workerId: this.workerId
|
|
3631
|
+
workerId: this.workerId,
|
|
3632
|
+
concurrency,
|
|
3633
|
+
batchSize
|
|
1296
3634
|
});
|
|
1297
3635
|
if (this.options.monitor) {
|
|
1298
3636
|
this.startHeartbeat();
|
|
1299
|
-
await this.publishLog(
|
|
3637
|
+
await this.publishLog(
|
|
3638
|
+
"info",
|
|
3639
|
+
`Consumer started on [${this.options.queues.join(", ")}] with concurrency ${concurrency}`
|
|
3640
|
+
);
|
|
1300
3641
|
}
|
|
1301
3642
|
while (this.running && !this.stopRequested) {
|
|
1302
|
-
|
|
3643
|
+
const capacity = concurrency - this.stats.active;
|
|
3644
|
+
if (capacity <= 0) {
|
|
3645
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
3646
|
+
continue;
|
|
3647
|
+
}
|
|
3648
|
+
const eligibleQueues = [];
|
|
1303
3649
|
for (const queue of this.options.queues) {
|
|
1304
3650
|
if (this.options.rateLimits?.[queue]) {
|
|
1305
3651
|
const limit = this.options.rateLimits[queue];
|
|
@@ -1315,60 +3661,74 @@ var Consumer = class {
|
|
|
1315
3661
|
console.error(`[Consumer] Error checking rate limit for "${queue}":`, err);
|
|
1316
3662
|
}
|
|
1317
3663
|
}
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
3664
|
+
eligibleQueues.push(queue);
|
|
3665
|
+
}
|
|
3666
|
+
if (eligibleQueues.length === 0) {
|
|
3667
|
+
await new Promise((resolve) => setTimeout(resolve, currentPollInterval));
|
|
3668
|
+
continue;
|
|
3669
|
+
}
|
|
3670
|
+
let jobs = [];
|
|
3671
|
+
let didBlock = false;
|
|
3672
|
+
try {
|
|
3673
|
+
const currentBatchSize = Math.min(batchSize, capacity);
|
|
3674
|
+
const driver = this.queueManager.getDriver(this.connectionName);
|
|
3675
|
+
if (currentBatchSize > 1) {
|
|
3676
|
+
for (const queue of eligibleQueues) {
|
|
3677
|
+
const fetched = await this.queueManager.popMany(
|
|
3678
|
+
queue,
|
|
3679
|
+
currentBatchSize,
|
|
3680
|
+
this.connectionName
|
|
3681
|
+
);
|
|
3682
|
+
if (fetched.length > 0) {
|
|
3683
|
+
jobs = fetched;
|
|
3684
|
+
break;
|
|
1324
3685
|
}
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
if (this.options.monitor) {
|
|
1344
|
-
await this.publishLog(
|
|
1345
|
-
"warning",
|
|
1346
|
-
`Job retrying in ${delaySec}s (Attempt ${job.attempts}/${maxAttempts})`,
|
|
1347
|
-
job.id
|
|
1348
|
-
);
|
|
1349
|
-
}
|
|
1350
|
-
} else {
|
|
1351
|
-
await this.queueManager.fail(job, err).catch((dlqErr) => {
|
|
1352
|
-
console.error(`[Consumer] Error moving job to DLQ:`, dlqErr);
|
|
1353
|
-
});
|
|
3686
|
+
}
|
|
3687
|
+
} else {
|
|
3688
|
+
if (useBlocking && driver.popBlocking) {
|
|
3689
|
+
didBlock = true;
|
|
3690
|
+
const job = await this.queueManager.popBlocking(
|
|
3691
|
+
eligibleQueues,
|
|
3692
|
+
blockingTimeout,
|
|
3693
|
+
this.connectionName
|
|
3694
|
+
);
|
|
3695
|
+
if (job) {
|
|
3696
|
+
jobs.push(job);
|
|
3697
|
+
}
|
|
3698
|
+
} else {
|
|
3699
|
+
for (const queue of eligibleQueues) {
|
|
3700
|
+
const job = await this.queueManager.pop(queue, this.connectionName);
|
|
3701
|
+
if (job) {
|
|
3702
|
+
jobs.push(job);
|
|
3703
|
+
break;
|
|
1354
3704
|
}
|
|
1355
|
-
} finally {
|
|
1356
|
-
await this.queueManager.complete(job).catch((err) => {
|
|
1357
|
-
console.error(`[Consumer] Error completing job in queue "${queue}":`, err);
|
|
1358
|
-
});
|
|
1359
3705
|
}
|
|
1360
3706
|
}
|
|
1361
|
-
} catch (error) {
|
|
1362
|
-
console.error(`[Consumer] Error polling queue "${queue}":`, error);
|
|
1363
3707
|
}
|
|
3708
|
+
if (jobs.length > 0) {
|
|
3709
|
+
this.stats.active += jobs.length;
|
|
3710
|
+
currentPollInterval = minPollInterval;
|
|
3711
|
+
for (const job of jobs) {
|
|
3712
|
+
this.runJob(job, worker).finally(() => {
|
|
3713
|
+
this.stats.active--;
|
|
3714
|
+
});
|
|
3715
|
+
}
|
|
3716
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
3717
|
+
continue;
|
|
3718
|
+
}
|
|
3719
|
+
} catch (error) {
|
|
3720
|
+
console.error("[Consumer] Loop error:", error);
|
|
1364
3721
|
}
|
|
1365
|
-
if (
|
|
3722
|
+
if (this.stats.active === 0 && !keepAlive) {
|
|
1366
3723
|
break;
|
|
1367
3724
|
}
|
|
1368
|
-
if (!this.stopRequested
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
3725
|
+
if (!this.stopRequested) {
|
|
3726
|
+
if (!didBlock) {
|
|
3727
|
+
await new Promise((resolve) => setTimeout(resolve, currentPollInterval));
|
|
3728
|
+
currentPollInterval = Math.min(currentPollInterval * backoffMultiplier, maxPollInterval);
|
|
3729
|
+
}
|
|
3730
|
+
} else {
|
|
3731
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
1372
3732
|
}
|
|
1373
3733
|
}
|
|
1374
3734
|
this.running = false;
|
|
@@ -1376,7 +3736,89 @@ var Consumer = class {
|
|
|
1376
3736
|
if (this.options.monitor) {
|
|
1377
3737
|
await this.publishLog("info", "Consumer stopped");
|
|
1378
3738
|
}
|
|
1379
|
-
|
|
3739
|
+
this.log("Stopped");
|
|
3740
|
+
}
|
|
3741
|
+
/**
|
|
3742
|
+
* Run a job with concurrency controls.
|
|
3743
|
+
*/
|
|
3744
|
+
async runJob(job, worker) {
|
|
3745
|
+
if (!job.groupId || this.options.groupJobsSequential === false) {
|
|
3746
|
+
return this.handleJob(job, worker);
|
|
3747
|
+
}
|
|
3748
|
+
let limiter = this.groupLimiters.get(job.groupId);
|
|
3749
|
+
if (!limiter) {
|
|
3750
|
+
limiter = (0, import_p_limit.default)(1);
|
|
3751
|
+
this.groupLimiters.set(job.groupId, limiter);
|
|
3752
|
+
}
|
|
3753
|
+
if (limiter.pendingCount > 0) {
|
|
3754
|
+
this.log(`Job ${job.id} queued behind group ${job.groupId}`);
|
|
3755
|
+
}
|
|
3756
|
+
await limiter(async () => {
|
|
3757
|
+
await this.handleJob(job, worker);
|
|
3758
|
+
});
|
|
3759
|
+
if (limiter.activeCount === 0 && limiter.pendingCount === 0) {
|
|
3760
|
+
this.groupLimiters.delete(job.groupId);
|
|
3761
|
+
}
|
|
3762
|
+
}
|
|
3763
|
+
/**
|
|
3764
|
+
* Handle a single job.
|
|
3765
|
+
*/
|
|
3766
|
+
async handleJob(job, worker) {
|
|
3767
|
+
const currentQueue = job.queueName || "default";
|
|
3768
|
+
const startTime = Date.now();
|
|
3769
|
+
this.log(`Processing job ${job.id} from ${currentQueue}`);
|
|
3770
|
+
this.emit("job:started", { job, queue: currentQueue });
|
|
3771
|
+
if (this.options.monitor) {
|
|
3772
|
+
await this.publishLog("info", `Processing job: ${job.id}`, job.id);
|
|
3773
|
+
}
|
|
3774
|
+
try {
|
|
3775
|
+
await worker.process(job);
|
|
3776
|
+
const duration = Date.now() - startTime;
|
|
3777
|
+
this.stats.processed++;
|
|
3778
|
+
this.emit("job:processed", { job, duration, queue: currentQueue });
|
|
3779
|
+
this.log(`Completed job ${job.id} in ${duration}ms`);
|
|
3780
|
+
if (this.options.monitor) {
|
|
3781
|
+
await this.publishLog("success", `Completed job: ${job.id}`, job.id);
|
|
3782
|
+
}
|
|
3783
|
+
} catch (err) {
|
|
3784
|
+
const error = err;
|
|
3785
|
+
const duration = Date.now() - startTime;
|
|
3786
|
+
this.emit("job:failed", { job, error, duration, queue: currentQueue });
|
|
3787
|
+
this.log(`Failed job ${job.id} in ${duration}ms`, { error: error.message });
|
|
3788
|
+
this.stats.failed++;
|
|
3789
|
+
if (this.options.monitor) {
|
|
3790
|
+
await this.publishLog("error", `Job failed: ${job.id} - ${error.message}`, job.id);
|
|
3791
|
+
}
|
|
3792
|
+
const attempts = job.attempts ?? 1;
|
|
3793
|
+
const maxAttempts = job.maxAttempts ?? this.options.workerOptions?.maxAttempts ?? 3;
|
|
3794
|
+
if (attempts < maxAttempts) {
|
|
3795
|
+
job.attempts = attempts + 1;
|
|
3796
|
+
const delayMs = job.getRetryDelay(job.attempts);
|
|
3797
|
+
const delaySec = Math.ceil(delayMs / 1e3);
|
|
3798
|
+
job.delay(delaySec);
|
|
3799
|
+
await this.queueManager.push(job);
|
|
3800
|
+
this.log(`Retrying job ${job.id} in ${delaySec}s (Attempt ${job.attempts}/${maxAttempts})`);
|
|
3801
|
+
this.stats.retried++;
|
|
3802
|
+
this.emit("job:retried", { job, attempt: job.attempts, delay: delaySec });
|
|
3803
|
+
if (this.options.monitor) {
|
|
3804
|
+
await this.publishLog(
|
|
3805
|
+
"warning",
|
|
3806
|
+
`Job retrying in ${delaySec}s (Attempt ${job.attempts}/${maxAttempts})`,
|
|
3807
|
+
job.id
|
|
3808
|
+
);
|
|
3809
|
+
}
|
|
3810
|
+
} else {
|
|
3811
|
+
this.emit("job:failed_permanently", { job, error });
|
|
3812
|
+
this.log(`Job ${job.id} failed permanently`);
|
|
3813
|
+
await this.queueManager.fail(job, error).catch((dlqErr) => {
|
|
3814
|
+
console.error("[Consumer] Error moving job to DLQ:", dlqErr);
|
|
3815
|
+
});
|
|
3816
|
+
}
|
|
3817
|
+
} finally {
|
|
3818
|
+
await this.queueManager.complete(job).catch((err) => {
|
|
3819
|
+
console.error(`[Consumer] Error completing job in queue "${currentQueue}":`, err);
|
|
3820
|
+
});
|
|
3821
|
+
}
|
|
1380
3822
|
}
|
|
1381
3823
|
startHeartbeat() {
|
|
1382
3824
|
const interval = typeof this.options.monitor === "object" ? this.options.monitor.interval ?? 5e3 : 5e3;
|
|
@@ -1396,7 +3838,8 @@ var Consumer = class {
|
|
|
1396
3838
|
rss: Math.floor(mem.rss / 1024 / 1024),
|
|
1397
3839
|
heapUsed: Math.floor(mem.heapUsed / 1024 / 1024),
|
|
1398
3840
|
total: Math.floor(os.totalmem() / 1024 / 1024)
|
|
1399
|
-
}
|
|
3841
|
+
},
|
|
3842
|
+
stats: this.stats
|
|
1400
3843
|
};
|
|
1401
3844
|
await driver.reportHeartbeat(
|
|
1402
3845
|
{
|
|
@@ -1446,7 +3889,7 @@ var Consumer = class {
|
|
|
1446
3889
|
* Stop the consumer loop (graceful shutdown).
|
|
1447
3890
|
*/
|
|
1448
3891
|
async stop() {
|
|
1449
|
-
|
|
3892
|
+
this.log("Stopping...");
|
|
1450
3893
|
this.stopRequested = true;
|
|
1451
3894
|
while (this.running) {
|
|
1452
3895
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
@@ -1458,6 +3901,20 @@ var Consumer = class {
|
|
|
1458
3901
|
isRunning() {
|
|
1459
3902
|
return this.running;
|
|
1460
3903
|
}
|
|
3904
|
+
/**
|
|
3905
|
+
* Get current consumer statistics.
|
|
3906
|
+
*/
|
|
3907
|
+
getStats() {
|
|
3908
|
+
return { ...this.stats };
|
|
3909
|
+
}
|
|
3910
|
+
/**
|
|
3911
|
+
* Reset statistics counters.
|
|
3912
|
+
*/
|
|
3913
|
+
resetStats() {
|
|
3914
|
+
this.stats.processed = 0;
|
|
3915
|
+
this.stats.failed = 0;
|
|
3916
|
+
this.stats.retried = 0;
|
|
3917
|
+
}
|
|
1461
3918
|
};
|
|
1462
3919
|
|
|
1463
3920
|
// src/index.ts
|
|
@@ -1467,6 +3924,10 @@ init_KafkaDriver();
|
|
|
1467
3924
|
// src/drivers/MemoryDriver.ts
|
|
1468
3925
|
var MemoryDriver = class {
|
|
1469
3926
|
queues = /* @__PURE__ */ new Map();
|
|
3927
|
+
maxSize;
|
|
3928
|
+
constructor(config = {}) {
|
|
3929
|
+
this.maxSize = config.maxSize ?? Infinity;
|
|
3930
|
+
}
|
|
1470
3931
|
/**
|
|
1471
3932
|
* Push a job to a queue.
|
|
1472
3933
|
*/
|
|
@@ -1474,7 +3935,11 @@ var MemoryDriver = class {
|
|
|
1474
3935
|
if (!this.queues.has(queue)) {
|
|
1475
3936
|
this.queues.set(queue, []);
|
|
1476
3937
|
}
|
|
1477
|
-
this.queues.get(queue)
|
|
3938
|
+
const q = this.queues.get(queue);
|
|
3939
|
+
if (q.length >= this.maxSize) {
|
|
3940
|
+
throw new Error(`[MemoryDriver] Queue '${queue}' is full (max size: ${this.maxSize})`);
|
|
3941
|
+
}
|
|
3942
|
+
q.push(job);
|
|
1478
3943
|
}
|
|
1479
3944
|
/**
|
|
1480
3945
|
* Pop a job from a queue (FIFO).
|
|
@@ -1505,6 +3970,39 @@ var MemoryDriver = class {
|
|
|
1505
3970
|
async clear(queue) {
|
|
1506
3971
|
this.queues.delete(queue);
|
|
1507
3972
|
}
|
|
3973
|
+
/**
|
|
3974
|
+
* Mark a job as permanently failed.
|
|
3975
|
+
*/
|
|
3976
|
+
async fail(queue, job) {
|
|
3977
|
+
const failedQueue = `failed:${queue}`;
|
|
3978
|
+
if (!this.queues.has(failedQueue)) {
|
|
3979
|
+
this.queues.set(failedQueue, []);
|
|
3980
|
+
}
|
|
3981
|
+
this.queues.get(failedQueue)?.push(job);
|
|
3982
|
+
}
|
|
3983
|
+
/**
|
|
3984
|
+
* Get queue statistics.
|
|
3985
|
+
*/
|
|
3986
|
+
async stats(queue) {
|
|
3987
|
+
const jobs = this.queues.get(queue) || [];
|
|
3988
|
+
const now = Date.now();
|
|
3989
|
+
let pending = 0;
|
|
3990
|
+
let delayed = 0;
|
|
3991
|
+
for (const job of jobs) {
|
|
3992
|
+
const isDelayed = job.delaySeconds && now < job.createdAt + job.delaySeconds * 1e3;
|
|
3993
|
+
if (isDelayed) {
|
|
3994
|
+
delayed++;
|
|
3995
|
+
} else {
|
|
3996
|
+
pending++;
|
|
3997
|
+
}
|
|
3998
|
+
}
|
|
3999
|
+
return {
|
|
4000
|
+
queue,
|
|
4001
|
+
size: pending,
|
|
4002
|
+
delayed,
|
|
4003
|
+
failed: this.queues.get(`failed:${queue}`)?.length || 0
|
|
4004
|
+
};
|
|
4005
|
+
}
|
|
1508
4006
|
/**
|
|
1509
4007
|
* Push multiple jobs.
|
|
1510
4008
|
*/
|
|
@@ -1639,6 +4137,35 @@ var Job = class {
|
|
|
1639
4137
|
}
|
|
1640
4138
|
};
|
|
1641
4139
|
|
|
4140
|
+
// src/serializers/CachedSerializer.ts
|
|
4141
|
+
var CachedSerializer = class {
|
|
4142
|
+
/**
|
|
4143
|
+
* @param delegate - The actual serializer to use for the first serialization
|
|
4144
|
+
*/
|
|
4145
|
+
constructor(delegate) {
|
|
4146
|
+
this.delegate = delegate;
|
|
4147
|
+
}
|
|
4148
|
+
cache = /* @__PURE__ */ new WeakMap();
|
|
4149
|
+
/**
|
|
4150
|
+
* Serialize a job with caching.
|
|
4151
|
+
*/
|
|
4152
|
+
serialize(job) {
|
|
4153
|
+
if (this.cache.has(job)) {
|
|
4154
|
+
return this.cache.get(job);
|
|
4155
|
+
}
|
|
4156
|
+
const serialized = this.delegate.serialize(job);
|
|
4157
|
+
this.cache.set(job, serialized);
|
|
4158
|
+
return serialized;
|
|
4159
|
+
}
|
|
4160
|
+
/**
|
|
4161
|
+
* Deserialize a job.
|
|
4162
|
+
* No caching for deserialization as we get new objects each time from the driver.
|
|
4163
|
+
*/
|
|
4164
|
+
deserialize(serialized) {
|
|
4165
|
+
return this.delegate.deserialize(serialized);
|
|
4166
|
+
}
|
|
4167
|
+
};
|
|
4168
|
+
|
|
1642
4169
|
// src/serializers/ClassNameSerializer.ts
|
|
1643
4170
|
var ClassNameSerializer = class {
|
|
1644
4171
|
/**
|
|
@@ -1665,7 +4192,7 @@ var ClassNameSerializer = class {
|
|
|
1665
4192
|
* Serialize a Job.
|
|
1666
4193
|
*/
|
|
1667
4194
|
serialize(job) {
|
|
1668
|
-
const id = job.id || `${Date.now()}-${
|
|
4195
|
+
const id = job.id || `${Date.now()}-${crypto.randomUUID()}`;
|
|
1669
4196
|
const className = job.constructor.name;
|
|
1670
4197
|
const properties = {};
|
|
1671
4198
|
for (const key in job) {
|
|
@@ -1677,10 +4204,7 @@ var ClassNameSerializer = class {
|
|
|
1677
4204
|
id,
|
|
1678
4205
|
type: "class",
|
|
1679
4206
|
className,
|
|
1680
|
-
data: JSON.stringify(
|
|
1681
|
-
class: className,
|
|
1682
|
-
properties
|
|
1683
|
-
}),
|
|
4207
|
+
data: JSON.stringify(properties),
|
|
1684
4208
|
createdAt: Date.now(),
|
|
1685
4209
|
...job.delaySeconds !== void 0 ? { delaySeconds: job.delaySeconds } : {},
|
|
1686
4210
|
attempts: job.attempts ?? 0,
|
|
@@ -1707,10 +4231,10 @@ var ClassNameSerializer = class {
|
|
|
1707
4231
|
`Job class "${serialized.className}" is not registered. Please register it using serializer.register().`
|
|
1708
4232
|
);
|
|
1709
4233
|
}
|
|
1710
|
-
const
|
|
4234
|
+
const properties = JSON.parse(serialized.data);
|
|
1711
4235
|
const job = new JobClass();
|
|
1712
|
-
if (
|
|
1713
|
-
Object.assign(job,
|
|
4236
|
+
if (properties) {
|
|
4237
|
+
Object.assign(job, properties);
|
|
1714
4238
|
}
|
|
1715
4239
|
job.id = serialized.id;
|
|
1716
4240
|
if (serialized.delaySeconds !== void 0) {
|
|
@@ -1744,14 +4268,17 @@ var JsonSerializer = class {
|
|
|
1744
4268
|
* Serialize a job.
|
|
1745
4269
|
*/
|
|
1746
4270
|
serialize(job) {
|
|
1747
|
-
const id = `${Date.now()}-${
|
|
4271
|
+
const id = job.id || `${Date.now()}-${crypto.randomUUID()}`;
|
|
4272
|
+
const properties = {};
|
|
4273
|
+
for (const key in job) {
|
|
4274
|
+
if (Object.hasOwn(job, key) && typeof job[key] !== "function") {
|
|
4275
|
+
properties[key] = job[key];
|
|
4276
|
+
}
|
|
4277
|
+
}
|
|
1748
4278
|
return {
|
|
1749
4279
|
id,
|
|
1750
4280
|
type: "json",
|
|
1751
|
-
data: JSON.stringify(
|
|
1752
|
-
job: job.constructor.name,
|
|
1753
|
-
properties: { ...job }
|
|
1754
|
-
}),
|
|
4281
|
+
data: JSON.stringify(properties),
|
|
1755
4282
|
createdAt: Date.now(),
|
|
1756
4283
|
...job.delaySeconds !== void 0 ? { delaySeconds: job.delaySeconds } : {},
|
|
1757
4284
|
attempts: job.attempts ?? 0,
|
|
@@ -1762,23 +4289,27 @@ var JsonSerializer = class {
|
|
|
1762
4289
|
}
|
|
1763
4290
|
/**
|
|
1764
4291
|
* Deserialize a job.
|
|
1765
|
-
*
|
|
1766
|
-
* Note: this implementation only restores properties and does not recreate class instances.
|
|
1767
|
-
* For class instances, use `ClassNameSerializer`.
|
|
1768
4292
|
*/
|
|
1769
4293
|
deserialize(serialized) {
|
|
1770
4294
|
if (serialized.type !== "json") {
|
|
1771
4295
|
throw new Error('Invalid serialization type: expected "json"');
|
|
1772
4296
|
}
|
|
1773
|
-
const
|
|
4297
|
+
const properties = JSON.parse(serialized.data);
|
|
1774
4298
|
const job = /* @__PURE__ */ Object.create({});
|
|
1775
|
-
Object.assign(job,
|
|
4299
|
+
Object.assign(job, properties);
|
|
4300
|
+
job.id = serialized.id;
|
|
1776
4301
|
if (serialized.groupId) {
|
|
1777
4302
|
job.groupId = serialized.groupId;
|
|
1778
4303
|
}
|
|
1779
4304
|
if (serialized.priority) {
|
|
1780
4305
|
job.priority = serialized.priority;
|
|
1781
4306
|
}
|
|
4307
|
+
if (serialized.delaySeconds !== void 0) {
|
|
4308
|
+
job.delaySeconds = serialized.delaySeconds;
|
|
4309
|
+
}
|
|
4310
|
+
if (serialized.attempts !== void 0) {
|
|
4311
|
+
job.attempts = serialized.attempts;
|
|
4312
|
+
}
|
|
1782
4313
|
return job;
|
|
1783
4314
|
}
|
|
1784
4315
|
};
|
|
@@ -1791,16 +4322,30 @@ var QueueManager = class {
|
|
|
1791
4322
|
defaultSerializer;
|
|
1792
4323
|
persistence;
|
|
1793
4324
|
scheduler;
|
|
1794
|
-
|
|
4325
|
+
debug;
|
|
1795
4326
|
constructor(config = {}) {
|
|
1796
4327
|
this.persistence = config.persistence;
|
|
1797
4328
|
this.defaultConnection = config.default ?? "default";
|
|
4329
|
+
this.debug = config.debug ?? false;
|
|
4330
|
+
if (this.persistence && (this.persistence.bufferSize || this.persistence.flushInterval)) {
|
|
4331
|
+
const { BufferedPersistence: BufferedPersistence2 } = (init_BufferedPersistence(), __toCommonJS(BufferedPersistence_exports));
|
|
4332
|
+
this.persistence.adapter = new BufferedPersistence2(this.persistence.adapter, {
|
|
4333
|
+
maxBufferSize: this.persistence.bufferSize,
|
|
4334
|
+
flushInterval: this.persistence.flushInterval
|
|
4335
|
+
});
|
|
4336
|
+
}
|
|
1798
4337
|
const serializerType = config.defaultSerializer ?? "class";
|
|
1799
4338
|
if (serializerType === "class") {
|
|
1800
4339
|
this.defaultSerializer = new ClassNameSerializer();
|
|
4340
|
+
} else if (serializerType === "msgpack") {
|
|
4341
|
+
const { MessagePackSerializer: MessagePackSerializer2 } = (init_MessagePackSerializer(), __toCommonJS(MessagePackSerializer_exports));
|
|
4342
|
+
this.defaultSerializer = new MessagePackSerializer2();
|
|
1801
4343
|
} else {
|
|
1802
4344
|
this.defaultSerializer = new JsonSerializer();
|
|
1803
4345
|
}
|
|
4346
|
+
if (config.useSerializationCache) {
|
|
4347
|
+
this.defaultSerializer = new CachedSerializer(this.defaultSerializer);
|
|
4348
|
+
}
|
|
1804
4349
|
if (!this.drivers.has("default")) {
|
|
1805
4350
|
this.drivers.set("default", new MemoryDriver());
|
|
1806
4351
|
}
|
|
@@ -1810,6 +4355,20 @@ var QueueManager = class {
|
|
|
1810
4355
|
}
|
|
1811
4356
|
}
|
|
1812
4357
|
}
|
|
4358
|
+
/**
|
|
4359
|
+
* Log debug message.
|
|
4360
|
+
*/
|
|
4361
|
+
log(message, data) {
|
|
4362
|
+
if (this.debug) {
|
|
4363
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
4364
|
+
const prefix = `[QueueManager] [${timestamp}]`;
|
|
4365
|
+
if (data) {
|
|
4366
|
+
console.log(prefix, message, data);
|
|
4367
|
+
} else {
|
|
4368
|
+
console.log(prefix, message);
|
|
4369
|
+
}
|
|
4370
|
+
}
|
|
4371
|
+
}
|
|
1813
4372
|
/**
|
|
1814
4373
|
* Register a connection.
|
|
1815
4374
|
* @param name - Connection name
|
|
@@ -1823,8 +4382,7 @@ var QueueManager = class {
|
|
|
1823
4382
|
break;
|
|
1824
4383
|
case "database": {
|
|
1825
4384
|
const { DatabaseDriver: DatabaseDriver2 } = (init_DatabaseDriver(), __toCommonJS(DatabaseDriver_exports));
|
|
1826
|
-
|
|
1827
|
-
if (!dbService) {
|
|
4385
|
+
if (!config.dbService) {
|
|
1828
4386
|
throw new Error(
|
|
1829
4387
|
"[QueueManager] DatabaseDriver requires dbService. Please provide a database service that implements DatabaseService interface."
|
|
1830
4388
|
);
|
|
@@ -1832,9 +4390,7 @@ var QueueManager = class {
|
|
|
1832
4390
|
this.drivers.set(
|
|
1833
4391
|
name,
|
|
1834
4392
|
new DatabaseDriver2({
|
|
1835
|
-
|
|
1836
|
-
dbService,
|
|
1837
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4393
|
+
dbService: config.dbService,
|
|
1838
4394
|
table: config.table
|
|
1839
4395
|
})
|
|
1840
4396
|
);
|
|
@@ -1842,8 +4398,7 @@ var QueueManager = class {
|
|
|
1842
4398
|
}
|
|
1843
4399
|
case "redis": {
|
|
1844
4400
|
const { RedisDriver: RedisDriver2 } = (init_RedisDriver(), __toCommonJS(RedisDriver_exports));
|
|
1845
|
-
|
|
1846
|
-
if (!client) {
|
|
4401
|
+
if (!config.client) {
|
|
1847
4402
|
throw new Error(
|
|
1848
4403
|
"[QueueManager] RedisDriver requires client. Please provide Redis client in connection config."
|
|
1849
4404
|
);
|
|
@@ -1851,9 +4406,7 @@ var QueueManager = class {
|
|
|
1851
4406
|
this.drivers.set(
|
|
1852
4407
|
name,
|
|
1853
4408
|
new RedisDriver2({
|
|
1854
|
-
|
|
1855
|
-
client,
|
|
1856
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4409
|
+
client: config.client,
|
|
1857
4410
|
prefix: config.prefix
|
|
1858
4411
|
})
|
|
1859
4412
|
);
|
|
@@ -1861,8 +4414,7 @@ var QueueManager = class {
|
|
|
1861
4414
|
}
|
|
1862
4415
|
case "kafka": {
|
|
1863
4416
|
const { KafkaDriver: KafkaDriver2 } = (init_KafkaDriver(), __toCommonJS(KafkaDriver_exports));
|
|
1864
|
-
|
|
1865
|
-
if (!client) {
|
|
4417
|
+
if (!config.client) {
|
|
1866
4418
|
throw new Error(
|
|
1867
4419
|
"[QueueManager] KafkaDriver requires client. Please provide Kafka client in connection config."
|
|
1868
4420
|
);
|
|
@@ -1870,9 +4422,7 @@ var QueueManager = class {
|
|
|
1870
4422
|
this.drivers.set(
|
|
1871
4423
|
name,
|
|
1872
4424
|
new KafkaDriver2({
|
|
1873
|
-
|
|
1874
|
-
client,
|
|
1875
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4425
|
+
client: config.client,
|
|
1876
4426
|
consumerGroupId: config.consumerGroupId
|
|
1877
4427
|
})
|
|
1878
4428
|
);
|
|
@@ -1880,8 +4430,7 @@ var QueueManager = class {
|
|
|
1880
4430
|
}
|
|
1881
4431
|
case "sqs": {
|
|
1882
4432
|
const { SQSDriver: SQSDriver2 } = (init_SQSDriver(), __toCommonJS(SQSDriver_exports));
|
|
1883
|
-
|
|
1884
|
-
if (!client) {
|
|
4433
|
+
if (!config.client) {
|
|
1885
4434
|
throw new Error(
|
|
1886
4435
|
"[QueueManager] SQSDriver requires client. Please provide SQS client in connection config."
|
|
1887
4436
|
);
|
|
@@ -1889,13 +4438,9 @@ var QueueManager = class {
|
|
|
1889
4438
|
this.drivers.set(
|
|
1890
4439
|
name,
|
|
1891
4440
|
new SQSDriver2({
|
|
1892
|
-
|
|
1893
|
-
client,
|
|
1894
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4441
|
+
client: config.client,
|
|
1895
4442
|
queueUrlPrefix: config.queueUrlPrefix,
|
|
1896
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
1897
4443
|
visibilityTimeout: config.visibilityTimeout,
|
|
1898
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
1899
4444
|
waitTimeSeconds: config.waitTimeSeconds
|
|
1900
4445
|
})
|
|
1901
4446
|
);
|
|
@@ -1903,8 +4448,7 @@ var QueueManager = class {
|
|
|
1903
4448
|
}
|
|
1904
4449
|
case "rabbitmq": {
|
|
1905
4450
|
const { RabbitMQDriver: RabbitMQDriver2 } = (init_RabbitMQDriver(), __toCommonJS(RabbitMQDriver_exports));
|
|
1906
|
-
|
|
1907
|
-
if (!client) {
|
|
4451
|
+
if (!config.client) {
|
|
1908
4452
|
throw new Error(
|
|
1909
4453
|
"[QueueManager] RabbitMQDriver requires client. Please provide RabbitMQ connection/channel in connection config."
|
|
1910
4454
|
);
|
|
@@ -1912,11 +4456,8 @@ var QueueManager = class {
|
|
|
1912
4456
|
this.drivers.set(
|
|
1913
4457
|
name,
|
|
1914
4458
|
new RabbitMQDriver2({
|
|
1915
|
-
|
|
1916
|
-
client,
|
|
1917
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
4459
|
+
client: config.client,
|
|
1918
4460
|
exchange: config.exchange,
|
|
1919
|
-
// biome-ignore lint/suspicious/noExplicitAny: Dynamic driver config type
|
|
1920
4461
|
exchangeType: config.exchangeType
|
|
1921
4462
|
})
|
|
1922
4463
|
);
|
|
@@ -1995,6 +4536,11 @@ var QueueManager = class {
|
|
|
1995
4536
|
pushOptions.priority = job.priority;
|
|
1996
4537
|
}
|
|
1997
4538
|
await driver.push(queue, serialized, pushOptions);
|
|
4539
|
+
this.log(`Pushed job to ${queue} (${connection})`, {
|
|
4540
|
+
id: serialized.id,
|
|
4541
|
+
job: serialized.className ?? "json",
|
|
4542
|
+
options: pushOptions
|
|
4543
|
+
});
|
|
1998
4544
|
if (this.persistence?.archiveEnqueued) {
|
|
1999
4545
|
this.persistence.adapter.archive(queue, serialized, "waiting").catch((err) => {
|
|
2000
4546
|
console.error("[QueueManager] Persistence archive failed (waiting):", err);
|
|
@@ -2007,16 +4553,19 @@ var QueueManager = class {
|
|
|
2007
4553
|
*
|
|
2008
4554
|
* @template T - The type of the jobs.
|
|
2009
4555
|
* @param jobs - Array of job instances.
|
|
4556
|
+
* @param options - Bulk push options.
|
|
2010
4557
|
*
|
|
2011
4558
|
* @example
|
|
2012
4559
|
* ```typescript
|
|
2013
|
-
* await manager.pushMany(
|
|
4560
|
+
* await manager.pushMany(jobs, { batchSize: 500, concurrency: 2 });
|
|
2014
4561
|
* ```
|
|
2015
4562
|
*/
|
|
2016
|
-
async pushMany(jobs) {
|
|
4563
|
+
async pushMany(jobs, options = {}) {
|
|
2017
4564
|
if (jobs.length === 0) {
|
|
2018
4565
|
return;
|
|
2019
4566
|
}
|
|
4567
|
+
const batchSize = options.batchSize ?? 100;
|
|
4568
|
+
const concurrency = options.concurrency ?? 1;
|
|
2020
4569
|
const groups = /* @__PURE__ */ new Map();
|
|
2021
4570
|
const serializer = this.getSerializer();
|
|
2022
4571
|
for (const job of jobs) {
|
|
@@ -2029,17 +4578,42 @@ var QueueManager = class {
|
|
|
2029
4578
|
}
|
|
2030
4579
|
groups.get(key)?.push(serialized);
|
|
2031
4580
|
}
|
|
4581
|
+
const processBatch = async (driver, queue, batch) => {
|
|
4582
|
+
if (driver.pushMany) {
|
|
4583
|
+
await driver.pushMany(queue, batch);
|
|
4584
|
+
} else {
|
|
4585
|
+
for (const job of batch) {
|
|
4586
|
+
await driver.push(queue, job);
|
|
4587
|
+
}
|
|
4588
|
+
}
|
|
4589
|
+
};
|
|
2032
4590
|
for (const [key, serializedJobs] of groups.entries()) {
|
|
2033
4591
|
const [connection, queue] = key.split(":");
|
|
2034
4592
|
if (!connection || !queue) {
|
|
2035
4593
|
continue;
|
|
2036
4594
|
}
|
|
2037
4595
|
const driver = this.getDriver(connection);
|
|
2038
|
-
|
|
2039
|
-
|
|
4596
|
+
this.log(`Pushing ${serializedJobs.length} jobs to ${queue} (${connection})`);
|
|
4597
|
+
const chunks = [];
|
|
4598
|
+
for (let i = 0; i < serializedJobs.length; i += batchSize) {
|
|
4599
|
+
chunks.push(serializedJobs.slice(i, i + batchSize));
|
|
4600
|
+
}
|
|
4601
|
+
if (concurrency > 1) {
|
|
4602
|
+
const activePromises = [];
|
|
4603
|
+
for (const chunk of chunks) {
|
|
4604
|
+
const promise = processBatch(driver, queue, chunk);
|
|
4605
|
+
activePromises.push(promise);
|
|
4606
|
+
if (activePromises.length >= concurrency) {
|
|
4607
|
+
await Promise.race(activePromises);
|
|
4608
|
+
}
|
|
4609
|
+
}
|
|
4610
|
+
for (let i = 0; i < chunks.length; i += concurrency) {
|
|
4611
|
+
const batchPromises = chunks.slice(i, i + concurrency).map((chunk) => processBatch(driver, queue, chunk));
|
|
4612
|
+
await Promise.all(batchPromises);
|
|
4613
|
+
}
|
|
2040
4614
|
} else {
|
|
2041
|
-
for (const
|
|
2042
|
-
await driver
|
|
4615
|
+
for (const chunk of chunks) {
|
|
4616
|
+
await processBatch(driver, queue, chunk);
|
|
2043
4617
|
}
|
|
2044
4618
|
}
|
|
2045
4619
|
}
|
|
@@ -2064,6 +4638,7 @@ var QueueManager = class {
|
|
|
2064
4638
|
if (!serialized) {
|
|
2065
4639
|
return null;
|
|
2066
4640
|
}
|
|
4641
|
+
this.log(`Popped job from ${queue} (${connection})`, { id: serialized.id });
|
|
2067
4642
|
try {
|
|
2068
4643
|
return serializer.deserialize(serialized);
|
|
2069
4644
|
} catch (error) {
|
|
@@ -2071,6 +4646,42 @@ var QueueManager = class {
|
|
|
2071
4646
|
return null;
|
|
2072
4647
|
}
|
|
2073
4648
|
}
|
|
4649
|
+
/**
|
|
4650
|
+
* Pop multiple jobs from the queue.
|
|
4651
|
+
*
|
|
4652
|
+
* @param queue - Queue name (default: 'default').
|
|
4653
|
+
* @param count - Number of jobs to pop (default: 10).
|
|
4654
|
+
* @param connection - Connection name (optional).
|
|
4655
|
+
* @returns Array of Job instances.
|
|
4656
|
+
*/
|
|
4657
|
+
async popMany(queue = "default", count = 10, connection = this.defaultConnection) {
|
|
4658
|
+
const driver = this.getDriver(connection);
|
|
4659
|
+
const serializer = this.getSerializer();
|
|
4660
|
+
const results = [];
|
|
4661
|
+
if (driver.popMany) {
|
|
4662
|
+
const serializedJobs = await driver.popMany(queue, count);
|
|
4663
|
+
if (serializedJobs.length > 0) {
|
|
4664
|
+
this.log(`Popped ${serializedJobs.length} jobs from ${queue} (${connection})`);
|
|
4665
|
+
}
|
|
4666
|
+
for (const serialized of serializedJobs) {
|
|
4667
|
+
try {
|
|
4668
|
+
results.push(serializer.deserialize(serialized));
|
|
4669
|
+
} catch (error) {
|
|
4670
|
+
console.error("[QueueManager] Failed to deserialize job:", error);
|
|
4671
|
+
}
|
|
4672
|
+
}
|
|
4673
|
+
} else {
|
|
4674
|
+
for (let i = 0; i < count; i++) {
|
|
4675
|
+
const job = await this.pop(queue, connection);
|
|
4676
|
+
if (job) {
|
|
4677
|
+
results.push(job);
|
|
4678
|
+
} else {
|
|
4679
|
+
break;
|
|
4680
|
+
}
|
|
4681
|
+
}
|
|
4682
|
+
}
|
|
4683
|
+
return results;
|
|
4684
|
+
}
|
|
2074
4685
|
/**
|
|
2075
4686
|
* Get queue size.
|
|
2076
4687
|
*
|
|
@@ -2082,6 +4693,35 @@ var QueueManager = class {
|
|
|
2082
4693
|
const driver = this.getDriver(connection);
|
|
2083
4694
|
return driver.size(queue);
|
|
2084
4695
|
}
|
|
4696
|
+
/**
|
|
4697
|
+
* Pop a job from the queue (blocking).
|
|
4698
|
+
*
|
|
4699
|
+
* @param queue - Queue name (default: 'default').
|
|
4700
|
+
* @param timeout - Timeout in seconds (default: 0, wait forever).
|
|
4701
|
+
* @param connection - Connection name (optional).
|
|
4702
|
+
*/
|
|
4703
|
+
async popBlocking(queues = "default", timeout = 0, connection = this.defaultConnection) {
|
|
4704
|
+
const driver = this.getDriver(connection);
|
|
4705
|
+
const serializer = this.getSerializer();
|
|
4706
|
+
if (!driver.popBlocking) {
|
|
4707
|
+
const q = Array.isArray(queues) ? queues[0] : queues;
|
|
4708
|
+
return this.pop(q, connection);
|
|
4709
|
+
}
|
|
4710
|
+
const serialized = await driver.popBlocking(queues, timeout);
|
|
4711
|
+
if (!serialized) {
|
|
4712
|
+
return null;
|
|
4713
|
+
}
|
|
4714
|
+
this.log(
|
|
4715
|
+
`Popped job (blocking) from ${Array.isArray(queues) ? queues.join(",") : queues} (${connection})`,
|
|
4716
|
+
{ id: serialized.id }
|
|
4717
|
+
);
|
|
4718
|
+
try {
|
|
4719
|
+
return serializer.deserialize(serialized);
|
|
4720
|
+
} catch (error) {
|
|
4721
|
+
console.error("[QueueManager] Failed to deserialize job:", error);
|
|
4722
|
+
return null;
|
|
4723
|
+
}
|
|
4724
|
+
}
|
|
2085
4725
|
/**
|
|
2086
4726
|
* Clear all jobs from a queue.
|
|
2087
4727
|
*
|
|
@@ -2092,6 +4732,23 @@ var QueueManager = class {
|
|
|
2092
4732
|
const driver = this.getDriver(connection);
|
|
2093
4733
|
await driver.clear(queue);
|
|
2094
4734
|
}
|
|
4735
|
+
/**
|
|
4736
|
+
* Get queue statistics including size, delayed, and failed job counts.
|
|
4737
|
+
*
|
|
4738
|
+
* @param queue - Queue name (default: 'default').
|
|
4739
|
+
* @param connection - Connection name (optional).
|
|
4740
|
+
* @returns Queue statistics object.
|
|
4741
|
+
*/
|
|
4742
|
+
async stats(queue = "default", connection = this.defaultConnection) {
|
|
4743
|
+
const driver = this.getDriver(connection);
|
|
4744
|
+
if (driver.stats) {
|
|
4745
|
+
return await driver.stats(queue);
|
|
4746
|
+
}
|
|
4747
|
+
return {
|
|
4748
|
+
queue,
|
|
4749
|
+
size: await driver.size(queue)
|
|
4750
|
+
};
|
|
4751
|
+
}
|
|
2095
4752
|
/**
|
|
2096
4753
|
* Mark a job as completed.
|
|
2097
4754
|
* @param job - Job instance
|
|
@@ -2104,6 +4761,7 @@ var QueueManager = class {
|
|
|
2104
4761
|
if (driver.complete) {
|
|
2105
4762
|
const serialized = serializer.serialize(job);
|
|
2106
4763
|
await driver.complete(queue, serialized);
|
|
4764
|
+
this.log(`Completed job ${job.id} in ${queue}`);
|
|
2107
4765
|
if (this.persistence?.archiveCompleted) {
|
|
2108
4766
|
await this.persistence.adapter.archive(queue, serialized, "completed").catch((err) => {
|
|
2109
4767
|
console.error("[QueueManager] Persistence archive failed (completed):", err);
|
|
@@ -2126,6 +4784,7 @@ var QueueManager = class {
|
|
|
2126
4784
|
serialized.error = error.message;
|
|
2127
4785
|
serialized.failedAt = Date.now();
|
|
2128
4786
|
await driver.fail(queue, serialized);
|
|
4787
|
+
this.log(`Failed job ${job.id} in ${queue}`, { error: error.message });
|
|
2129
4788
|
if (this.persistence?.archiveFailed) {
|
|
2130
4789
|
await this.persistence.adapter.archive(queue, serialized, "failed").catch((err) => {
|
|
2131
4790
|
console.error("[QueueManager] Persistence archive failed (failed):", err);
|
|
@@ -2259,39 +4918,51 @@ var OrbitStream = class _OrbitStream {
|
|
|
2259
4918
|
}
|
|
2260
4919
|
};
|
|
2261
4920
|
|
|
4921
|
+
// src/index.ts
|
|
4922
|
+
init_BufferedPersistence();
|
|
4923
|
+
|
|
2262
4924
|
// src/persistence/MySQLPersistence.ts
|
|
2263
4925
|
var import_atlas = require("@gravito/atlas");
|
|
2264
4926
|
var MySQLPersistence = class {
|
|
2265
4927
|
/**
|
|
2266
4928
|
* @param db - An Atlas DB instance or compatible QueryBuilder.
|
|
2267
4929
|
* @param table - The name of the table to store archived jobs.
|
|
4930
|
+
* @param logsTable - The name of the table to store system logs.
|
|
4931
|
+
* @param options - Buffering options (Deprecated: Use BufferedPersistence wrapper instead).
|
|
2268
4932
|
*/
|
|
2269
|
-
constructor(db, table = "flux_job_archive", logsTable = "flux_system_logs") {
|
|
4933
|
+
constructor(db, table = "flux_job_archive", logsTable = "flux_system_logs", _options = {}) {
|
|
2270
4934
|
this.db = db;
|
|
2271
4935
|
this.table = table;
|
|
2272
4936
|
this.logsTable = logsTable;
|
|
2273
4937
|
}
|
|
2274
|
-
/**
|
|
2275
|
-
* Archive a job.
|
|
2276
|
-
*/
|
|
2277
4938
|
async archive(queue, job, status) {
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
4939
|
+
await this.archiveMany([{ queue, job, status }]);
|
|
4940
|
+
}
|
|
4941
|
+
async archiveMany(jobs) {
|
|
4942
|
+
if (jobs.length === 0) {
|
|
4943
|
+
return;
|
|
4944
|
+
}
|
|
4945
|
+
const batchSize = 500;
|
|
4946
|
+
for (let i = 0; i < jobs.length; i += batchSize) {
|
|
4947
|
+
const chunk = jobs.slice(i, i + batchSize);
|
|
4948
|
+
try {
|
|
4949
|
+
const records = chunk.map((item) => ({
|
|
4950
|
+
job_id: item.job.id,
|
|
4951
|
+
queue: item.queue,
|
|
4952
|
+
status: item.status,
|
|
4953
|
+
payload: JSON.stringify(item.job),
|
|
4954
|
+
error: item.job.error || null,
|
|
4955
|
+
created_at: new Date(item.job.createdAt),
|
|
4956
|
+
archived_at: /* @__PURE__ */ new Date()
|
|
4957
|
+
}));
|
|
4958
|
+
await this.db.table(this.table).insert(records);
|
|
4959
|
+
} catch (err) {
|
|
4960
|
+
console.error(`[MySQLPersistence] Failed to archive ${chunk.length} jobs:`, err);
|
|
4961
|
+
}
|
|
2290
4962
|
}
|
|
2291
4963
|
}
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
*/
|
|
4964
|
+
async flush() {
|
|
4965
|
+
}
|
|
2295
4966
|
async find(queue, id) {
|
|
2296
4967
|
const row = await this.db.table(this.table).where("queue", queue).where("job_id", id).first();
|
|
2297
4968
|
if (!row) {
|
|
@@ -2329,7 +5000,9 @@ var MySQLPersistence = class {
|
|
|
2329
5000
|
} catch (_e) {
|
|
2330
5001
|
return null;
|
|
2331
5002
|
}
|
|
2332
|
-
}).filter(
|
|
5003
|
+
}).filter(
|
|
5004
|
+
(item) => !!item
|
|
5005
|
+
);
|
|
2333
5006
|
}
|
|
2334
5007
|
/**
|
|
2335
5008
|
* Search jobs from the archive.
|
|
@@ -2349,22 +5022,35 @@ var MySQLPersistence = class {
|
|
|
2349
5022
|
} catch (_e) {
|
|
2350
5023
|
return null;
|
|
2351
5024
|
}
|
|
2352
|
-
}).filter(
|
|
5025
|
+
}).filter(
|
|
5026
|
+
(item) => !!item
|
|
5027
|
+
);
|
|
2353
5028
|
}
|
|
2354
5029
|
/**
|
|
2355
|
-
* Archive a system log message.
|
|
5030
|
+
* Archive a system log message (buffered).
|
|
2356
5031
|
*/
|
|
2357
5032
|
async archiveLog(log) {
|
|
5033
|
+
await this.archiveLogMany([log]);
|
|
5034
|
+
}
|
|
5035
|
+
/**
|
|
5036
|
+
* Archive multiple log messages (direct batch write).
|
|
5037
|
+
*/
|
|
5038
|
+
async archiveLogMany(logs) {
|
|
5039
|
+
if (logs.length === 0) {
|
|
5040
|
+
return;
|
|
5041
|
+
}
|
|
2358
5042
|
try {
|
|
2359
|
-
|
|
5043
|
+
const records = logs.map((log) => ({
|
|
2360
5044
|
level: log.level,
|
|
2361
5045
|
message: log.message,
|
|
2362
5046
|
worker_id: log.workerId,
|
|
2363
5047
|
queue: log.queue || null,
|
|
2364
5048
|
timestamp: log.timestamp
|
|
2365
|
-
});
|
|
5049
|
+
}));
|
|
5050
|
+
await this.db.table(this.logsTable).insert(records);
|
|
2366
5051
|
} catch (err) {
|
|
2367
|
-
|
|
5052
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
5053
|
+
console.error(`[MySQLPersistence] Failed to archive ${logs.length} logs:`, error.message);
|
|
2368
5054
|
}
|
|
2369
5055
|
}
|
|
2370
5056
|
/**
|
|
@@ -2415,8 +5101,8 @@ var MySQLPersistence = class {
|
|
|
2415
5101
|
if (options.endTime) {
|
|
2416
5102
|
query = query.where("timestamp", "<=", options.endTime);
|
|
2417
5103
|
}
|
|
2418
|
-
const result = await query.count(
|
|
2419
|
-
return result
|
|
5104
|
+
const result = await query.count();
|
|
5105
|
+
return Number(result) || 0;
|
|
2420
5106
|
}
|
|
2421
5107
|
/**
|
|
2422
5108
|
* Remove old records from the archive.
|
|
@@ -2428,7 +5114,7 @@ var MySQLPersistence = class {
|
|
|
2428
5114
|
this.db.table(this.table).where("archived_at", "<", threshold).delete(),
|
|
2429
5115
|
this.db.table(this.logsTable).where("timestamp", "<", threshold).delete()
|
|
2430
5116
|
]);
|
|
2431
|
-
return (jobsDeleted || 0) + (logsDeleted || 0);
|
|
5117
|
+
return (Number(jobsDeleted) || 0) + (Number(logsDeleted) || 0);
|
|
2432
5118
|
}
|
|
2433
5119
|
/**
|
|
2434
5120
|
* Count jobs in the archive.
|
|
@@ -2447,8 +5133,8 @@ var MySQLPersistence = class {
|
|
|
2447
5133
|
if (options.endTime) {
|
|
2448
5134
|
query = query.where("archived_at", "<=", options.endTime);
|
|
2449
5135
|
}
|
|
2450
|
-
const result = await query.count(
|
|
2451
|
-
return result
|
|
5136
|
+
const result = await query.count();
|
|
5137
|
+
return Number(result) || 0;
|
|
2452
5138
|
}
|
|
2453
5139
|
/**
|
|
2454
5140
|
* Help script to create the necessary table.
|
|
@@ -2504,33 +5190,49 @@ var SQLitePersistence = class {
|
|
|
2504
5190
|
/**
|
|
2505
5191
|
* @param db - An Atlas DB instance (SQLite driver).
|
|
2506
5192
|
* @param table - The name of the table to store archived jobs.
|
|
5193
|
+
* @param logsTable - The name of the table to store system logs.
|
|
5194
|
+
* @param options - Buffering options (Deprecated: Use BufferedPersistence wrapper instead).
|
|
2507
5195
|
*/
|
|
2508
|
-
constructor(db, table = "flux_job_archive", logsTable = "flux_system_logs") {
|
|
5196
|
+
constructor(db, table = "flux_job_archive", logsTable = "flux_system_logs", _options = {}) {
|
|
2509
5197
|
this.db = db;
|
|
2510
5198
|
this.table = table;
|
|
2511
5199
|
this.logsTable = logsTable;
|
|
2512
5200
|
}
|
|
2513
|
-
/**
|
|
2514
|
-
* Archive a job.
|
|
2515
|
-
*/
|
|
2516
5201
|
async archive(queue, job, status) {
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
5202
|
+
await this.archiveMany([{ queue, job, status }]);
|
|
5203
|
+
}
|
|
5204
|
+
async archiveMany(jobs) {
|
|
5205
|
+
if (jobs.length === 0) {
|
|
5206
|
+
return;
|
|
5207
|
+
}
|
|
5208
|
+
const batchSize = 200;
|
|
5209
|
+
for (let i = 0; i < jobs.length; i += batchSize) {
|
|
5210
|
+
const chunk = jobs.slice(i, i + batchSize);
|
|
5211
|
+
try {
|
|
5212
|
+
const records = chunk.map((item) => ({
|
|
5213
|
+
job_id: item.job.id,
|
|
5214
|
+
queue: item.queue,
|
|
5215
|
+
status: item.status,
|
|
5216
|
+
payload: JSON.stringify(item.job),
|
|
5217
|
+
error: item.job.error || null,
|
|
5218
|
+
created_at: new Date(item.job.createdAt),
|
|
5219
|
+
archived_at: /* @__PURE__ */ new Date()
|
|
5220
|
+
}));
|
|
5221
|
+
if (typeof this.db.transaction === "function") {
|
|
5222
|
+
await this.db.transaction(async (trx) => {
|
|
5223
|
+
await trx.table(this.table).insert(records);
|
|
5224
|
+
});
|
|
5225
|
+
} else {
|
|
5226
|
+
await this.db.table(this.table).insert(records);
|
|
5227
|
+
}
|
|
5228
|
+
} catch (err) {
|
|
5229
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
5230
|
+
console.error(`[SQLitePersistence] Failed to archive ${chunk.length} jobs:`, error.message);
|
|
5231
|
+
}
|
|
2529
5232
|
}
|
|
2530
5233
|
}
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
*/
|
|
5234
|
+
async flush() {
|
|
5235
|
+
}
|
|
2534
5236
|
async find(queue, id) {
|
|
2535
5237
|
const row = await this.db.table(this.table).where("queue", queue).where("job_id", id).first();
|
|
2536
5238
|
if (!row) {
|
|
@@ -2568,7 +5270,9 @@ var SQLitePersistence = class {
|
|
|
2568
5270
|
} catch (_e) {
|
|
2569
5271
|
return null;
|
|
2570
5272
|
}
|
|
2571
|
-
}).filter(
|
|
5273
|
+
}).filter(
|
|
5274
|
+
(item) => !!item
|
|
5275
|
+
);
|
|
2572
5276
|
}
|
|
2573
5277
|
/**
|
|
2574
5278
|
* Search jobs from the archive.
|
|
@@ -2588,22 +5292,35 @@ var SQLitePersistence = class {
|
|
|
2588
5292
|
} catch (_e) {
|
|
2589
5293
|
return null;
|
|
2590
5294
|
}
|
|
2591
|
-
}).filter(
|
|
5295
|
+
}).filter(
|
|
5296
|
+
(item) => !!item
|
|
5297
|
+
);
|
|
2592
5298
|
}
|
|
2593
5299
|
/**
|
|
2594
|
-
* Archive a system log message.
|
|
5300
|
+
* Archive a system log message (buffered).
|
|
2595
5301
|
*/
|
|
2596
5302
|
async archiveLog(log) {
|
|
5303
|
+
await this.archiveLogMany([log]);
|
|
5304
|
+
}
|
|
5305
|
+
/**
|
|
5306
|
+
* Archive multiple log messages (direct batch write).
|
|
5307
|
+
*/
|
|
5308
|
+
async archiveLogMany(logs) {
|
|
5309
|
+
if (logs.length === 0) {
|
|
5310
|
+
return;
|
|
5311
|
+
}
|
|
2597
5312
|
try {
|
|
2598
|
-
|
|
5313
|
+
const records = logs.map((log) => ({
|
|
2599
5314
|
level: log.level,
|
|
2600
5315
|
message: log.message,
|
|
2601
5316
|
worker_id: log.workerId,
|
|
2602
5317
|
queue: log.queue || null,
|
|
2603
5318
|
timestamp: log.timestamp
|
|
2604
|
-
});
|
|
5319
|
+
}));
|
|
5320
|
+
await this.db.table(this.logsTable).insert(records);
|
|
2605
5321
|
} catch (err) {
|
|
2606
|
-
|
|
5322
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
5323
|
+
console.error(`[SQLitePersistence] Failed to archive ${logs.length} logs:`, error.message);
|
|
2607
5324
|
}
|
|
2608
5325
|
}
|
|
2609
5326
|
/**
|
|
@@ -2654,8 +5371,8 @@ var SQLitePersistence = class {
|
|
|
2654
5371
|
if (options.endTime) {
|
|
2655
5372
|
query = query.where("timestamp", "<=", options.endTime);
|
|
2656
5373
|
}
|
|
2657
|
-
const result = await query.count(
|
|
2658
|
-
return result
|
|
5374
|
+
const result = await query.count();
|
|
5375
|
+
return Number(result) || 0;
|
|
2659
5376
|
}
|
|
2660
5377
|
/**
|
|
2661
5378
|
* Remove old records from the archive.
|
|
@@ -2667,7 +5384,7 @@ var SQLitePersistence = class {
|
|
|
2667
5384
|
this.db.table(this.table).where("archived_at", "<", threshold).delete(),
|
|
2668
5385
|
this.db.table(this.logsTable).where("timestamp", "<", threshold).delete()
|
|
2669
5386
|
]);
|
|
2670
|
-
return (jobsDeleted || 0) + (logsDeleted || 0);
|
|
5387
|
+
return (Number(jobsDeleted) || 0) + (Number(logsDeleted) || 0);
|
|
2671
5388
|
}
|
|
2672
5389
|
/**
|
|
2673
5390
|
* Count jobs in the archive.
|
|
@@ -2686,8 +5403,8 @@ var SQLitePersistence = class {
|
|
|
2686
5403
|
if (options.endTime) {
|
|
2687
5404
|
query = query.where("archived_at", "<=", options.endTime);
|
|
2688
5405
|
}
|
|
2689
|
-
const result = await query.count(
|
|
2690
|
-
return result
|
|
5406
|
+
const result = await query.count();
|
|
5407
|
+
return Number(result) || 0;
|
|
2691
5408
|
}
|
|
2692
5409
|
/**
|
|
2693
5410
|
* Setup table for SQLite.
|
|
@@ -2739,6 +5456,7 @@ var SQLitePersistence = class {
|
|
|
2739
5456
|
init_Scheduler();
|
|
2740
5457
|
// Annotate the CommonJS export names for ESM import in node:
|
|
2741
5458
|
0 && (module.exports = {
|
|
5459
|
+
BufferedPersistence,
|
|
2742
5460
|
ClassNameSerializer,
|
|
2743
5461
|
Consumer,
|
|
2744
5462
|
DatabaseDriver,
|