@adtrackify/at-service-common 3.18.7 → 3.18.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/__tests__/clients/sqs-unbundle.spec.js +456 -5
- package/dist/cjs/__tests__/clients/sqs-unbundle.spec.js.map +1 -1
- package/dist/cjs/clients/generic/sqs-bundled-client.types.d.ts +2 -1
- package/dist/cjs/clients/generic/sqs-bundled-client.types.js +1 -0
- package/dist/cjs/clients/generic/sqs-bundled-client.types.js.map +1 -1
- package/dist/cjs/types/shopify-rest-types.d.ts +1 -0
- package/dist/cjs/utils/compression.d.ts +1 -0
- package/dist/cjs/utils/compression.js +23 -1
- package/dist/cjs/utils/compression.js.map +1 -1
- package/dist/esm/__tests__/clients/sqs-unbundle.spec.js +457 -6
- package/dist/esm/__tests__/clients/sqs-unbundle.spec.js.map +1 -1
- package/dist/esm/clients/generic/sqs-bundled-client.types.d.ts +2 -1
- package/dist/esm/clients/generic/sqs-bundled-client.types.js +1 -0
- package/dist/esm/clients/generic/sqs-bundled-client.types.js.map +1 -1
- package/dist/esm/types/shopify-rest-types.d.ts +1 -0
- package/dist/esm/utils/compression.d.ts +1 -0
- package/dist/esm/utils/compression.js +22 -1
- package/dist/esm/utils/compression.js.map +1 -1
- package/package.json +1 -1
|
@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const sqs_unbundle_1 = require("../../clients/generic/sqs-unbundle");
|
|
4
4
|
const sqs_bundled_client_types_1 = require("../../clients/generic/sqs-bundled-client.types");
|
|
5
5
|
const compression_1 = require("../../utils/compression");
|
|
6
|
+
const node_zlib_1 = require("node:zlib");
|
|
7
|
+
const node_util_1 = require("node:util");
|
|
8
|
+
const gzipAsync = (0, node_util_1.promisify)(node_zlib_1.gzip);
|
|
6
9
|
jest.mock('../../helpers/logging-helper', () => ({
|
|
7
10
|
Logger: {
|
|
8
11
|
debug: jest.fn(),
|
|
@@ -41,6 +44,18 @@ async function makeCompressedEnvelope(items, level = 3) {
|
|
|
41
44
|
p: base64,
|
|
42
45
|
};
|
|
43
46
|
}
|
|
47
|
+
async function makeGzipCompressedEnvelope(items) {
|
|
48
|
+
const json = JSON.stringify(items);
|
|
49
|
+
const compressed = await gzipAsync(Buffer.from(json, 'utf8'));
|
|
50
|
+
const base64 = compressed.toString('base64');
|
|
51
|
+
return {
|
|
52
|
+
v: 1,
|
|
53
|
+
c: sqs_bundled_client_types_1.CompressionAlgorithm.GZIP,
|
|
54
|
+
n: items.length,
|
|
55
|
+
s: json.length,
|
|
56
|
+
p: base64,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
44
59
|
describe('isBundledEnvelope', () => {
|
|
45
60
|
it('returns true for valid bundled envelope with NONE compression', () => {
|
|
46
61
|
const envelope = makeBundledEnvelope([makeTestEvent('1')]);
|
|
@@ -63,8 +78,12 @@ describe('isBundledEnvelope', () => {
|
|
|
63
78
|
const envelope = { v: 2, c: 'none', n: 1, p: [] };
|
|
64
79
|
expect((0, sqs_unbundle_1.isBundledEnvelope)(envelope)).toBe(false);
|
|
65
80
|
});
|
|
81
|
+
it('returns true for valid bundled envelope with GZIP compression', async () => {
|
|
82
|
+
const envelope = await makeGzipCompressedEnvelope([makeTestEvent('1')]);
|
|
83
|
+
expect((0, sqs_unbundle_1.isBundledEnvelope)(envelope)).toBe(true);
|
|
84
|
+
});
|
|
66
85
|
it('returns false for invalid compression algorithm', () => {
|
|
67
|
-
const envelope = { v: 1, c: '
|
|
86
|
+
const envelope = { v: 1, c: 'lz4', n: 1, p: [] };
|
|
68
87
|
expect((0, sqs_unbundle_1.isBundledEnvelope)(envelope)).toBe(false);
|
|
69
88
|
});
|
|
70
89
|
it('returns false for missing count', () => {
|
|
@@ -114,6 +133,27 @@ describe('unbundleMessage', () => {
|
|
|
114
133
|
const envelope = { v: 1, c: 'zstd', n: 1, p: [] };
|
|
115
134
|
await expect((0, sqs_unbundle_1.unbundleMessage)(envelope)).rejects.toThrow('Invalid bundle: zstd compression but payload is not a string');
|
|
116
135
|
});
|
|
136
|
+
it('unbundles GZIP compressed envelope', async () => {
|
|
137
|
+
const items = [makeTestEvent('1'), makeTestEvent('2'), makeTestEvent('3')];
|
|
138
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
139
|
+
const result = await (0, sqs_unbundle_1.unbundleMessage)(envelope);
|
|
140
|
+
expect(result).toHaveLength(3);
|
|
141
|
+
expect(result[0].eventId).toBe('1');
|
|
142
|
+
expect(result[1].eventId).toBe('2');
|
|
143
|
+
expect(result[2].eventId).toBe('3');
|
|
144
|
+
});
|
|
145
|
+
it('unbundles large batches with GZIP compression', async () => {
|
|
146
|
+
const items = Array.from({ length: 500 }, (_, i) => makeTestEvent(`event-${i}`, `event_${i}`, i * 10));
|
|
147
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
148
|
+
const result = await (0, sqs_unbundle_1.unbundleMessage)(envelope);
|
|
149
|
+
expect(result).toHaveLength(500);
|
|
150
|
+
expect(result[0].eventId).toBe('event-0');
|
|
151
|
+
expect(result[499].eventId).toBe('event-499');
|
|
152
|
+
});
|
|
153
|
+
it('throws on invalid payload type for GZIP compression', async () => {
|
|
154
|
+
const envelope = { v: 1, c: 'gzip', n: 1, p: [] };
|
|
155
|
+
await expect((0, sqs_unbundle_1.unbundleMessage)(envelope)).rejects.toThrow('Invalid bundle: gzip compression but payload is not a string');
|
|
156
|
+
});
|
|
117
157
|
});
|
|
118
158
|
describe('unbundleRecords', () => {
|
|
119
159
|
describe('legacy (non-bundled) messages', () => {
|
|
@@ -203,6 +243,37 @@ describe('unbundleRecords', () => {
|
|
|
203
243
|
expect(stats.totalItems).toBe(500);
|
|
204
244
|
});
|
|
205
245
|
});
|
|
246
|
+
describe('bundled messages (GZIP compression - CF Worker)', () => {
|
|
247
|
+
it('unbundles GZIP compressed envelope', async () => {
|
|
248
|
+
const items = [makeTestEvent('g1'), makeTestEvent('g2')];
|
|
249
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
250
|
+
const record = makeSqsRecord({ messageBody: envelope });
|
|
251
|
+
const { items: result, stats } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
252
|
+
expect(result).toHaveLength(2);
|
|
253
|
+
expect(result[0].eventId).toBe('g1');
|
|
254
|
+
expect(result[1].eventId).toBe('g2');
|
|
255
|
+
expect(stats.bundledSqsRecords).toBe(1);
|
|
256
|
+
});
|
|
257
|
+
it('handles large GZIP compressed bundles (500 items)', async () => {
|
|
258
|
+
const items = Array.from({ length: 500 }, (_, i) => makeTestEvent(`item-${i}`));
|
|
259
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
260
|
+
const record = makeSqsRecord({ messageBody: envelope });
|
|
261
|
+
const { items: result, stats } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
262
|
+
expect(result).toHaveLength(500);
|
|
263
|
+
expect(stats.bundledSqsRecords).toBe(1);
|
|
264
|
+
expect(stats.totalItems).toBe(500);
|
|
265
|
+
});
|
|
266
|
+
it('maps all items from GZIP bundle to same messageId', async () => {
|
|
267
|
+
const items = [makeTestEvent('x1'), makeTestEvent('x2'), makeTestEvent('x3')];
|
|
268
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
269
|
+
const record = makeSqsRecord({ messageBody: envelope }, 'gzip-bundle-msg-1');
|
|
270
|
+
const { items: result, recordMap } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
271
|
+
expect(result).toHaveLength(3);
|
|
272
|
+
expect(recordMap.get(result[0])).toBe('gzip-bundle-msg-1');
|
|
273
|
+
expect(recordMap.get(result[1])).toBe('gzip-bundle-msg-1');
|
|
274
|
+
expect(recordMap.get(result[2])).toBe('gzip-bundle-msg-1');
|
|
275
|
+
});
|
|
276
|
+
});
|
|
206
277
|
describe('mixed message formats', () => {
|
|
207
278
|
it('handles mix of legacy and bundled messages', async () => {
|
|
208
279
|
const legacyEvent = makeTestEvent('legacy-1');
|
|
@@ -235,6 +306,29 @@ describe('unbundleRecords', () => {
|
|
|
235
306
|
expect(stats.bundledSqsRecords).toBe(2);
|
|
236
307
|
expect(stats.totalItems).toBe(3);
|
|
237
308
|
});
|
|
309
|
+
it('handles mix of ZSTD, GZIP, and legacy messages', async () => {
|
|
310
|
+
const legacyEvent = makeTestEvent('legacy-1');
|
|
311
|
+
const zstdItems = [makeTestEvent('zstd-1'), makeTestEvent('zstd-2')];
|
|
312
|
+
const gzipItems = [makeTestEvent('gzip-1'), makeTestEvent('gzip-2'), makeTestEvent('gzip-3')];
|
|
313
|
+
const zstdEnvelope = await makeCompressedEnvelope(zstdItems);
|
|
314
|
+
const gzipEnvelope = await makeGzipCompressedEnvelope(gzipItems);
|
|
315
|
+
const records = [
|
|
316
|
+
makeSqsRecord(legacyEvent, 'msg-legacy'),
|
|
317
|
+
makeSqsRecord({ messageBody: zstdEnvelope }, 'msg-zstd'),
|
|
318
|
+
makeSqsRecord({ messageBody: gzipEnvelope }, 'msg-gzip'),
|
|
319
|
+
];
|
|
320
|
+
const { items, stats, recordMap } = await (0, sqs_unbundle_1.unbundleRecords)(records);
|
|
321
|
+
expect(items).toHaveLength(6);
|
|
322
|
+
expect(stats.legacySqsRecords).toBe(1);
|
|
323
|
+
expect(stats.bundledSqsRecords).toBe(2);
|
|
324
|
+
expect(stats.totalItems).toBe(6);
|
|
325
|
+
expect(recordMap.get(items[0])).toBe('msg-legacy');
|
|
326
|
+
expect(recordMap.get(items[1])).toBe('msg-zstd');
|
|
327
|
+
expect(recordMap.get(items[2])).toBe('msg-zstd');
|
|
328
|
+
expect(recordMap.get(items[3])).toBe('msg-gzip');
|
|
329
|
+
expect(recordMap.get(items[4])).toBe('msg-gzip');
|
|
330
|
+
expect(recordMap.get(items[5])).toBe('msg-gzip');
|
|
331
|
+
});
|
|
238
332
|
});
|
|
239
333
|
describe('error handling', () => {
|
|
240
334
|
it('returns failed messageId for invalid JSON', async () => {
|
|
@@ -246,7 +340,7 @@ describe('unbundleRecords', () => {
|
|
|
246
340
|
expect(stats.failedRecords).toBe(1);
|
|
247
341
|
expect(stats.legacySqsRecords).toBe(1);
|
|
248
342
|
});
|
|
249
|
-
it('returns failed messageId for corrupt compressed data', async () => {
|
|
343
|
+
it('returns failed messageId for corrupt ZSTD compressed data', async () => {
|
|
250
344
|
const corruptEnvelope = {
|
|
251
345
|
v: 1,
|
|
252
346
|
c: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
@@ -263,6 +357,23 @@ describe('unbundleRecords', () => {
|
|
|
263
357
|
expect(failedMessageIds).toContain('msg-corrupt');
|
|
264
358
|
expect(stats.failedRecords).toBe(1);
|
|
265
359
|
});
|
|
360
|
+
it('returns failed messageId for corrupt GZIP compressed data', async () => {
|
|
361
|
+
const corruptEnvelope = {
|
|
362
|
+
v: 1,
|
|
363
|
+
c: sqs_bundled_client_types_1.CompressionAlgorithm.GZIP,
|
|
364
|
+
n: 10,
|
|
365
|
+
p: 'dGhpcyBpcyBub3QgdmFsaWQgZ3ppcA==',
|
|
366
|
+
};
|
|
367
|
+
const records = [
|
|
368
|
+
makeSqsRecord({ messageBody: corruptEnvelope }, 'msg-corrupt-gzip'),
|
|
369
|
+
makeSqsRecord(makeTestEvent('valid'), 'msg-valid'),
|
|
370
|
+
];
|
|
371
|
+
const { items, failedMessageIds, stats } = await (0, sqs_unbundle_1.unbundleRecords)(records);
|
|
372
|
+
expect(items).toHaveLength(1);
|
|
373
|
+
expect(items[0].eventId).toBe('valid');
|
|
374
|
+
expect(failedMessageIds).toContain('msg-corrupt-gzip');
|
|
375
|
+
expect(stats.failedRecords).toBe(1);
|
|
376
|
+
});
|
|
266
377
|
it('returns empty result for all failed records', async () => {
|
|
267
378
|
const records = [makeSqsRecord('invalid-1', 'msg-1'), makeSqsRecord('invalid-2', 'msg-2')];
|
|
268
379
|
const { items, failedMessageIds, stats } = await (0, sqs_unbundle_1.unbundleRecords)(records);
|
|
@@ -278,7 +389,7 @@ describe('unbundleRecords', () => {
|
|
|
278
389
|
expect(stats.totalRecords).toBe(0);
|
|
279
390
|
expect(stats.totalItems).toBe(0);
|
|
280
391
|
});
|
|
281
|
-
it('rejects payload exceeding max decompressed size', async () => {
|
|
392
|
+
it('rejects ZSTD payload exceeding max decompressed size', async () => {
|
|
282
393
|
const largeItems = Array.from({ length: 1000 }, (_, i) => ({
|
|
283
394
|
id: `item-${i}`,
|
|
284
395
|
data: 'x'.repeat(1000),
|
|
@@ -291,6 +402,19 @@ describe('unbundleRecords', () => {
|
|
|
291
402
|
expect(failedMessageIds).toContain('msg-too-large');
|
|
292
403
|
expect(stats.failedRecords).toBe(1);
|
|
293
404
|
});
|
|
405
|
+
it('rejects GZIP payload exceeding max decompressed size (post-decompress check)', async () => {
|
|
406
|
+
const largeItems = Array.from({ length: 1000 }, (_, i) => ({
|
|
407
|
+
id: `item-${i}`,
|
|
408
|
+
data: 'x'.repeat(1000),
|
|
409
|
+
}));
|
|
410
|
+
const envelope = await makeGzipCompressedEnvelope(largeItems);
|
|
411
|
+
const record = makeSqsRecord({ messageBody: envelope }, 'msg-too-large-gzip');
|
|
412
|
+
const { failedMessageIds, stats } = await (0, sqs_unbundle_1.unbundleRecords)([record], {
|
|
413
|
+
maxDecompressedSizeBytes: 100 * 1024,
|
|
414
|
+
});
|
|
415
|
+
expect(failedMessageIds).toContain('msg-too-large-gzip');
|
|
416
|
+
expect(stats.failedRecords).toBe(1);
|
|
417
|
+
});
|
|
294
418
|
it('fails on invalid base64 in compressed payload', async () => {
|
|
295
419
|
const badEnvelope = {
|
|
296
420
|
v: 1,
|
|
@@ -328,7 +452,7 @@ describe('unbundleRecords', () => {
|
|
|
328
452
|
});
|
|
329
453
|
});
|
|
330
454
|
describe('timeout protection', () => {
|
|
331
|
-
it('respects timeoutMs option', async () => {
|
|
455
|
+
it('respects timeoutMs option for ZSTD', async () => {
|
|
332
456
|
const items = [makeTestEvent('1')];
|
|
333
457
|
const envelope = await makeCompressedEnvelope(items);
|
|
334
458
|
const record = makeSqsRecord({ messageBody: envelope });
|
|
@@ -338,6 +462,16 @@ describe('unbundleRecords', () => {
|
|
|
338
462
|
expect(result).toHaveLength(1);
|
|
339
463
|
expect(failedMessageIds).toHaveLength(0);
|
|
340
464
|
});
|
|
465
|
+
it('respects timeoutMs option for GZIP', async () => {
|
|
466
|
+
const items = [makeTestEvent('1')];
|
|
467
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
468
|
+
const record = makeSqsRecord({ messageBody: envelope });
|
|
469
|
+
const { items: result, failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record], {
|
|
470
|
+
timeoutMs: 5000,
|
|
471
|
+
});
|
|
472
|
+
expect(result).toHaveLength(1);
|
|
473
|
+
expect(failedMessageIds).toHaveLength(0);
|
|
474
|
+
});
|
|
341
475
|
it('passes timeoutMs through to decompression', async () => {
|
|
342
476
|
const items = [makeTestEvent('1')];
|
|
343
477
|
const envelope = await makeCompressedEnvelope(items);
|
|
@@ -352,7 +486,7 @@ describe('unbundleRecords', () => {
|
|
|
352
486
|
expect(error.timeoutMs).toBe(5000);
|
|
353
487
|
expect(error.message).toContain('timed out after 5000ms');
|
|
354
488
|
});
|
|
355
|
-
it('includes timeout info in failure logging', async () => {
|
|
489
|
+
it('includes timeout info in failure logging for ZSTD', async () => {
|
|
356
490
|
const corruptEnvelope = {
|
|
357
491
|
v: 1,
|
|
358
492
|
c: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
@@ -366,6 +500,20 @@ describe('unbundleRecords', () => {
|
|
|
366
500
|
});
|
|
367
501
|
expect(failedMessageIds).toContain('msg-timeout-test');
|
|
368
502
|
});
|
|
503
|
+
it('includes timeout info in failure logging for GZIP', async () => {
|
|
504
|
+
const corruptEnvelope = {
|
|
505
|
+
v: 1,
|
|
506
|
+
c: sqs_bundled_client_types_1.CompressionAlgorithm.GZIP,
|
|
507
|
+
n: 1,
|
|
508
|
+
s: 100,
|
|
509
|
+
p: 'dGhpcyBpcyBub3QgdmFsaWQgZ3ppcA==',
|
|
510
|
+
};
|
|
511
|
+
const record = makeSqsRecord({ messageBody: corruptEnvelope }, 'msg-gzip-timeout-test');
|
|
512
|
+
const { failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record], {
|
|
513
|
+
timeoutMs: 5000,
|
|
514
|
+
});
|
|
515
|
+
expect(failedMessageIds).toContain('msg-gzip-timeout-test');
|
|
516
|
+
});
|
|
369
517
|
});
|
|
370
518
|
describe('end-to-end: producer bundle → consumer unbundle', () => {
|
|
371
519
|
it('simulates full bundle → unbundle flow with compression', async () => {
|
|
@@ -774,5 +922,308 @@ describe('unbundleRecords', () => {
|
|
|
774
922
|
expect(stats.totalItems).toBe(0);
|
|
775
923
|
});
|
|
776
924
|
});
|
|
925
|
+
describe('GZIP comprehensive tests', () => {
|
|
926
|
+
describe('GZIP pre-decompression bomb protection', () => {
|
|
927
|
+
it('rejects GZIP with fake ISIZE claiming huge size BEFORE decompression starts', async () => {
|
|
928
|
+
const gzipHeader = Buffer.from([0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03]);
|
|
929
|
+
const emptyDeflate = Buffer.from([0x03, 0x00]);
|
|
930
|
+
const crc32 = Buffer.from([0x00, 0x00, 0x00, 0x00]);
|
|
931
|
+
const fakeIsize = Buffer.alloc(4);
|
|
932
|
+
fakeIsize.writeUInt32LE(100 * 1024 * 1024, 0);
|
|
933
|
+
const fakeGzip = Buffer.concat([gzipHeader, emptyDeflate, crc32, fakeIsize]);
|
|
934
|
+
const base64 = fakeGzip.toString('base64');
|
|
935
|
+
const bombEnvelope = {
|
|
936
|
+
v: 1,
|
|
937
|
+
c: sqs_bundled_client_types_1.CompressionAlgorithm.GZIP,
|
|
938
|
+
n: 1,
|
|
939
|
+
s: 100,
|
|
940
|
+
p: base64,
|
|
941
|
+
};
|
|
942
|
+
const record = makeSqsRecord({ messageBody: bombEnvelope }, 'msg-gzip-bomb');
|
|
943
|
+
const { failedMessageIds, stats } = await (0, sqs_unbundle_1.unbundleRecords)([record], {
|
|
944
|
+
maxDecompressedSizeBytes: 10 * 1024 * 1024,
|
|
945
|
+
});
|
|
946
|
+
expect(failedMessageIds).toContain('msg-gzip-bomb');
|
|
947
|
+
expect(stats.failedRecords).toBe(1);
|
|
948
|
+
});
|
|
949
|
+
it('allows GZIP with ISIZE within limit', async () => {
|
|
950
|
+
const items = [makeTestEvent('valid-1')];
|
|
951
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
952
|
+
const record = makeSqsRecord({ messageBody: envelope }, 'msg-gzip-valid');
|
|
953
|
+
const { items: result, failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record], {
|
|
954
|
+
maxDecompressedSizeBytes: 10 * 1024 * 1024,
|
|
955
|
+
});
|
|
956
|
+
expect(result).toHaveLength(1);
|
|
957
|
+
expect(failedMessageIds).toHaveLength(0);
|
|
958
|
+
});
|
|
959
|
+
it('pre-check catches bomb even when ISIZE is exactly at limit + 1', async () => {
|
|
960
|
+
const limit = 1000;
|
|
961
|
+
const gzipHeader = Buffer.from([0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03]);
|
|
962
|
+
const emptyDeflate = Buffer.from([0x03, 0x00]);
|
|
963
|
+
const crc32 = Buffer.from([0x00, 0x00, 0x00, 0x00]);
|
|
964
|
+
const fakeIsize = Buffer.alloc(4);
|
|
965
|
+
fakeIsize.writeUInt32LE(limit + 1, 0);
|
|
966
|
+
const fakeGzip = Buffer.concat([gzipHeader, emptyDeflate, crc32, fakeIsize]);
|
|
967
|
+
const base64 = fakeGzip.toString('base64');
|
|
968
|
+
const bombEnvelope = {
|
|
969
|
+
v: 1,
|
|
970
|
+
c: sqs_bundled_client_types_1.CompressionAlgorithm.GZIP,
|
|
971
|
+
n: 1,
|
|
972
|
+
s: 100,
|
|
973
|
+
p: base64,
|
|
974
|
+
};
|
|
975
|
+
const record = makeSqsRecord({ messageBody: bombEnvelope }, 'msg-boundary-bomb');
|
|
976
|
+
const { failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record], {
|
|
977
|
+
maxDecompressedSizeBytes: limit,
|
|
978
|
+
});
|
|
979
|
+
expect(failedMessageIds).toContain('msg-boundary-bomb');
|
|
980
|
+
});
|
|
981
|
+
});
|
|
982
|
+
describe('empty GZIP bundle', () => {
|
|
983
|
+
it('unbundles empty GZIP compressed array', async () => {
|
|
984
|
+
const envelope = await makeGzipCompressedEnvelope([]);
|
|
985
|
+
const record = makeSqsRecord({ messageBody: envelope }, 'msg-empty-gzip');
|
|
986
|
+
const { items, stats } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
987
|
+
expect(items).toHaveLength(0);
|
|
988
|
+
expect(stats.bundledSqsRecords).toBe(1);
|
|
989
|
+
expect(stats.totalItems).toBe(0);
|
|
990
|
+
});
|
|
991
|
+
});
|
|
992
|
+
describe('Unicode characters in GZIP payload', () => {
|
|
993
|
+
it('preserves emoji through GZIP compression', async () => {
|
|
994
|
+
const items = [
|
|
995
|
+
{ eventId: 'emoji-1', content: '🎉🚀💻🌍✨' },
|
|
996
|
+
{ eventId: 'emoji-2', content: '👋🏽👨👩👧👦🏳️🌈' },
|
|
997
|
+
];
|
|
998
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
999
|
+
const record = makeSqsRecord({ messageBody: envelope });
|
|
1000
|
+
const { items: result } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
1001
|
+
expect(result).toHaveLength(2);
|
|
1002
|
+
expect(result[0].content).toBe('🎉🚀💻🌍✨');
|
|
1003
|
+
expect(result[1].content).toBe('👋🏽👨👩👧👦🏳️🌈');
|
|
1004
|
+
});
|
|
1005
|
+
it('preserves CJK characters through GZIP compression', async () => {
|
|
1006
|
+
const items = [
|
|
1007
|
+
{ eventId: 'cjk-1', content: '中文测试:欢迎使用' },
|
|
1008
|
+
{ eventId: 'cjk-2', content: 'こんにちは世界 ひらがな カタカナ' },
|
|
1009
|
+
{ eventId: 'cjk-3', content: '안녕하세요 세계' },
|
|
1010
|
+
];
|
|
1011
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
1012
|
+
const record = makeSqsRecord({ messageBody: envelope });
|
|
1013
|
+
const { items: result } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
1014
|
+
expect(result).toHaveLength(3);
|
|
1015
|
+
expect(result[0].content).toBe('中文测试:欢迎使用');
|
|
1016
|
+
expect(result[1].content).toBe('こんにちは世界 ひらがな カタカナ');
|
|
1017
|
+
expect(result[2].content).toBe('안녕하세요 세계');
|
|
1018
|
+
});
|
|
1019
|
+
it('preserves special characters and mixed scripts through GZIP compression', async () => {
|
|
1020
|
+
const items = [
|
|
1021
|
+
{ eventId: 'special-1', content: '∑∫∂∇ε → ∞ × ∈ ⊆ ∀∃' },
|
|
1022
|
+
{ eventId: 'special-2', content: 'العربية: مرحبا بكم' },
|
|
1023
|
+
{ eventId: 'mixed', content: 'Hello 世界 🌏 مرحبا Привет' },
|
|
1024
|
+
{ eventId: 'combining', content: 'e\u0301 n\u0303 c\u0327' },
|
|
1025
|
+
];
|
|
1026
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
1027
|
+
const record = makeSqsRecord({ messageBody: envelope });
|
|
1028
|
+
const { items: result } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
1029
|
+
expect(result).toHaveLength(4);
|
|
1030
|
+
for (let i = 0; i < items.length; i++) {
|
|
1031
|
+
expect(result[i].content).toBe(items[i].content);
|
|
1032
|
+
const originalBytes = Buffer.from(items[i].content, 'utf8');
|
|
1033
|
+
const recoveredBytes = Buffer.from(result[i].content, 'utf8');
|
|
1034
|
+
expect(recoveredBytes.equals(originalBytes)).toBe(true);
|
|
1035
|
+
}
|
|
1036
|
+
});
|
|
1037
|
+
});
|
|
1038
|
+
describe('GZIP with invalid magic bytes', () => {
|
|
1039
|
+
it('fails gracefully for buffer with wrong magic bytes', async () => {
|
|
1040
|
+
const wrongMagic = Buffer.from([0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03]);
|
|
1041
|
+
const filler = Buffer.alloc(10);
|
|
1042
|
+
const invalidGzip = Buffer.concat([wrongMagic, filler]);
|
|
1043
|
+
const base64 = invalidGzip.toString('base64');
|
|
1044
|
+
const badEnvelope = {
|
|
1045
|
+
v: 1,
|
|
1046
|
+
c: sqs_bundled_client_types_1.CompressionAlgorithm.GZIP,
|
|
1047
|
+
n: 1,
|
|
1048
|
+
s: 100,
|
|
1049
|
+
p: base64,
|
|
1050
|
+
};
|
|
1051
|
+
const record = makeSqsRecord({ messageBody: badEnvelope }, 'msg-wrong-magic');
|
|
1052
|
+
const { failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
1053
|
+
expect(failedMessageIds).toContain('msg-wrong-magic');
|
|
1054
|
+
});
|
|
1055
|
+
it('fails gracefully for buffer starting with valid magic but corrupted data', async () => {
|
|
1056
|
+
const validMagic = Buffer.from([0x1f, 0x8b]);
|
|
1057
|
+
const garbage = Buffer.from('this is not valid gzip data after magic');
|
|
1058
|
+
const invalidGzip = Buffer.concat([validMagic, garbage]);
|
|
1059
|
+
const base64 = invalidGzip.toString('base64');
|
|
1060
|
+
const badEnvelope = {
|
|
1061
|
+
v: 1,
|
|
1062
|
+
c: sqs_bundled_client_types_1.CompressionAlgorithm.GZIP,
|
|
1063
|
+
n: 1,
|
|
1064
|
+
s: 100,
|
|
1065
|
+
p: base64,
|
|
1066
|
+
};
|
|
1067
|
+
const record = makeSqsRecord({ messageBody: badEnvelope }, 'msg-corrupt-gzip');
|
|
1068
|
+
const { failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
1069
|
+
expect(failedMessageIds).toContain('msg-corrupt-gzip');
|
|
1070
|
+
});
|
|
1071
|
+
});
|
|
1072
|
+
describe('GZIP minimum size edge cases', () => {
|
|
1073
|
+
it('handles buffer exactly 18 bytes (minimum valid)', async () => {
|
|
1074
|
+
const minBuffer = Buffer.alloc(18);
|
|
1075
|
+
minBuffer[0] = 0x1f;
|
|
1076
|
+
minBuffer[1] = 0x8b;
|
|
1077
|
+
const size = (0, compression_1.getGzipUncompressedSize)(minBuffer);
|
|
1078
|
+
expect(size).toBe(0);
|
|
1079
|
+
});
|
|
1080
|
+
it('returns undefined for buffer exactly 17 bytes (too small)', async () => {
|
|
1081
|
+
const tooSmall = Buffer.alloc(17);
|
|
1082
|
+
tooSmall[0] = 0x1f;
|
|
1083
|
+
tooSmall[1] = 0x8b;
|
|
1084
|
+
const size = (0, compression_1.getGzipUncompressedSize)(tooSmall);
|
|
1085
|
+
expect(size).toBeUndefined();
|
|
1086
|
+
});
|
|
1087
|
+
it('fails gracefully for 17-byte GZIP envelope', async () => {
|
|
1088
|
+
const tooSmall = Buffer.alloc(17);
|
|
1089
|
+
tooSmall[0] = 0x1f;
|
|
1090
|
+
tooSmall[1] = 0x8b;
|
|
1091
|
+
const base64 = tooSmall.toString('base64');
|
|
1092
|
+
const badEnvelope = {
|
|
1093
|
+
v: 1,
|
|
1094
|
+
c: sqs_bundled_client_types_1.CompressionAlgorithm.GZIP,
|
|
1095
|
+
n: 1,
|
|
1096
|
+
s: 100,
|
|
1097
|
+
p: base64,
|
|
1098
|
+
};
|
|
1099
|
+
const record = makeSqsRecord({ messageBody: badEnvelope }, 'msg-too-small');
|
|
1100
|
+
const { failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
1101
|
+
expect(failedMessageIds).toContain('msg-too-small');
|
|
1102
|
+
});
|
|
1103
|
+
});
|
|
1104
|
+
describe('GZIP timeout protection', () => {
|
|
1105
|
+
it('CompressionTimeoutError has correct properties', () => {
|
|
1106
|
+
const error = new compression_1.CompressionTimeoutError(compression_1.CompressionOperation.DECOMPRESS, 100);
|
|
1107
|
+
expect(error).toBeInstanceOf(Error);
|
|
1108
|
+
expect(error.name).toBe('CompressionTimeoutError');
|
|
1109
|
+
expect(error.operation).toBe('decompress');
|
|
1110
|
+
expect(error.timeoutMs).toBe(100);
|
|
1111
|
+
expect(error.message).toContain('timed out after 100ms');
|
|
1112
|
+
});
|
|
1113
|
+
it('GZIP decompression respects timeout option', async () => {
|
|
1114
|
+
const items = [makeTestEvent('timeout-test')];
|
|
1115
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
1116
|
+
const record = makeSqsRecord({ messageBody: envelope });
|
|
1117
|
+
const { items: result, failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record], {
|
|
1118
|
+
timeoutMs: 10000,
|
|
1119
|
+
});
|
|
1120
|
+
expect(result).toHaveLength(1);
|
|
1121
|
+
expect(failedMessageIds).toHaveLength(0);
|
|
1122
|
+
});
|
|
1123
|
+
});
|
|
1124
|
+
describe('large GZIP near SQS limit', () => {
|
|
1125
|
+
it('decompresses large GZIP bundle near 256KB compressed', async () => {
|
|
1126
|
+
const items = Array.from({ length: 200 }, (_, i) => ({
|
|
1127
|
+
eventId: `large-${i}`,
|
|
1128
|
+
name: 'page_view',
|
|
1129
|
+
data: `repeated-content-for-compression-${i}-`.repeat(50),
|
|
1130
|
+
value: i * 1.5,
|
|
1131
|
+
}));
|
|
1132
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
1133
|
+
const compressedSize = Buffer.byteLength(JSON.stringify(envelope.p), 'utf8');
|
|
1134
|
+
console.log(`Large GZIP test: compressed payload ~${Math.round(compressedSize / 1024)}KB`);
|
|
1135
|
+
const record = makeSqsRecord({ messageBody: envelope }, 'msg-large-gzip');
|
|
1136
|
+
const { items: result, failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record]);
|
|
1137
|
+
expect(failedMessageIds).toHaveLength(0);
|
|
1138
|
+
expect(result).toHaveLength(200);
|
|
1139
|
+
expect(result[0].eventId).toBe('large-0');
|
|
1140
|
+
expect(result[199].eventId).toBe('large-199');
|
|
1141
|
+
});
|
|
1142
|
+
it('handles GZIP with maximum decompressed size exactly at limit', async () => {
|
|
1143
|
+
const items = [{ eventId: 'boundary', data: 'x'.repeat(5000) }];
|
|
1144
|
+
const envelope = await makeGzipCompressedEnvelope(items);
|
|
1145
|
+
const record = makeSqsRecord({ messageBody: envelope });
|
|
1146
|
+
const { items: result, failedMessageIds } = await (0, sqs_unbundle_1.unbundleRecords)([record], {
|
|
1147
|
+
maxDecompressedSizeBytes: 10 * 1024,
|
|
1148
|
+
});
|
|
1149
|
+
expect(failedMessageIds).toHaveLength(0);
|
|
1150
|
+
expect(result).toHaveLength(1);
|
|
1151
|
+
});
|
|
1152
|
+
});
|
|
1153
|
+
});
|
|
1154
|
+
});
|
|
1155
|
+
describe('getGzipUncompressedSize', () => {
|
|
1156
|
+
it('returns correct size for valid GZIP', async () => {
|
|
1157
|
+
const originalData = 'Hello, World! This is a test.';
|
|
1158
|
+
const compressed = await gzipAsync(Buffer.from(originalData, 'utf8'));
|
|
1159
|
+
const size = (0, compression_1.getGzipUncompressedSize)(compressed);
|
|
1160
|
+
expect(size).toBe(Buffer.byteLength(originalData, 'utf8'));
|
|
1161
|
+
});
|
|
1162
|
+
it('returns correct size for larger payload', async () => {
|
|
1163
|
+
const originalData = 'x'.repeat(10000);
|
|
1164
|
+
const compressed = await gzipAsync(Buffer.from(originalData, 'utf8'));
|
|
1165
|
+
const size = (0, compression_1.getGzipUncompressedSize)(compressed);
|
|
1166
|
+
expect(size).toBe(10000);
|
|
1167
|
+
});
|
|
1168
|
+
it('returns undefined for buffer too small', () => {
|
|
1169
|
+
const tooSmall = Buffer.alloc(17);
|
|
1170
|
+
tooSmall[0] = 0x1f;
|
|
1171
|
+
tooSmall[1] = 0x8b;
|
|
1172
|
+
expect((0, compression_1.getGzipUncompressedSize)(tooSmall)).toBeUndefined();
|
|
1173
|
+
});
|
|
1174
|
+
it('returns undefined for wrong magic bytes', () => {
|
|
1175
|
+
const wrongMagic = Buffer.alloc(20);
|
|
1176
|
+
wrongMagic[0] = 0x00;
|
|
1177
|
+
wrongMagic[1] = 0x00;
|
|
1178
|
+
expect((0, compression_1.getGzipUncompressedSize)(wrongMagic)).toBeUndefined();
|
|
1179
|
+
});
|
|
1180
|
+
it('returns undefined for empty buffer', () => {
|
|
1181
|
+
expect((0, compression_1.getGzipUncompressedSize)(Buffer.alloc(0))).toBeUndefined();
|
|
1182
|
+
});
|
|
1183
|
+
it('returns undefined for single byte buffer', () => {
|
|
1184
|
+
expect((0, compression_1.getGzipUncompressedSize)(Buffer.from([0x1f]))).toBeUndefined();
|
|
1185
|
+
});
|
|
1186
|
+
it('returns undefined for buffer with only magic bytes', () => {
|
|
1187
|
+
expect((0, compression_1.getGzipUncompressedSize)(Buffer.from([0x1f, 0x8b]))).toBeUndefined();
|
|
1188
|
+
});
|
|
1189
|
+
it('returns 0 for GZIP with ISIZE = 0', () => {
|
|
1190
|
+
const buffer = Buffer.alloc(18);
|
|
1191
|
+
buffer[0] = 0x1f;
|
|
1192
|
+
buffer[1] = 0x8b;
|
|
1193
|
+
const size = (0, compression_1.getGzipUncompressedSize)(buffer);
|
|
1194
|
+
expect(size).toBe(0);
|
|
1195
|
+
});
|
|
1196
|
+
it('reads ISIZE correctly as little-endian', async () => {
|
|
1197
|
+
const buffer = Buffer.alloc(18);
|
|
1198
|
+
buffer[0] = 0x1f;
|
|
1199
|
+
buffer[1] = 0x8b;
|
|
1200
|
+
buffer.writeUInt32LE(0x12345678, 14);
|
|
1201
|
+
const size = (0, compression_1.getGzipUncompressedSize)(buffer);
|
|
1202
|
+
expect(size).toBe(0x12345678);
|
|
1203
|
+
});
|
|
1204
|
+
it('handles maximum 32-bit ISIZE value', () => {
|
|
1205
|
+
const buffer = Buffer.alloc(18);
|
|
1206
|
+
buffer[0] = 0x1f;
|
|
1207
|
+
buffer[1] = 0x8b;
|
|
1208
|
+
buffer.writeUInt32LE(0xffffffff, 14);
|
|
1209
|
+
const size = (0, compression_1.getGzipUncompressedSize)(buffer);
|
|
1210
|
+
expect(size).toBe(0xffffffff);
|
|
1211
|
+
});
|
|
1212
|
+
it('correctly extracts size from real GZIP data', async () => {
|
|
1213
|
+
const testSizes = [1, 10, 100, 1000, 50000];
|
|
1214
|
+
for (const testSize of testSizes) {
|
|
1215
|
+
const originalData = 'a'.repeat(testSize);
|
|
1216
|
+
const compressed = await gzipAsync(Buffer.from(originalData, 'utf8'));
|
|
1217
|
+
const size = (0, compression_1.getGzipUncompressedSize)(compressed);
|
|
1218
|
+
expect(size).toBe(testSize);
|
|
1219
|
+
}
|
|
1220
|
+
});
|
|
1221
|
+
it('handles JSON array compression correctly', async () => {
|
|
1222
|
+
const items = [{ id: 1 }, { id: 2 }, { id: 3 }];
|
|
1223
|
+
const json = JSON.stringify(items);
|
|
1224
|
+
const compressed = await gzipAsync(Buffer.from(json, 'utf8'));
|
|
1225
|
+
const size = (0, compression_1.getGzipUncompressedSize)(compressed);
|
|
1226
|
+
expect(size).toBe(Buffer.byteLength(json, 'utf8'));
|
|
1227
|
+
});
|
|
777
1228
|
});
|
|
778
1229
|
//# sourceMappingURL=sqs-unbundle.spec.js.map
|