@adtrackify/at-service-common 3.18.0 → 3.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/__tests__/clients/sqs-bundled-client.spec.js +684 -0
- package/dist/cjs/__tests__/clients/sqs-bundled-client.spec.js.map +1 -1
- package/dist/cjs/__tests__/clients/sqs-unbundle.spec.js +411 -0
- package/dist/cjs/__tests__/clients/sqs-unbundle.spec.js.map +1 -1
- package/dist/cjs/__tests__/integration/sqs-bundling-roundtrip.spec.d.ts +1 -0
- package/dist/cjs/__tests__/integration/sqs-bundling-roundtrip.spec.js +582 -0
- package/dist/cjs/__tests__/integration/sqs-bundling-roundtrip.spec.js.map +1 -0
- package/dist/cjs/clients/generic/sqs-bundled-client.js +56 -18
- package/dist/cjs/clients/generic/sqs-bundled-client.js.map +1 -1
- package/dist/cjs/clients/generic/sqs-bundled-client.types.d.ts +10 -1
- package/dist/cjs/utils/compression.d.ts +1 -1
- package/dist/cjs/utils/compression.js +1 -1
- package/dist/cjs/utils/compression.js.map +1 -1
- package/dist/esm/__tests__/clients/sqs-bundled-client.spec.js +684 -0
- package/dist/esm/__tests__/clients/sqs-bundled-client.spec.js.map +1 -1
- package/dist/esm/__tests__/clients/sqs-unbundle.spec.js +412 -1
- package/dist/esm/__tests__/clients/sqs-unbundle.spec.js.map +1 -1
- package/dist/esm/__tests__/integration/sqs-bundling-roundtrip.spec.d.ts +1 -0
- package/dist/esm/__tests__/integration/sqs-bundling-roundtrip.spec.js +580 -0
- package/dist/esm/__tests__/integration/sqs-bundling-roundtrip.spec.js.map +1 -0
- package/dist/esm/clients/generic/sqs-bundled-client.js +56 -18
- package/dist/esm/clients/generic/sqs-bundled-client.js.map +1 -1
- package/dist/esm/clients/generic/sqs-bundled-client.types.d.ts +10 -1
- package/dist/esm/utils/compression.d.ts +1 -1
- package/dist/esm/utils/compression.js +2 -2
- package/dist/esm/utils/compression.js.map +1 -1
- package/package.json +2 -2
|
@@ -33,6 +33,12 @@ jest.mock('../../helpers/logging-helper', () => ({
|
|
|
33
33
|
error: jest.fn(),
|
|
34
34
|
},
|
|
35
35
|
}));
|
|
36
|
+
const actualSizeModule = jest.requireActual('../../utils/size');
|
|
37
|
+
const mockSizeInBytes = jest.fn(actualSizeModule.sizeInBytes);
|
|
38
|
+
jest.mock('../../utils/size', () => ({
|
|
39
|
+
...jest.requireActual('../../utils/size'),
|
|
40
|
+
sizeInBytes: (obj) => mockSizeInBytes(obj),
|
|
41
|
+
}));
|
|
36
42
|
function makeTestEvent(id, name = 'test', value) {
|
|
37
43
|
return { id, name, value };
|
|
38
44
|
}
|
|
@@ -166,6 +172,83 @@ describe('BundledSQSClient', () => {
|
|
|
166
172
|
expect(client.config.autoResize).toBe(false);
|
|
167
173
|
});
|
|
168
174
|
});
|
|
175
|
+
describe('skippedItems behavior', () => {
|
|
176
|
+
afterEach(() => {
|
|
177
|
+
mockSizeInBytes.mockImplementation(actualSizeModule.sizeInBytes);
|
|
178
|
+
});
|
|
179
|
+
it('reports skippedItems as 0 when all items sent successfully', async () => {
|
|
180
|
+
const items = [makeTestEvent('1'), makeTestEvent('2')];
|
|
181
|
+
const result = await bundledClient.sendBundled('test-message', items);
|
|
182
|
+
expect(result.skippedItems).toBe(0);
|
|
183
|
+
expect(result.totalItems).toBe(2);
|
|
184
|
+
expect(result.failedBundles).toBe(0);
|
|
185
|
+
});
|
|
186
|
+
it('tracks skippedItems when autoResize disabled and bundle exceeds limit', async () => {
|
|
187
|
+
const noResizeClient = new sqs_bundled_client_1.BundledSQSClient(mockSqsClient, {
|
|
188
|
+
compression: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
189
|
+
autoResize: false,
|
|
190
|
+
});
|
|
191
|
+
const items = [makeTestEvent('1'), makeTestEvent('2'), makeTestEvent('3')];
|
|
192
|
+
mockSizeInBytes.mockImplementation((obj) => {
|
|
193
|
+
if (typeof obj === 'object' && obj !== null && 'v' in obj) {
|
|
194
|
+
return 2000000;
|
|
195
|
+
}
|
|
196
|
+
return 100;
|
|
197
|
+
});
|
|
198
|
+
const result = await noResizeClient.sendBundled('test-message', items);
|
|
199
|
+
expect(result.skippedItems).toBe(3);
|
|
200
|
+
expect(result.totalItems).toBe(3);
|
|
201
|
+
expect(mockSqsClient.buildAndSendMessagesV2).not.toHaveBeenCalled();
|
|
202
|
+
});
|
|
203
|
+
it('tracks skippedItems when single item cannot fit (splitAndRetry terminates)', async () => {
|
|
204
|
+
const client = new sqs_bundled_client_1.BundledSQSClient(mockSqsClient, {
|
|
205
|
+
compression: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
206
|
+
autoResize: true,
|
|
207
|
+
});
|
|
208
|
+
const items = [makeTestEvent('oversized-item')];
|
|
209
|
+
mockSizeInBytes.mockImplementation((obj) => {
|
|
210
|
+
if (typeof obj === 'object' && obj !== null && 'v' in obj) {
|
|
211
|
+
return 2000000;
|
|
212
|
+
}
|
|
213
|
+
return 100;
|
|
214
|
+
});
|
|
215
|
+
const result = await client.sendBundled('test-message', items);
|
|
216
|
+
expect(result.skippedItems).toBe(1);
|
|
217
|
+
expect(result.totalItems).toBe(1);
|
|
218
|
+
});
|
|
219
|
+
it('reports partial skippedItems when some items are too large', async () => {
|
|
220
|
+
const client = new sqs_bundled_client_1.BundledSQSClient(mockSqsClient, {
|
|
221
|
+
compression: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
222
|
+
autoResize: true,
|
|
223
|
+
maxItemsPerBundle: 1,
|
|
224
|
+
});
|
|
225
|
+
const items = [
|
|
226
|
+
makeTestEvent('small-1'),
|
|
227
|
+
makeTestEvent('oversized'),
|
|
228
|
+
makeTestEvent('small-2'),
|
|
229
|
+
];
|
|
230
|
+
let envelopeValidationCount = 0;
|
|
231
|
+
mockSizeInBytes.mockImplementation((obj) => {
|
|
232
|
+
if (typeof obj === 'object' && obj !== null && 'v' in obj) {
|
|
233
|
+
envelopeValidationCount++;
|
|
234
|
+
if (envelopeValidationCount === 2) {
|
|
235
|
+
return 2000000;
|
|
236
|
+
}
|
|
237
|
+
return 500;
|
|
238
|
+
}
|
|
239
|
+
return 100;
|
|
240
|
+
});
|
|
241
|
+
const result = await client.sendBundled('test-message', items);
|
|
242
|
+
expect(result.skippedItems).toBe(1);
|
|
243
|
+
expect(result.totalItems).toBe(3);
|
|
244
|
+
expect(mockSqsClient.buildAndSendMessagesV2).toHaveBeenCalled();
|
|
245
|
+
});
|
|
246
|
+
it('emptyResult returns skippedItems as 0', async () => {
|
|
247
|
+
const result = await bundledClient.sendBundled('test-message', []);
|
|
248
|
+
expect(result.skippedItems).toBe(0);
|
|
249
|
+
expect(result.totalItems).toBe(0);
|
|
250
|
+
});
|
|
251
|
+
});
|
|
169
252
|
describe('integration: end-to-end bundle flow', () => {
|
|
170
253
|
it('produces envelopes that unbundleRecords can process', async () => {
|
|
171
254
|
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
@@ -221,5 +304,606 @@ describe('BundledSQSClient', () => {
|
|
|
221
304
|
expect(unbundledItems[199].id).toBe('event-199');
|
|
222
305
|
});
|
|
223
306
|
});
|
|
307
|
+
describe('producer edge cases - real sequences', () => {
|
|
308
|
+
it('handles 1000 items with real bundling and verifies envelope structure', async () => {
|
|
309
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
310
|
+
const items = Array.from({ length: 1000 }, (_, i) => ({
|
|
311
|
+
id: `item-${i}`,
|
|
312
|
+
name: `payload-${i}`,
|
|
313
|
+
value: i * 10,
|
|
314
|
+
data: `extra-data-${i}`,
|
|
315
|
+
}));
|
|
316
|
+
const capturedEnvelopes = [];
|
|
317
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
318
|
+
capturedEnvelopes.push(...envelopes);
|
|
319
|
+
return {
|
|
320
|
+
successCount: envelopes.length,
|
|
321
|
+
failedCount: 0,
|
|
322
|
+
batchCount: 1,
|
|
323
|
+
failedMessages: [],
|
|
324
|
+
};
|
|
325
|
+
});
|
|
326
|
+
const result = await bundledClient.sendBundled('test', items);
|
|
327
|
+
expect(result.totalItems).toBe(1000);
|
|
328
|
+
expect(capturedEnvelopes.length).toBeGreaterThan(0);
|
|
329
|
+
let totalItemsInEnvelopes = 0;
|
|
330
|
+
for (const env of capturedEnvelopes) {
|
|
331
|
+
expect(env.v).toBe(1);
|
|
332
|
+
expect(env.c).toBe(sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD);
|
|
333
|
+
expect(typeof env.p).toBe('string');
|
|
334
|
+
expect(typeof env.n).toBe('number');
|
|
335
|
+
expect(env.n).toBeGreaterThan(0);
|
|
336
|
+
totalItemsInEnvelopes += env.n;
|
|
337
|
+
}
|
|
338
|
+
expect(totalItemsInEnvelopes).toBe(1000);
|
|
339
|
+
const sqsRecords = capturedEnvelopes.map((env, i) => ({
|
|
340
|
+
messageId: `msg-${i}`,
|
|
341
|
+
body: JSON.stringify({ messageBody: env }),
|
|
342
|
+
}));
|
|
343
|
+
const { items: unbundledItems } = await unbundleRecords(sqsRecords);
|
|
344
|
+
expect(unbundledItems).toHaveLength(1000);
|
|
345
|
+
expect(unbundledItems[0]).toEqual({ id: 'item-0', name: 'payload-0', value: 0, data: 'extra-data-0' });
|
|
346
|
+
expect(unbundledItems[999]).toEqual({ id: 'item-999', name: 'payload-999', value: 9990, data: 'extra-data-999' });
|
|
347
|
+
});
|
|
348
|
+
it('achieves compression effectiveness with repetitive data', async () => {
|
|
349
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
350
|
+
const repetitiveText = 'AAAAAAAAAA'.repeat(100);
|
|
351
|
+
const items = Array.from({ length: 100 }, (_, i) => ({
|
|
352
|
+
id: `item-${i}`,
|
|
353
|
+
name: 'repetitive-test',
|
|
354
|
+
value: i,
|
|
355
|
+
data: repetitiveText,
|
|
356
|
+
}));
|
|
357
|
+
const capturedEnvelopes = [];
|
|
358
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
359
|
+
capturedEnvelopes.push(envelopes[0]);
|
|
360
|
+
return {
|
|
361
|
+
successCount: envelopes.length,
|
|
362
|
+
failedCount: 0,
|
|
363
|
+
batchCount: 1,
|
|
364
|
+
failedMessages: [],
|
|
365
|
+
};
|
|
366
|
+
});
|
|
367
|
+
const result = await bundledClient.sendBundled('test', items);
|
|
368
|
+
expect(result.metrics.compressionRatio).toBeGreaterThan(1);
|
|
369
|
+
expect(result.metrics.finalSizeBytes).toBeLessThan(result.metrics.originalSizeBytes);
|
|
370
|
+
expect(result.metrics.compressionRatio).toBeGreaterThan(5);
|
|
371
|
+
expect(capturedEnvelopes).toHaveLength(1);
|
|
372
|
+
const envelope = capturedEnvelopes[0];
|
|
373
|
+
expect(typeof envelope.p).toBe('string');
|
|
374
|
+
const { items: unbundledItems } = await unbundleRecords([{
|
|
375
|
+
messageId: 'msg-1',
|
|
376
|
+
body: JSON.stringify({ messageBody: envelope }),
|
|
377
|
+
}]);
|
|
378
|
+
expect(unbundledItems).toHaveLength(100);
|
|
379
|
+
expect(unbundledItems[0].data).toBe(repetitiveText);
|
|
380
|
+
expect(unbundledItems[99].data).toBe(repetitiveText);
|
|
381
|
+
});
|
|
382
|
+
it('handles various item sizes correctly', async () => {
|
|
383
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
384
|
+
const tinyItems = Array.from({ length: 10 }, (_, i) => ({
|
|
385
|
+
id: `tiny-${i}`,
|
|
386
|
+
name: 't',
|
|
387
|
+
}));
|
|
388
|
+
const smallItems = Array.from({ length: 10 }, (_, i) => ({
|
|
389
|
+
id: `small-${i}`,
|
|
390
|
+
name: 'small-item',
|
|
391
|
+
value: i,
|
|
392
|
+
data: 'x'.repeat(100),
|
|
393
|
+
}));
|
|
394
|
+
const mediumItems = Array.from({ length: 10 }, (_, i) => ({
|
|
395
|
+
id: `medium-${i}`,
|
|
396
|
+
name: 'medium-item',
|
|
397
|
+
value: i,
|
|
398
|
+
data: 'y'.repeat(10000),
|
|
399
|
+
}));
|
|
400
|
+
const largerItems = Array.from({ length: 5 }, (_, i) => ({
|
|
401
|
+
id: `larger-${i}`,
|
|
402
|
+
name: 'larger-item',
|
|
403
|
+
value: i,
|
|
404
|
+
data: 'z'.repeat(100000),
|
|
405
|
+
}));
|
|
406
|
+
const allItems = [...tinyItems, ...smallItems, ...mediumItems, ...largerItems];
|
|
407
|
+
const totalExpectedItems = allItems.length;
|
|
408
|
+
const capturedEnvelopes = [];
|
|
409
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
410
|
+
capturedEnvelopes.push(...envelopes);
|
|
411
|
+
return {
|
|
412
|
+
successCount: envelopes.length,
|
|
413
|
+
failedCount: 0,
|
|
414
|
+
batchCount: 1,
|
|
415
|
+
failedMessages: [],
|
|
416
|
+
};
|
|
417
|
+
});
|
|
418
|
+
const result = await bundledClient.sendBundled('test', allItems);
|
|
419
|
+
expect(result.totalItems).toBe(totalExpectedItems);
|
|
420
|
+
expect(result.skippedItems).toBe(0);
|
|
421
|
+
const totalInEnvelopes = capturedEnvelopes.reduce((sum, env) => sum + env.n, 0);
|
|
422
|
+
expect(totalInEnvelopes).toBe(totalExpectedItems);
|
|
423
|
+
const sqsRecords = capturedEnvelopes.map((env, i) => ({
|
|
424
|
+
messageId: `msg-${i}`,
|
|
425
|
+
body: JSON.stringify({ messageBody: env }),
|
|
426
|
+
}));
|
|
427
|
+
const { items: unbundledItems } = await unbundleRecords(sqsRecords);
|
|
428
|
+
expect(unbundledItems).toHaveLength(totalExpectedItems);
|
|
429
|
+
const unbundledTiny = unbundledItems.filter(item => item.id.startsWith('tiny-'));
|
|
430
|
+
const unbundledSmall = unbundledItems.filter(item => item.id.startsWith('small-'));
|
|
431
|
+
const unbundledMedium = unbundledItems.filter(item => item.id.startsWith('medium-'));
|
|
432
|
+
const unbundledLarger = unbundledItems.filter(item => item.id.startsWith('larger-'));
|
|
433
|
+
expect(unbundledTiny).toHaveLength(10);
|
|
434
|
+
expect(unbundledSmall).toHaveLength(10);
|
|
435
|
+
expect(unbundledMedium).toHaveLength(10);
|
|
436
|
+
expect(unbundledLarger).toHaveLength(5);
|
|
437
|
+
expect(unbundledSmall[0].data).toBe('x'.repeat(100));
|
|
438
|
+
expect(unbundledMedium[0].data).toBe('y'.repeat(10000));
|
|
439
|
+
expect(unbundledLarger[0].data).toBe('z'.repeat(100000));
|
|
440
|
+
});
|
|
441
|
+
it('preserves unicode and special characters through compression', async () => {
|
|
442
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
443
|
+
const unicodeItems = [
|
|
444
|
+
{
|
|
445
|
+
id: 'emoji',
|
|
446
|
+
name: '🎉🚀💯🔥✨',
|
|
447
|
+
data: 'Celebration 🎊 with many emojis 🌟⭐💫',
|
|
448
|
+
},
|
|
449
|
+
{
|
|
450
|
+
id: 'chinese',
|
|
451
|
+
name: '你好世界',
|
|
452
|
+
data: '这是一个测试消息,包含中文字符。北京、上海、广州。',
|
|
453
|
+
},
|
|
454
|
+
{
|
|
455
|
+
id: 'arabic',
|
|
456
|
+
name: 'مرحبا بالعالم',
|
|
457
|
+
data: 'هذه رسالة اختبار باللغة العربية',
|
|
458
|
+
},
|
|
459
|
+
{
|
|
460
|
+
id: 'japanese',
|
|
461
|
+
name: 'こんにちは世界',
|
|
462
|
+
data: 'テスト メッセージです。東京、大阪、京都。',
|
|
463
|
+
},
|
|
464
|
+
{
|
|
465
|
+
id: 'korean',
|
|
466
|
+
name: '안녕하세요 세계',
|
|
467
|
+
data: '테스트 메시지입니다. 서울, 부산, 인천.',
|
|
468
|
+
},
|
|
469
|
+
{
|
|
470
|
+
id: 'special-symbols',
|
|
471
|
+
name: '†‡§¶•‰™©®',
|
|
472
|
+
data: 'Special: «»‹› ′″ ∞∑∏√∫ ≤≥≠≈ αβγδε',
|
|
473
|
+
},
|
|
474
|
+
{
|
|
475
|
+
id: 'mixed-unicode',
|
|
476
|
+
name: 'Mixed: Ñ ü ö ä ß',
|
|
477
|
+
data: 'Ümlauts: äöüÄÖÜß — dashes—and–more Ç ñ ¡¿',
|
|
478
|
+
},
|
|
479
|
+
{
|
|
480
|
+
id: 'mathematical',
|
|
481
|
+
name: '∀x∈ℝ: x² ≥ 0',
|
|
482
|
+
data: 'Math: ∫₀^∞ e^(-x²) dx = √π/2, ∑_{n=1}^{∞} 1/n² = π²/6',
|
|
483
|
+
},
|
|
484
|
+
{
|
|
485
|
+
id: 'currency',
|
|
486
|
+
name: '€£¥₹₽¢',
|
|
487
|
+
data: 'Prices: $100, €85, £70, ¥10000, ₹8000, ₽7500',
|
|
488
|
+
},
|
|
489
|
+
{
|
|
490
|
+
id: 'newlines-tabs',
|
|
491
|
+
name: 'control\tchars',
|
|
492
|
+
data: 'Line 1\nLine 2\rLine 3\r\nLine 4\tTabbed',
|
|
493
|
+
},
|
|
494
|
+
];
|
|
495
|
+
let capturedEnvelope = null;
|
|
496
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
497
|
+
capturedEnvelope = envelopes[0];
|
|
498
|
+
return {
|
|
499
|
+
successCount: envelopes.length,
|
|
500
|
+
failedCount: 0,
|
|
501
|
+
batchCount: 1,
|
|
502
|
+
failedMessages: [],
|
|
503
|
+
};
|
|
504
|
+
});
|
|
505
|
+
const result = await bundledClient.sendBundled('test', unicodeItems);
|
|
506
|
+
expect(result.totalItems).toBe(unicodeItems.length);
|
|
507
|
+
expect(result.skippedItems).toBe(0);
|
|
508
|
+
expect(capturedEnvelope).not.toBeNull();
|
|
509
|
+
const { items: unbundledItems } = await unbundleRecords([{
|
|
510
|
+
messageId: 'msg-1',
|
|
511
|
+
body: JSON.stringify({ messageBody: capturedEnvelope }),
|
|
512
|
+
}]);
|
|
513
|
+
expect(unbundledItems).toHaveLength(unicodeItems.length);
|
|
514
|
+
for (let i = 0; i < unicodeItems.length; i++) {
|
|
515
|
+
expect(unbundledItems[i]).toEqual(unicodeItems[i]);
|
|
516
|
+
}
|
|
517
|
+
const emojiItem = unbundledItems.find(item => item.id === 'emoji');
|
|
518
|
+
expect(emojiItem?.name).toBe('🎉🚀💯🔥✨');
|
|
519
|
+
const chineseItem = unbundledItems.find(item => item.id === 'chinese');
|
|
520
|
+
expect(chineseItem?.name).toBe('你好世界');
|
|
521
|
+
const arabicItem = unbundledItems.find(item => item.id === 'arabic');
|
|
522
|
+
expect(arabicItem?.name).toBe('مرحبا بالعالم');
|
|
523
|
+
});
|
|
524
|
+
it('preserves deeply nested object structures', async () => {
|
|
525
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
526
|
+
const nestedItems = [
|
|
527
|
+
{
|
|
528
|
+
id: 'deep-nested',
|
|
529
|
+
name: 'deep',
|
|
530
|
+
nested: {
|
|
531
|
+
level1: {
|
|
532
|
+
level2: {
|
|
533
|
+
level3: {
|
|
534
|
+
level4: {
|
|
535
|
+
level5: {
|
|
536
|
+
level6: {
|
|
537
|
+
value: 'deep-value',
|
|
538
|
+
number: 12345,
|
|
539
|
+
boolean: true,
|
|
540
|
+
},
|
|
541
|
+
},
|
|
542
|
+
},
|
|
543
|
+
},
|
|
544
|
+
},
|
|
545
|
+
},
|
|
546
|
+
},
|
|
547
|
+
},
|
|
548
|
+
{
|
|
549
|
+
id: 'array-of-objects',
|
|
550
|
+
name: 'arrays',
|
|
551
|
+
items: [
|
|
552
|
+
{ subId: 'sub-1', data: { nested: { value: 'a' } } },
|
|
553
|
+
{ subId: 'sub-2', data: { nested: { value: 'b' } } },
|
|
554
|
+
{ subId: 'sub-3', data: { nested: { value: 'c' } } },
|
|
555
|
+
],
|
|
556
|
+
},
|
|
557
|
+
{
|
|
558
|
+
id: 'mixed-structures',
|
|
559
|
+
name: 'mixed',
|
|
560
|
+
config: {
|
|
561
|
+
settings: {
|
|
562
|
+
features: ['feature-1', 'feature-2', 'feature-3'],
|
|
563
|
+
metadata: {
|
|
564
|
+
tags: {
|
|
565
|
+
primary: ['tag-a', 'tag-b'],
|
|
566
|
+
secondary: {
|
|
567
|
+
group1: ['tag-c', 'tag-d'],
|
|
568
|
+
group2: ['tag-e', 'tag-f'],
|
|
569
|
+
},
|
|
570
|
+
},
|
|
571
|
+
},
|
|
572
|
+
},
|
|
573
|
+
},
|
|
574
|
+
},
|
|
575
|
+
{
|
|
576
|
+
id: 'null-undefined-handling',
|
|
577
|
+
name: 'nulls',
|
|
578
|
+
value: null,
|
|
579
|
+
nested: {
|
|
580
|
+
nullField: null,
|
|
581
|
+
emptyObject: {},
|
|
582
|
+
emptyArray: [],
|
|
583
|
+
zeroValue: 0,
|
|
584
|
+
falseValue: false,
|
|
585
|
+
emptyString: '',
|
|
586
|
+
},
|
|
587
|
+
},
|
|
588
|
+
{
|
|
589
|
+
id: 'large-array',
|
|
590
|
+
name: 'large-array',
|
|
591
|
+
items: Array.from({ length: 100 }, (_, i) => ({
|
|
592
|
+
index: i,
|
|
593
|
+
data: `item-${i}`,
|
|
594
|
+
nested: { level: 1, value: i * 2 },
|
|
595
|
+
})),
|
|
596
|
+
},
|
|
597
|
+
];
|
|
598
|
+
let capturedEnvelope = null;
|
|
599
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
600
|
+
capturedEnvelope = envelopes[0];
|
|
601
|
+
return {
|
|
602
|
+
successCount: envelopes.length,
|
|
603
|
+
failedCount: 0,
|
|
604
|
+
batchCount: 1,
|
|
605
|
+
failedMessages: [],
|
|
606
|
+
};
|
|
607
|
+
});
|
|
608
|
+
const genericClient = new sqs_bundled_client_1.BundledSQSClient(mockSqsClient, {
|
|
609
|
+
compression: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
610
|
+
});
|
|
611
|
+
const result = await genericClient.sendBundled('test', nestedItems);
|
|
612
|
+
expect(result.totalItems).toBe(nestedItems.length);
|
|
613
|
+
expect(result.skippedItems).toBe(0);
|
|
614
|
+
const { items: unbundledItems } = await unbundleRecords([{
|
|
615
|
+
messageId: 'msg-1',
|
|
616
|
+
body: JSON.stringify({ messageBody: capturedEnvelope }),
|
|
617
|
+
}]);
|
|
618
|
+
expect(unbundledItems).toHaveLength(nestedItems.length);
|
|
619
|
+
const deepItem = unbundledItems.find(item => item.id === 'deep-nested');
|
|
620
|
+
expect(deepItem?.nested?.level1?.level2?.level3?.level4?.level5?.level6?.value).toBe('deep-value');
|
|
621
|
+
expect(deepItem?.nested?.level1?.level2?.level3?.level4?.level5?.level6?.number).toBe(12345);
|
|
622
|
+
expect(deepItem?.nested?.level1?.level2?.level3?.level4?.level5?.level6?.boolean).toBe(true);
|
|
623
|
+
const arrayItem = unbundledItems.find(item => item.id === 'array-of-objects');
|
|
624
|
+
expect(arrayItem?.items?.length).toBe(3);
|
|
625
|
+
expect(arrayItem?.items?.[0]?.data?.nested?.value).toBe('a');
|
|
626
|
+
expect(arrayItem?.items?.[2]?.data?.nested?.value).toBe('c');
|
|
627
|
+
const mixedItem = unbundledItems.find(item => item.id === 'mixed-structures');
|
|
628
|
+
expect(mixedItem?.config?.settings?.features).toEqual(['feature-1', 'feature-2', 'feature-3']);
|
|
629
|
+
expect(mixedItem?.config?.settings?.metadata?.tags?.secondary?.group2).toEqual(['tag-e', 'tag-f']);
|
|
630
|
+
const nullItem = unbundledItems.find(item => item.id === 'null-undefined-handling');
|
|
631
|
+
expect(nullItem?.value).toBeNull();
|
|
632
|
+
expect(nullItem?.nested?.nullField).toBeNull();
|
|
633
|
+
expect(nullItem?.nested?.emptyObject).toEqual({});
|
|
634
|
+
expect(nullItem?.nested?.emptyArray).toEqual([]);
|
|
635
|
+
expect(nullItem?.nested?.zeroValue).toBe(0);
|
|
636
|
+
expect(nullItem?.nested?.falseValue).toBe(false);
|
|
637
|
+
expect(nullItem?.nested?.emptyString).toBe('');
|
|
638
|
+
const largeArrayItem = unbundledItems.find(item => item.id === 'large-array');
|
|
639
|
+
expect(largeArrayItem?.items).toHaveLength(100);
|
|
640
|
+
expect(largeArrayItem?.items?.[0]).toEqual({ index: 0, data: 'item-0', nested: { level: 1, value: 0 } });
|
|
641
|
+
expect(largeArrayItem?.items?.[99]).toEqual({ index: 99, data: 'item-99', nested: { level: 1, value: 198 } });
|
|
642
|
+
});
|
|
643
|
+
it('handles exactly 1 item', async () => {
|
|
644
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
645
|
+
const items = [{ id: 'single', name: 'only-one', value: 42 }];
|
|
646
|
+
const capturedEnvelopes = [];
|
|
647
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
648
|
+
capturedEnvelopes.push(envelopes[0]);
|
|
649
|
+
return {
|
|
650
|
+
successCount: envelopes.length,
|
|
651
|
+
failedCount: 0,
|
|
652
|
+
batchCount: 1,
|
|
653
|
+
failedMessages: [],
|
|
654
|
+
};
|
|
655
|
+
});
|
|
656
|
+
const result = await bundledClient.sendBundled('test', items);
|
|
657
|
+
expect(result.totalItems).toBe(1);
|
|
658
|
+
expect(result.bundleCount).toBe(1);
|
|
659
|
+
expect(result.messageCount).toBe(1);
|
|
660
|
+
expect(capturedEnvelopes).toHaveLength(1);
|
|
661
|
+
expect(capturedEnvelopes[0].n).toBe(1);
|
|
662
|
+
const { items: unbundledItems } = await unbundleRecords([{
|
|
663
|
+
messageId: 'msg-1',
|
|
664
|
+
body: JSON.stringify({ messageBody: capturedEnvelopes[0] }),
|
|
665
|
+
}]);
|
|
666
|
+
expect(unbundledItems).toHaveLength(1);
|
|
667
|
+
expect(unbundledItems[0]).toEqual(items[0]);
|
|
668
|
+
});
|
|
669
|
+
it('handles exactly maxItemsPerBundle items', async () => {
|
|
670
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
671
|
+
const maxItems = 50;
|
|
672
|
+
const client = new sqs_bundled_client_1.BundledSQSClient(mockSqsClient, {
|
|
673
|
+
compression: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
674
|
+
maxItemsPerBundle: maxItems,
|
|
675
|
+
});
|
|
676
|
+
const items = Array.from({ length: maxItems }, (_, i) => ({
|
|
677
|
+
id: `item-${i}`,
|
|
678
|
+
name: `name-${i}`,
|
|
679
|
+
value: i,
|
|
680
|
+
}));
|
|
681
|
+
const capturedEnvelopes = [];
|
|
682
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
683
|
+
capturedEnvelopes.push(...envelopes);
|
|
684
|
+
return {
|
|
685
|
+
successCount: envelopes.length,
|
|
686
|
+
failedCount: 0,
|
|
687
|
+
batchCount: 1,
|
|
688
|
+
failedMessages: [],
|
|
689
|
+
};
|
|
690
|
+
});
|
|
691
|
+
const result = await client.sendBundled('test', items);
|
|
692
|
+
expect(result.totalItems).toBe(maxItems);
|
|
693
|
+
expect(capturedEnvelopes).toHaveLength(1);
|
|
694
|
+
expect(capturedEnvelopes[0].n).toBe(maxItems);
|
|
695
|
+
const sqsRecords = capturedEnvelopes.map((env, i) => ({
|
|
696
|
+
messageId: `msg-${i}`,
|
|
697
|
+
body: JSON.stringify({ messageBody: env }),
|
|
698
|
+
}));
|
|
699
|
+
const { items: unbundledItems } = await unbundleRecords(sqsRecords);
|
|
700
|
+
expect(unbundledItems).toHaveLength(maxItems);
|
|
701
|
+
});
|
|
702
|
+
it('handles maxItemsPerBundle + 1 items (splits correctly)', async () => {
|
|
703
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
704
|
+
const maxItems = 50;
|
|
705
|
+
const client = new sqs_bundled_client_1.BundledSQSClient(mockSqsClient, {
|
|
706
|
+
compression: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
707
|
+
maxItemsPerBundle: maxItems,
|
|
708
|
+
});
|
|
709
|
+
const items = Array.from({ length: maxItems + 1 }, (_, i) => ({
|
|
710
|
+
id: `item-${i}`,
|
|
711
|
+
name: `name-${i}`,
|
|
712
|
+
value: i,
|
|
713
|
+
}));
|
|
714
|
+
const capturedEnvelopes = [];
|
|
715
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
716
|
+
capturedEnvelopes.push(...envelopes);
|
|
717
|
+
return {
|
|
718
|
+
successCount: envelopes.length,
|
|
719
|
+
failedCount: 0,
|
|
720
|
+
batchCount: 1,
|
|
721
|
+
failedMessages: [],
|
|
722
|
+
};
|
|
723
|
+
});
|
|
724
|
+
const result = await client.sendBundled('test', items);
|
|
725
|
+
expect(result.totalItems).toBe(maxItems + 1);
|
|
726
|
+
expect(capturedEnvelopes).toHaveLength(2);
|
|
727
|
+
expect(capturedEnvelopes[0].n).toBe(maxItems);
|
|
728
|
+
expect(capturedEnvelopes[1].n).toBe(1);
|
|
729
|
+
const sqsRecords = capturedEnvelopes.map((env, i) => ({
|
|
730
|
+
messageId: `msg-${i}`,
|
|
731
|
+
body: JSON.stringify({ messageBody: env }),
|
|
732
|
+
}));
|
|
733
|
+
const { items: unbundledItems } = await unbundleRecords(sqsRecords);
|
|
734
|
+
expect(unbundledItems).toHaveLength(maxItems + 1);
|
|
735
|
+
});
|
|
736
|
+
it('handles uncompressed mode (NONE) with various data', async () => {
|
|
737
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
738
|
+
const noCompressionClient = new sqs_bundled_client_1.BundledSQSClient(mockSqsClient, {
|
|
739
|
+
compression: sqs_bundled_client_types_1.CompressionAlgorithm.NONE,
|
|
740
|
+
});
|
|
741
|
+
const items = Array.from({ length: 50 }, (_, i) => ({
|
|
742
|
+
id: `uncompressed-${i}`,
|
|
743
|
+
name: `test-${i}`,
|
|
744
|
+
value: i * 100,
|
|
745
|
+
data: `Some data for item ${i}`,
|
|
746
|
+
}));
|
|
747
|
+
const capturedEnvelopes = [];
|
|
748
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
749
|
+
capturedEnvelopes.push(envelopes[0]);
|
|
750
|
+
return {
|
|
751
|
+
successCount: envelopes.length,
|
|
752
|
+
failedCount: 0,
|
|
753
|
+
batchCount: 1,
|
|
754
|
+
failedMessages: [],
|
|
755
|
+
};
|
|
756
|
+
});
|
|
757
|
+
const result = await noCompressionClient.sendBundled('test', items);
|
|
758
|
+
expect(result.totalItems).toBe(50);
|
|
759
|
+
expect(capturedEnvelopes).toHaveLength(1);
|
|
760
|
+
const envelope = capturedEnvelopes[0];
|
|
761
|
+
expect(envelope.c).toBe(sqs_bundled_client_types_1.CompressionAlgorithm.NONE);
|
|
762
|
+
expect(Array.isArray(envelope.p)).toBe(true);
|
|
763
|
+
expect(envelope.p.length).toBe(50);
|
|
764
|
+
expect(result.metrics.compressionRatio).toBeCloseTo(1, 0);
|
|
765
|
+
const { items: unbundledItems } = await unbundleRecords([{
|
|
766
|
+
messageId: 'msg-1',
|
|
767
|
+
body: JSON.stringify({ messageBody: envelope }),
|
|
768
|
+
}]);
|
|
769
|
+
expect(unbundledItems).toHaveLength(50);
|
|
770
|
+
expect(unbundledItems[0]).toEqual(items[0]);
|
|
771
|
+
expect(unbundledItems[49]).toEqual(items[49]);
|
|
772
|
+
});
|
|
773
|
+
it('produces correct metrics for a large mixed batch', async () => {
|
|
774
|
+
const items = Array.from({ length: 500 }, (_, i) => ({
|
|
775
|
+
id: `metric-test-${i}`,
|
|
776
|
+
name: `item-${i % 10}`,
|
|
777
|
+
value: i,
|
|
778
|
+
data: i % 5 === 0 ? 'repeated-data'.repeat(100) : `unique-${i}`,
|
|
779
|
+
}));
|
|
780
|
+
const capturedEnvelopes = [];
|
|
781
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
782
|
+
capturedEnvelopes.push(...envelopes);
|
|
783
|
+
return {
|
|
784
|
+
successCount: envelopes.length,
|
|
785
|
+
failedCount: 0,
|
|
786
|
+
batchCount: 1,
|
|
787
|
+
failedMessages: [],
|
|
788
|
+
};
|
|
789
|
+
});
|
|
790
|
+
const result = await bundledClient.sendBundled('test', items);
|
|
791
|
+
expect(result.totalItems).toBe(500);
|
|
792
|
+
expect(result.metrics.originalSizeBytes).toBeGreaterThan(0);
|
|
793
|
+
expect(result.metrics.finalSizeBytes).toBeGreaterThan(0);
|
|
794
|
+
expect(result.metrics.compressionRatio).toBeGreaterThan(1);
|
|
795
|
+
expect(result.metrics.compressionTimeMs).toBeGreaterThanOrEqual(0);
|
|
796
|
+
expect(result.metrics.avgItemsPerBundle).toBeGreaterThan(0);
|
|
797
|
+
const expectedAvg = items.length / capturedEnvelopes.length;
|
|
798
|
+
expect(result.metrics.avgItemsPerBundle).toBeCloseTo(expectedAvg, 1);
|
|
799
|
+
expect(result.billableRequests).toBeGreaterThanOrEqual(1);
|
|
800
|
+
});
|
|
801
|
+
it('handles items with special JSON edge cases', async () => {
|
|
802
|
+
const { unbundleRecords } = await Promise.resolve().then(() => __importStar(require('../../clients/generic/sqs-unbundle')));
|
|
803
|
+
const edgeCaseItems = [
|
|
804
|
+
{
|
|
805
|
+
id: 'large-numbers',
|
|
806
|
+
name: 'numbers',
|
|
807
|
+
bigInt: 9007199254740991,
|
|
808
|
+
scientific: 1.23e10,
|
|
809
|
+
decimal: 0.1 + 0.2,
|
|
810
|
+
negative: -999999,
|
|
811
|
+
zero: 0,
|
|
812
|
+
negativeZero: -0,
|
|
813
|
+
},
|
|
814
|
+
{
|
|
815
|
+
id: 'special-strings',
|
|
816
|
+
name: 'strings',
|
|
817
|
+
data: JSON.stringify({ nested: 'json' }),
|
|
818
|
+
withQuotes: 'He said "hello"',
|
|
819
|
+
withBackslash: 'path\\to\\file',
|
|
820
|
+
withSlash: 'https://example.com/path',
|
|
821
|
+
},
|
|
822
|
+
{
|
|
823
|
+
id: 'boolean-variants',
|
|
824
|
+
name: 'booleans',
|
|
825
|
+
trueVal: true,
|
|
826
|
+
falseVal: false,
|
|
827
|
+
truthy: 1,
|
|
828
|
+
falsy: 0,
|
|
829
|
+
},
|
|
830
|
+
{
|
|
831
|
+
id: 'dates-as-strings',
|
|
832
|
+
name: 'dates',
|
|
833
|
+
isoDate: '2024-01-15T10:30:00.000Z',
|
|
834
|
+
timestamp: 1705315800000,
|
|
835
|
+
},
|
|
836
|
+
];
|
|
837
|
+
let capturedEnvelope = null;
|
|
838
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
839
|
+
capturedEnvelope = envelopes[0];
|
|
840
|
+
return {
|
|
841
|
+
successCount: envelopes.length,
|
|
842
|
+
failedCount: 0,
|
|
843
|
+
batchCount: 1,
|
|
844
|
+
failedMessages: [],
|
|
845
|
+
};
|
|
846
|
+
});
|
|
847
|
+
const genericClient = new sqs_bundled_client_1.BundledSQSClient(mockSqsClient, {
|
|
848
|
+
compression: sqs_bundled_client_types_1.CompressionAlgorithm.ZSTD,
|
|
849
|
+
});
|
|
850
|
+
const result = await genericClient.sendBundled('test', edgeCaseItems);
|
|
851
|
+
expect(result.totalItems).toBe(edgeCaseItems.length);
|
|
852
|
+
expect(result.skippedItems).toBe(0);
|
|
853
|
+
const { items: unbundledItems } = await unbundleRecords([{
|
|
854
|
+
messageId: 'msg-1',
|
|
855
|
+
body: JSON.stringify({ messageBody: capturedEnvelope }),
|
|
856
|
+
}]);
|
|
857
|
+
expect(unbundledItems).toHaveLength(edgeCaseItems.length);
|
|
858
|
+
const numbersItem = unbundledItems.find(item => item.id === 'large-numbers');
|
|
859
|
+
expect(numbersItem?.bigInt).toBe(9007199254740991);
|
|
860
|
+
expect(numbersItem?.scientific).toBe(1.23e10);
|
|
861
|
+
expect(numbersItem?.negative).toBe(-999999);
|
|
862
|
+
expect(numbersItem?.zero).toBe(0);
|
|
863
|
+
const stringsItem = unbundledItems.find(item => item.id === 'special-strings');
|
|
864
|
+
expect(stringsItem?.data).toBe(JSON.stringify({ nested: 'json' }));
|
|
865
|
+
expect(stringsItem?.withQuotes).toBe('He said "hello"');
|
|
866
|
+
expect(stringsItem?.withBackslash).toBe('path\\to\\file');
|
|
867
|
+
const booleansItem = unbundledItems.find(item => item.id === 'boolean-variants');
|
|
868
|
+
expect(booleansItem?.trueVal).toBe(true);
|
|
869
|
+
expect(booleansItem?.falseVal).toBe(false);
|
|
870
|
+
});
|
|
871
|
+
it('handles concurrent sends without interference', async () => {
|
|
872
|
+
const batch1 = Array.from({ length: 100 }, (_, i) => ({
|
|
873
|
+
id: `batch1-${i}`,
|
|
874
|
+
name: 'batch-one',
|
|
875
|
+
value: i,
|
|
876
|
+
}));
|
|
877
|
+
const batch2 = Array.from({ length: 100 }, (_, i) => ({
|
|
878
|
+
id: `batch2-${i}`,
|
|
879
|
+
name: 'batch-two',
|
|
880
|
+
value: i * 2,
|
|
881
|
+
}));
|
|
882
|
+
const batch3 = Array.from({ length: 100 }, (_, i) => ({
|
|
883
|
+
id: `batch3-${i}`,
|
|
884
|
+
name: 'batch-three',
|
|
885
|
+
value: i * 3,
|
|
886
|
+
}));
|
|
887
|
+
mockSqsClient.buildAndSendMessagesV2.mockImplementation(async (_, envelopes) => {
|
|
888
|
+
return {
|
|
889
|
+
successCount: envelopes.length,
|
|
890
|
+
failedCount: 0,
|
|
891
|
+
batchCount: 1,
|
|
892
|
+
failedMessages: [],
|
|
893
|
+
};
|
|
894
|
+
});
|
|
895
|
+
const [result1, result2, result3] = await Promise.all([
|
|
896
|
+
bundledClient.sendBundled('test', batch1),
|
|
897
|
+
bundledClient.sendBundled('test', batch2),
|
|
898
|
+
bundledClient.sendBundled('test', batch3),
|
|
899
|
+
]);
|
|
900
|
+
expect(result1.totalItems).toBe(100);
|
|
901
|
+
expect(result2.totalItems).toBe(100);
|
|
902
|
+
expect(result3.totalItems).toBe(100);
|
|
903
|
+
expect(result1.skippedItems).toBe(0);
|
|
904
|
+
expect(result2.skippedItems).toBe(0);
|
|
905
|
+
expect(result3.skippedItems).toBe(0);
|
|
906
|
+
});
|
|
907
|
+
});
|
|
224
908
|
});
|
|
225
909
|
//# sourceMappingURL=sqs-bundled-client.spec.js.map
|