@ftschopp/dynatable-core 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/builders/batch-get/create-batch-get-builder.d.ts.map +1 -1
- package/dist/builders/batch-get/create-batch-get-builder.js +8 -3
- package/dist/builders/get/create-get-builder.d.ts.map +1 -1
- package/dist/builders/get/create-get-builder.js +2 -19
- package/dist/builders/query/create-query-builder.d.ts.map +1 -1
- package/dist/builders/query/create-query-builder.js +34 -3
- package/dist/builders/query/types.d.ts +16 -0
- package/dist/builders/query/types.d.ts.map +1 -1
- package/dist/builders/scan/create-scan-builder.d.ts.map +1 -1
- package/dist/builders/scan/create-scan-builder.js +42 -2
- package/dist/builders/scan/types.d.ts +35 -0
- package/dist/builders/scan/types.d.ts.map +1 -1
- package/dist/builders/shared/index.d.ts +1 -0
- package/dist/builders/shared/index.d.ts.map +1 -1
- package/dist/builders/shared/index.js +1 -0
- package/dist/builders/shared/projection.d.ts +15 -0
- package/dist/builders/shared/projection.d.ts.map +1 -0
- package/dist/builders/shared/projection.js +26 -0
- package/package.json +1 -1
- package/src/builders/batch-get/create-batch-get-builder.test.ts +30 -2
- package/src/builders/batch-get/create-batch-get-builder.ts +10 -3
- package/src/builders/get/create-get-builder.ts +1 -24
- package/src/builders/query/create-query-builder.test.ts +205 -1
- package/src/builders/query/create-query-builder.ts +40 -5
- package/src/builders/query/types.ts +17 -0
- package/src/builders/scan/create-scan-builder.test.ts +162 -6
- package/src/builders/scan/create-scan-builder.ts +52 -4
- package/src/builders/scan/types.ts +39 -0
- package/src/builders/shared/index.ts +1 -0
- package/src/builders/shared/projection.ts +28 -0
- package/tests/integration/instagram-clone.integration.test.ts +23 -4
|
@@ -60,7 +60,13 @@ describe('QueryBuilder - Pagination', () => {
|
|
|
60
60
|
expect(params.ExclusiveStartKey).toEqual(startKey);
|
|
61
61
|
expect(params.Limit).toBe(10);
|
|
62
62
|
expect(params.ScanIndexForward).toBe(false);
|
|
63
|
-
expect(params.ProjectionExpression).toBe('name, age');
|
|
63
|
+
expect(params.ProjectionExpression).toBe('#name, #age');
|
|
64
|
+
expect(params.ExpressionAttributeNames).toEqual(
|
|
65
|
+
expect.objectContaining({
|
|
66
|
+
'#name': 'name',
|
|
67
|
+
'#age': 'age',
|
|
68
|
+
})
|
|
69
|
+
);
|
|
64
70
|
});
|
|
65
71
|
|
|
66
72
|
test('should not include ExclusiveStartKey if not set', () => {
|
|
@@ -227,6 +233,126 @@ describe('QueryBuilder - Pagination', () => {
|
|
|
227
233
|
expect(builder2.dbParams().ExclusiveStartKey).toEqual(startKey);
|
|
228
234
|
});
|
|
229
235
|
});
|
|
236
|
+
|
|
237
|
+
describe('iterate method', () => {
|
|
238
|
+
test('walks every page transparently and yields items in order', async () => {
|
|
239
|
+
const page1 = [
|
|
240
|
+
{ pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
|
|
241
|
+
{ pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
|
|
242
|
+
];
|
|
243
|
+
const page2 = [
|
|
244
|
+
{ pk: 'USER#3', sk: 'USER#3', name: 'User 3' },
|
|
245
|
+
{ pk: 'USER#4', sk: 'USER#4', name: 'User 4' },
|
|
246
|
+
];
|
|
247
|
+
const page3 = [{ pk: 'USER#5', sk: 'USER#5', name: 'User 5' }];
|
|
248
|
+
|
|
249
|
+
ddbMock
|
|
250
|
+
.on(QueryCommand)
|
|
251
|
+
.resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
|
|
252
|
+
.resolvesOnce({ Items: page2, LastEvaluatedKey: { pk: 'USER#4', sk: 'USER#4' } })
|
|
253
|
+
.resolvesOnce({ Items: page3 });
|
|
254
|
+
|
|
255
|
+
const collected: TestModel[] = [];
|
|
256
|
+
for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
257
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
|
|
258
|
+
.iterate()) {
|
|
259
|
+
collected.push(item);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
expect(collected).toHaveLength(5);
|
|
263
|
+
expect(collected.map((u) => u.name)).toEqual([
|
|
264
|
+
'User 1',
|
|
265
|
+
'User 2',
|
|
266
|
+
'User 3',
|
|
267
|
+
'User 4',
|
|
268
|
+
'User 5',
|
|
269
|
+
]);
|
|
270
|
+
expect(ddbMock.calls()).toHaveLength(3);
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
test('forwards ExclusiveStartKey from each response into the next call', async () => {
|
|
274
|
+
const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
|
|
275
|
+
const cursor2 = { pk: 'USER#4', sk: 'USER#4' };
|
|
276
|
+
|
|
277
|
+
ddbMock
|
|
278
|
+
.on(QueryCommand)
|
|
279
|
+
.resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
|
|
280
|
+
.resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }], LastEvaluatedKey: cursor2 })
|
|
281
|
+
.resolvesOnce({ Items: [{ pk: 'USER#5', sk: 'USER#5' }] });
|
|
282
|
+
|
|
283
|
+
const iterator = createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
284
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
|
|
285
|
+
.iterate();
|
|
286
|
+
|
|
287
|
+
// Consume the iterator
|
|
288
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
289
|
+
for await (const _ of iterator) {
|
|
290
|
+
// drain
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const calls = ddbMock.calls();
|
|
294
|
+
expect(calls).toHaveLength(3);
|
|
295
|
+
expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
|
|
296
|
+
expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
|
|
297
|
+
expect((calls[2]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor2);
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
test('starts from the user-provided cursor on the first call', async () => {
|
|
301
|
+
const startKey = { pk: 'USER#10', sk: 'USER#10' };
|
|
302
|
+
|
|
303
|
+
ddbMock.on(QueryCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
|
|
304
|
+
|
|
305
|
+
const iterator = createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
306
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
|
|
307
|
+
.startFrom(startKey)
|
|
308
|
+
.iterate();
|
|
309
|
+
|
|
310
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
311
|
+
for await (const _ of iterator) {
|
|
312
|
+
// drain
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const calls = ddbMock.calls();
|
|
316
|
+
expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
test('break out of the loop stops further DynamoDB calls', async () => {
|
|
320
|
+
ddbMock
|
|
321
|
+
.on(QueryCommand)
|
|
322
|
+
.resolvesOnce({
|
|
323
|
+
Items: [
|
|
324
|
+
{ pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
|
|
325
|
+
{ pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
|
|
326
|
+
],
|
|
327
|
+
LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
const collected: TestModel[] = [];
|
|
331
|
+
for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
332
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
|
|
333
|
+
.iterate()) {
|
|
334
|
+
collected.push(item);
|
|
335
|
+
if (collected.length >= 1) break;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
expect(collected).toHaveLength(1);
|
|
339
|
+
expect(ddbMock.calls()).toHaveLength(1);
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
test('handles an empty result without making extra calls', async () => {
|
|
343
|
+
ddbMock.on(QueryCommand).resolves({ Items: [] });
|
|
344
|
+
|
|
345
|
+
const collected: TestModel[] = [];
|
|
346
|
+
for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
347
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'NONE#'))
|
|
348
|
+
.iterate()) {
|
|
349
|
+
collected.push(item);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
expect(collected).toEqual([]);
|
|
353
|
+
expect(ddbMock.calls()).toHaveLength(1);
|
|
354
|
+
});
|
|
355
|
+
});
|
|
230
356
|
});
|
|
231
357
|
|
|
232
358
|
describe('QueryBuilder - GSI key recognition', () => {
|
|
@@ -502,3 +628,81 @@ describe('QueryBuilder - entity type auto filter', () => {
|
|
|
502
628
|
expect(params.ExpressionAttributeValues?.[':_type']).toBe('AirportPersonnel');
|
|
503
629
|
});
|
|
504
630
|
});
|
|
631
|
+
|
|
632
|
+
describe('QueryBuilder - projection placeholders', () => {
|
|
633
|
+
const client = new DynamoDBClient({});
|
|
634
|
+
const tableName = 'TestTable';
|
|
635
|
+
|
|
636
|
+
interface UserModel {
|
|
637
|
+
pk: string;
|
|
638
|
+
sk: string;
|
|
639
|
+
name?: string;
|
|
640
|
+
status?: string;
|
|
641
|
+
age?: number;
|
|
642
|
+
type?: string;
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
const userModel: ModelDefinition = {
|
|
646
|
+
key: {
|
|
647
|
+
PK: { type: String, value: 'USER#${username}' },
|
|
648
|
+
SK: { type: String, value: 'USER#${username}' },
|
|
649
|
+
},
|
|
650
|
+
attributes: {
|
|
651
|
+
username: { type: String, required: true },
|
|
652
|
+
name: { type: String },
|
|
653
|
+
status: { type: String },
|
|
654
|
+
age: { type: Number },
|
|
655
|
+
type: { type: String },
|
|
656
|
+
},
|
|
657
|
+
};
|
|
658
|
+
|
|
659
|
+
test('projects reserved DynamoDB words via #-placeholders', () => {
|
|
660
|
+
const params = createQueryBuilder<UserModel>(tableName, client, userModel)
|
|
661
|
+
.where((attr, op) => op.eq(attr.pk, 'USER#alice'))
|
|
662
|
+
.select(['name', 'status', 'type'])
|
|
663
|
+
.dbParams();
|
|
664
|
+
|
|
665
|
+
expect(params.ProjectionExpression).toBe('#name, #status, #type');
|
|
666
|
+
expect(params.ExpressionAttributeNames).toEqual(
|
|
667
|
+
expect.objectContaining({
|
|
668
|
+
'#name': 'name',
|
|
669
|
+
'#status': 'status',
|
|
670
|
+
'#type': 'type',
|
|
671
|
+
})
|
|
672
|
+
);
|
|
673
|
+
});
|
|
674
|
+
|
|
675
|
+
test('merges projection names with key/filter names without clobbering', () => {
|
|
676
|
+
const params = createQueryBuilder<UserModel & { username: string }>(
|
|
677
|
+
tableName,
|
|
678
|
+
client,
|
|
679
|
+
userModel
|
|
680
|
+
)
|
|
681
|
+
.where((attr, op) => op.and(op.eq(attr.username, 'alice'), op.gt(attr.age, 18)))
|
|
682
|
+
.select(['name', 'status'])
|
|
683
|
+
.dbParams();
|
|
684
|
+
|
|
685
|
+
// Projection placeholders (#name, #status) merged with key (#PK from
|
|
686
|
+
// the username→PK template rewrite) and filter (#age) — none clobbered.
|
|
687
|
+
expect(params.ExpressionAttributeNames).toEqual(
|
|
688
|
+
expect.objectContaining({
|
|
689
|
+
'#name': 'name',
|
|
690
|
+
'#status': 'status',
|
|
691
|
+
'#PK': 'PK',
|
|
692
|
+
'#age': 'age',
|
|
693
|
+
})
|
|
694
|
+
);
|
|
695
|
+
});
|
|
696
|
+
|
|
697
|
+
test('shares the same placeholder when an attribute is both projected and filtered (no key duplication)', () => {
|
|
698
|
+
const params = createQueryBuilder<UserModel>(tableName, client, userModel)
|
|
699
|
+
.where((attr, op) => op.and(op.eq(attr.pk, 'USER#alice'), op.eq(attr.status, 'active')))
|
|
700
|
+
.select(['name', 'status'])
|
|
701
|
+
.dbParams();
|
|
702
|
+
|
|
703
|
+
// The map has #status exactly once, mapping to "status".
|
|
704
|
+
expect(params.ExpressionAttributeNames!['#status']).toBe('status');
|
|
705
|
+
expect(params.ProjectionExpression).toBe('#name, #status');
|
|
706
|
+
expect(params.FilterExpression).toMatch(/#status = :status_\d+/);
|
|
707
|
+
});
|
|
708
|
+
});
|
|
@@ -4,6 +4,7 @@ import { QueryCommand, QueryCommandInput } from '@aws-sdk/lib-dynamodb';
|
|
|
4
4
|
import { Condition, AttrBuilder, AttrRef } from '../shared';
|
|
5
5
|
import { createOpBuilder } from '../shared/operators';
|
|
6
6
|
import { buildExpression } from '../shared/conditions';
|
|
7
|
+
import { buildProjectionExpression } from '../shared/projection';
|
|
7
8
|
import { QueryBuilder, QueryExecutor, QueryResult } from './types';
|
|
8
9
|
import { ModelDefinition } from '../../core/types';
|
|
9
10
|
import { extractTemplateVars } from '../../utils/model-utils';
|
|
@@ -294,7 +295,7 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
|
|
|
294
295
|
}
|
|
295
296
|
|
|
296
297
|
// Merge attribute names and values from both expressions
|
|
297
|
-
const allNames = {
|
|
298
|
+
const allNames: Record<string, string> = {
|
|
298
299
|
...(keyExpr.names ?? {}),
|
|
299
300
|
...(filterExpr.names ?? {}),
|
|
300
301
|
};
|
|
@@ -303,6 +304,19 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
|
|
|
303
304
|
...(filterExpr.values ?? {}),
|
|
304
305
|
};
|
|
305
306
|
|
|
307
|
+
// Build ProjectionExpression with placeholders so reserved words
|
|
308
|
+
// (name, date, status, type, …) don't blow up at DynamoDB.
|
|
309
|
+
// Idempotent merge: filter/key and projection placeholders both map
|
|
310
|
+
// `#name → name`, so collisions resolve to the same value.
|
|
311
|
+
let projectionExpression: string | undefined;
|
|
312
|
+
if (state.projection && state.projection.length > 0) {
|
|
313
|
+
const proj = buildProjectionExpression(
|
|
314
|
+
(state.projection as (keyof Model)[]).map((a) => String(a))
|
|
315
|
+
);
|
|
316
|
+
projectionExpression = proj.ProjectionExpression;
|
|
317
|
+
Object.assign(allNames, proj.ExpressionAttributeNames);
|
|
318
|
+
}
|
|
319
|
+
|
|
306
320
|
return {
|
|
307
321
|
TableName: state.tableName,
|
|
308
322
|
...(keyExpr.expression && {
|
|
@@ -322,10 +336,9 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
|
|
|
322
336
|
...(state.scanForward !== undefined && {
|
|
323
337
|
ScanIndexForward: state.scanForward,
|
|
324
338
|
}),
|
|
325
|
-
...(
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
}),
|
|
339
|
+
...(projectionExpression && {
|
|
340
|
+
ProjectionExpression: projectionExpression,
|
|
341
|
+
}),
|
|
329
342
|
...(state.consistentReadEnabled && { ConsistentRead: true }),
|
|
330
343
|
...(state.exclusiveStartKey && {
|
|
331
344
|
ExclusiveStartKey: state.exclusiveStartKey,
|
|
@@ -333,6 +346,14 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
|
|
|
333
346
|
};
|
|
334
347
|
},
|
|
335
348
|
|
|
349
|
+
/**
|
|
350
|
+
* ⚠️ Returns only the FIRST page of results. DynamoDB caps each Query
|
|
351
|
+
* response at ~1MB (or `Limit` if set). If the matching set is larger,
|
|
352
|
+
* the remaining items are silently dropped.
|
|
353
|
+
*
|
|
354
|
+
* Use {@link executeWithPagination} when you need to drive pagination
|
|
355
|
+
* yourself, or {@link iterate} to walk every matching item lazily.
|
|
356
|
+
*/
|
|
336
357
|
async execute(): Promise<Model[]> {
|
|
337
358
|
const params = this.dbParams();
|
|
338
359
|
const result = await state.client.send(new QueryCommand(params));
|
|
@@ -351,6 +372,20 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
|
|
|
351
372
|
scannedCount: result.ScannedCount,
|
|
352
373
|
};
|
|
353
374
|
},
|
|
375
|
+
|
|
376
|
+
async *iterate(): AsyncIterableIterator<Model> {
|
|
377
|
+
const baseParams = this.dbParams();
|
|
378
|
+
let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
|
|
379
|
+
do {
|
|
380
|
+
const params = { ...baseParams, ExclusiveStartKey: cursor };
|
|
381
|
+
const result = await state.client.send(new QueryCommand(params));
|
|
382
|
+
state.logger?.log('QueryCommand', params, result);
|
|
383
|
+
for (const item of (result.Items ?? []) as unknown as Model[]) {
|
|
384
|
+
yield item;
|
|
385
|
+
}
|
|
386
|
+
cursor = result.LastEvaluatedKey;
|
|
387
|
+
} while (cursor);
|
|
388
|
+
},
|
|
354
389
|
};
|
|
355
390
|
}
|
|
356
391
|
|
|
@@ -74,6 +74,23 @@ export interface QueryExecutor<Model> extends ExecutableBuilder<Model[]> {
|
|
|
74
74
|
* Executes the query and returns result with pagination metadata
|
|
75
75
|
*/
|
|
76
76
|
executeWithPagination(): Promise<QueryResult<Model>>;
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Returns an async iterator that paginates internally and yields one item at
|
|
80
|
+
* a time. Memory stays at one page; you can `break` out of the loop early.
|
|
81
|
+
*
|
|
82
|
+
* `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
|
|
83
|
+
* cap, not a total cap. To cap the total, count yourself inside the loop:
|
|
84
|
+
*
|
|
85
|
+
* ```ts
|
|
86
|
+
* let n = 0;
|
|
87
|
+
* for await (const item of builder.iterate()) {
|
|
88
|
+
* if (n++ >= 1000) break;
|
|
89
|
+
* process(item);
|
|
90
|
+
* }
|
|
91
|
+
* ```
|
|
92
|
+
*/
|
|
93
|
+
iterate(): AsyncIterableIterator<Model>;
|
|
77
94
|
}
|
|
78
95
|
|
|
79
96
|
/**
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
2
|
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
|
|
3
|
+
import { ScanCommand } from '@aws-sdk/lib-dynamodb';
|
|
4
|
+
import { mockClient } from 'aws-sdk-client-mock';
|
|
3
5
|
import { createScanBuilder } from './create-scan-builder';
|
|
4
6
|
|
|
7
|
+
const ddbMock = mockClient(DynamoDBClient);
|
|
8
|
+
|
|
5
9
|
describe('ScanBuilder', () => {
|
|
6
10
|
const client = new DynamoDBClient({});
|
|
7
11
|
const tableName = 'TestTable';
|
|
@@ -73,17 +77,52 @@ describe('ScanBuilder', () => {
|
|
|
73
77
|
.select(['name', 'age', 'status'])
|
|
74
78
|
.dbParams();
|
|
75
79
|
|
|
76
|
-
expect(params.ProjectionExpression).toBe('name, age, status');
|
|
80
|
+
expect(params.ProjectionExpression).toBe('#name, #age, #status');
|
|
81
|
+
expect(params.ExpressionAttributeNames).toEqual({
|
|
82
|
+
'#name': 'name',
|
|
83
|
+
'#age': 'age',
|
|
84
|
+
'#status': 'status',
|
|
85
|
+
});
|
|
77
86
|
});
|
|
78
87
|
|
|
79
|
-
test('should build params with select and filter', () => {
|
|
88
|
+
test('should build params with select and filter, merging ExpressionAttributeNames', () => {
|
|
80
89
|
const params = createScanBuilder<TestModel>(tableName, client)
|
|
81
90
|
.select(['name', 'age'])
|
|
82
|
-
.filter((attr, op) => op.gt(attr.
|
|
91
|
+
.filter((attr, op) => op.gt(attr.score, 50))
|
|
92
|
+
.dbParams();
|
|
93
|
+
|
|
94
|
+
expect(params.ProjectionExpression).toBe('#name, #age');
|
|
95
|
+
expect(params.FilterExpression).toMatch(/#score > :score_\d+/);
|
|
96
|
+
// Merged names from filter (#score) + projection (#name, #age)
|
|
97
|
+
expect(params.ExpressionAttributeNames).toEqual({
|
|
98
|
+
'#name': 'name',
|
|
99
|
+
'#age': 'age',
|
|
100
|
+
'#score': 'score',
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('placeholders survive when projected attribute is also referenced in the filter (no key collision)', () => {
|
|
105
|
+
const params = createScanBuilder<TestModel>(tableName, client)
|
|
106
|
+
.select(['name', 'age'])
|
|
107
|
+
.filter((attr, op) => op.eq(attr.name, 'alice'))
|
|
108
|
+
.dbParams();
|
|
109
|
+
|
|
110
|
+
expect(params.ProjectionExpression).toBe('#name, #age');
|
|
111
|
+
// #name resolves to "name" — filter and projection share the placeholder.
|
|
112
|
+
expect(params.ExpressionAttributeNames!['#name']).toBe('name');
|
|
113
|
+
expect(params.ExpressionAttributeNames!['#age']).toBe('age');
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test('reserved DynamoDB words like "name", "date", "status", "type" can be projected', () => {
|
|
117
|
+
const params = createScanBuilder<TestModel>(tableName, client)
|
|
118
|
+
.select(['name', 'status'])
|
|
83
119
|
.dbParams();
|
|
84
120
|
|
|
85
|
-
expect(params.ProjectionExpression).toBe('name,
|
|
86
|
-
expect(params.
|
|
121
|
+
expect(params.ProjectionExpression).toBe('#name, #status');
|
|
122
|
+
expect(params.ExpressionAttributeNames).toEqual({
|
|
123
|
+
'#name': 'name',
|
|
124
|
+
'#status': 'status',
|
|
125
|
+
});
|
|
87
126
|
});
|
|
88
127
|
});
|
|
89
128
|
|
|
@@ -180,7 +219,7 @@ describe('ScanBuilder', () => {
|
|
|
180
219
|
|
|
181
220
|
expect(params.TableName).toBe(tableName);
|
|
182
221
|
expect(params.FilterExpression).toMatch(/\(#age > :age_\d+\) AND \(#status = :status_\d+\)/);
|
|
183
|
-
expect(params.ProjectionExpression).toBe('name, age, status');
|
|
222
|
+
expect(params.ProjectionExpression).toBe('#name, #age, #status');
|
|
184
223
|
expect(params.Limit).toBe(100);
|
|
185
224
|
expect(params.ConsistentRead).toBe(true);
|
|
186
225
|
expect(params.IndexName).toBe('GSI1');
|
|
@@ -257,4 +296,121 @@ describe('ScanBuilder', () => {
|
|
|
257
296
|
expect(Object.values(params.ExpressionAttributeValues || {})).toContain(65);
|
|
258
297
|
});
|
|
259
298
|
});
|
|
299
|
+
|
|
300
|
+
describe('executeWithPagination', () => {
|
|
301
|
+
beforeEach(() => {
|
|
302
|
+
ddbMock.reset();
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
test('returns items, lastEvaluatedKey and counts', async () => {
|
|
306
|
+
const items = [
|
|
307
|
+
{ pk: 'USER#alice', sk: 'USER#alice', name: 'Alice' },
|
|
308
|
+
{ pk: 'USER#bob', sk: 'USER#bob', name: 'Bob' },
|
|
309
|
+
];
|
|
310
|
+
const lastKey = { pk: 'USER#bob', sk: 'USER#bob' };
|
|
311
|
+
|
|
312
|
+
ddbMock.on(ScanCommand).resolves({
|
|
313
|
+
Items: items,
|
|
314
|
+
LastEvaluatedKey: lastKey,
|
|
315
|
+
Count: 2,
|
|
316
|
+
ScannedCount: 5,
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
|
|
320
|
+
|
|
321
|
+
expect(result.items).toEqual(items);
|
|
322
|
+
expect(result.lastEvaluatedKey).toEqual(lastKey);
|
|
323
|
+
expect(result.count).toBe(2);
|
|
324
|
+
expect(result.scannedCount).toBe(5);
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
test('returns undefined lastEvaluatedKey on the final page', async () => {
|
|
328
|
+
ddbMock.on(ScanCommand).resolves({ Items: [], Count: 0, ScannedCount: 0 });
|
|
329
|
+
|
|
330
|
+
const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
|
|
331
|
+
|
|
332
|
+
expect(result.items).toEqual([]);
|
|
333
|
+
expect(result.lastEvaluatedKey).toBeUndefined();
|
|
334
|
+
});
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
describe('iterate', () => {
|
|
338
|
+
beforeEach(() => {
|
|
339
|
+
ddbMock.reset();
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
test('walks every page transparently and yields items in order', async () => {
|
|
343
|
+
const page1 = [
|
|
344
|
+
{ pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
|
|
345
|
+
{ pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
|
|
346
|
+
];
|
|
347
|
+
const page2 = [{ pk: 'USER#3', sk: 'USER#3', name: 'User 3' }];
|
|
348
|
+
|
|
349
|
+
ddbMock
|
|
350
|
+
.on(ScanCommand)
|
|
351
|
+
.resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
|
|
352
|
+
.resolvesOnce({ Items: page2 });
|
|
353
|
+
|
|
354
|
+
const collected: TestModel[] = [];
|
|
355
|
+
for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
|
|
356
|
+
collected.push(item);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
expect(collected.map((u) => u.name)).toEqual(['User 1', 'User 2', 'User 3']);
|
|
360
|
+
expect(ddbMock.calls()).toHaveLength(2);
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
test('forwards LastEvaluatedKey from each response into the next call', async () => {
|
|
364
|
+
const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
|
|
365
|
+
|
|
366
|
+
ddbMock
|
|
367
|
+
.on(ScanCommand)
|
|
368
|
+
.resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
|
|
369
|
+
.resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }] });
|
|
370
|
+
|
|
371
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
372
|
+
for await (const _ of createScanBuilder<TestModel>(tableName, client).iterate()) {
|
|
373
|
+
// drain
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
const calls = ddbMock.calls();
|
|
377
|
+
expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
|
|
378
|
+
expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
test('starts from the user-provided cursor on the first call', async () => {
|
|
382
|
+
const startKey = { pk: 'USER#10', sk: 'USER#10' };
|
|
383
|
+
|
|
384
|
+
ddbMock.on(ScanCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
|
|
385
|
+
|
|
386
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
387
|
+
for await (const _ of createScanBuilder<TestModel>(tableName, client)
|
|
388
|
+
.startFrom(startKey)
|
|
389
|
+
.iterate()) {
|
|
390
|
+
// drain
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
const calls = ddbMock.calls();
|
|
394
|
+
expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
test('break out of the loop stops further DynamoDB calls', async () => {
|
|
398
|
+
ddbMock.on(ScanCommand).resolvesOnce({
|
|
399
|
+
Items: [
|
|
400
|
+
{ pk: 'USER#1', sk: 'USER#1' },
|
|
401
|
+
{ pk: 'USER#2', sk: 'USER#2' },
|
|
402
|
+
],
|
|
403
|
+
LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
const collected: TestModel[] = [];
|
|
407
|
+
for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
|
|
408
|
+
collected.push(item);
|
|
409
|
+
if (collected.length >= 1) break;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
expect(collected).toHaveLength(1);
|
|
413
|
+
expect(ddbMock.calls()).toHaveLength(1);
|
|
414
|
+
});
|
|
415
|
+
});
|
|
260
416
|
});
|
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
2
|
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
|
|
3
3
|
import { ScanCommand } from '@aws-sdk/lib-dynamodb';
|
|
4
|
-
import {
|
|
5
|
-
|
|
4
|
+
import {
|
|
5
|
+
buildExpression,
|
|
6
|
+
AttrBuilder,
|
|
7
|
+
Condition,
|
|
8
|
+
createOpBuilder,
|
|
9
|
+
buildProjectionExpression,
|
|
10
|
+
} from '../shared';
|
|
11
|
+
import { ScanBuilder, ScanResult } from './types';
|
|
6
12
|
import { DynamoDBLogger } from '../../utils/dynamodb-logger';
|
|
7
13
|
|
|
8
14
|
/**
|
|
@@ -155,10 +161,18 @@ export function createScanBuilder<Model>(
|
|
|
155
161
|
expressionAttributeValues = result.values;
|
|
156
162
|
}
|
|
157
163
|
|
|
158
|
-
// Build ProjectionExpression
|
|
164
|
+
// Build ProjectionExpression with placeholders so reserved words
|
|
165
|
+
// (name, date, status, type, …) don't blow up at DynamoDB.
|
|
159
166
|
let projectionExpression = '';
|
|
160
167
|
if (projectionAttrs.length > 0) {
|
|
161
|
-
|
|
168
|
+
const proj = buildProjectionExpression(projectionAttrs.map((attr) => String(attr)));
|
|
169
|
+
projectionExpression = proj.ProjectionExpression;
|
|
170
|
+
// Idempotent merge: filter and projection placeholders both map
|
|
171
|
+
// `#name → name`, so collisions resolve to the same value.
|
|
172
|
+
expressionAttributeNames = {
|
|
173
|
+
...proj.ExpressionAttributeNames,
|
|
174
|
+
...expressionAttributeNames,
|
|
175
|
+
};
|
|
162
176
|
}
|
|
163
177
|
|
|
164
178
|
const params: any = {
|
|
@@ -205,12 +219,46 @@ export function createScanBuilder<Model>(
|
|
|
205
219
|
return params;
|
|
206
220
|
},
|
|
207
221
|
|
|
222
|
+
/**
|
|
223
|
+
* ⚠️ Returns only the FIRST page of results. DynamoDB caps each Scan
|
|
224
|
+
* response at ~1MB (or `Limit` if set). If the matching set is larger,
|
|
225
|
+
* the remaining items are silently dropped.
|
|
226
|
+
*
|
|
227
|
+
* Use {@link executeWithPagination} when you need to drive pagination
|
|
228
|
+
* yourself, or {@link iterate} to walk every matching item lazily.
|
|
229
|
+
*/
|
|
208
230
|
async execute() {
|
|
209
231
|
const params = build().dbParams();
|
|
210
232
|
const response = await client.send(new ScanCommand(params));
|
|
211
233
|
logger?.log('ScanCommand', params, response);
|
|
212
234
|
return (response.Items || []) as Model[];
|
|
213
235
|
},
|
|
236
|
+
|
|
237
|
+
async executeWithPagination(): Promise<ScanResult<Model>> {
|
|
238
|
+
const params = build().dbParams();
|
|
239
|
+
const response = await client.send(new ScanCommand(params));
|
|
240
|
+
logger?.log('ScanCommand', params, response);
|
|
241
|
+
return {
|
|
242
|
+
items: (response.Items ?? []) as Model[],
|
|
243
|
+
lastEvaluatedKey: response.LastEvaluatedKey,
|
|
244
|
+
count: response.Count,
|
|
245
|
+
scannedCount: response.ScannedCount,
|
|
246
|
+
};
|
|
247
|
+
},
|
|
248
|
+
|
|
249
|
+
async *iterate(): AsyncIterableIterator<Model> {
|
|
250
|
+
const baseParams = build().dbParams();
|
|
251
|
+
let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
|
|
252
|
+
do {
|
|
253
|
+
const params = { ...baseParams, ExclusiveStartKey: cursor };
|
|
254
|
+
const response = await client.send(new ScanCommand(params));
|
|
255
|
+
logger?.log('ScanCommand', params, response);
|
|
256
|
+
for (const item of (response.Items ?? []) as Model[]) {
|
|
257
|
+
yield item;
|
|
258
|
+
}
|
|
259
|
+
cursor = response.LastEvaluatedKey;
|
|
260
|
+
} while (cursor);
|
|
261
|
+
},
|
|
214
262
|
});
|
|
215
263
|
|
|
216
264
|
return build();
|
|
@@ -1,5 +1,20 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
1
2
|
import { ExecutableBuilder, AttrBuilder, OpBuilder, Condition } from '../shared';
|
|
2
3
|
|
|
4
|
+
/**
|
|
5
|
+
* A single page of scan results plus metadata for manual pagination.
|
|
6
|
+
*/
|
|
7
|
+
export interface ScanResult<Model> {
|
|
8
|
+
/** Items returned in this page. */
|
|
9
|
+
items: Model[];
|
|
10
|
+
/** Cursor for the next page; `undefined` when there are no more results. */
|
|
11
|
+
lastEvaluatedKey?: Record<string, any>;
|
|
12
|
+
/** Number of items returned after applying the filter. */
|
|
13
|
+
count?: number;
|
|
14
|
+
/** Number of items examined before applying the filter. */
|
|
15
|
+
scannedCount?: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
3
18
|
/**
|
|
4
19
|
* Builder for DynamoDB Scan operations.
|
|
5
20
|
* Scans the entire table or index without requiring key conditions.
|
|
@@ -46,4 +61,28 @@ export interface ScanBuilder<Model> extends ExecutableBuilder<Model[]> {
|
|
|
46
61
|
* Returns a new immutable builder.
|
|
47
62
|
*/
|
|
48
63
|
segment(segmentNumber: number, totalSegments: number): ScanBuilder<Model>;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Executes a single Scan request and returns the page along with the
|
|
67
|
+
* `lastEvaluatedKey` cursor and counts. Use this when you want to drive
|
|
68
|
+
* pagination yourself (e.g. expose a cursor to a client).
|
|
69
|
+
*/
|
|
70
|
+
executeWithPagination(): Promise<ScanResult<Model>>;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Returns an async iterator that paginates internally and yields one item at
|
|
74
|
+
* a time. Memory stays at one page; you can `break` out of the loop early.
|
|
75
|
+
*
|
|
76
|
+
* `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
|
|
77
|
+
* cap, not a total cap. To cap the total, count yourself inside the loop:
|
|
78
|
+
*
|
|
79
|
+
* ```ts
|
|
80
|
+
* let n = 0;
|
|
81
|
+
* for await (const item of builder.iterate()) {
|
|
82
|
+
* if (n++ >= 1000) break;
|
|
83
|
+
* process(item);
|
|
84
|
+
* }
|
|
85
|
+
* ```
|
|
86
|
+
*/
|
|
87
|
+
iterate(): AsyncIterableIterator<Model>;
|
|
49
88
|
}
|