@ftschopp/dynatable-core 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/dist/builders/batch-get/create-batch-get-builder.d.ts.map +1 -1
  3. package/dist/builders/batch-get/create-batch-get-builder.js +8 -3
  4. package/dist/builders/get/create-get-builder.d.ts.map +1 -1
  5. package/dist/builders/get/create-get-builder.js +2 -19
  6. package/dist/builders/query/create-query-builder.d.ts.map +1 -1
  7. package/dist/builders/query/create-query-builder.js +34 -3
  8. package/dist/builders/query/types.d.ts +16 -0
  9. package/dist/builders/query/types.d.ts.map +1 -1
  10. package/dist/builders/scan/create-scan-builder.d.ts.map +1 -1
  11. package/dist/builders/scan/create-scan-builder.js +42 -2
  12. package/dist/builders/scan/types.d.ts +35 -0
  13. package/dist/builders/scan/types.d.ts.map +1 -1
  14. package/dist/builders/shared/index.d.ts +1 -0
  15. package/dist/builders/shared/index.d.ts.map +1 -1
  16. package/dist/builders/shared/index.js +1 -0
  17. package/dist/builders/shared/projection.d.ts +15 -0
  18. package/dist/builders/shared/projection.d.ts.map +1 -0
  19. package/dist/builders/shared/projection.js +26 -0
  20. package/package.json +1 -1
  21. package/src/builders/batch-get/create-batch-get-builder.test.ts +30 -2
  22. package/src/builders/batch-get/create-batch-get-builder.ts +10 -3
  23. package/src/builders/get/create-get-builder.ts +1 -24
  24. package/src/builders/query/create-query-builder.test.ts +205 -1
  25. package/src/builders/query/create-query-builder.ts +40 -5
  26. package/src/builders/query/types.ts +17 -0
  27. package/src/builders/scan/create-scan-builder.test.ts +162 -6
  28. package/src/builders/scan/create-scan-builder.ts +52 -4
  29. package/src/builders/scan/types.ts +39 -0
  30. package/src/builders/shared/index.ts +1 -0
  31. package/src/builders/shared/projection.ts +28 -0
  32. package/tests/integration/instagram-clone.integration.test.ts +23 -4
@@ -60,7 +60,13 @@ describe('QueryBuilder - Pagination', () => {
60
60
  expect(params.ExclusiveStartKey).toEqual(startKey);
61
61
  expect(params.Limit).toBe(10);
62
62
  expect(params.ScanIndexForward).toBe(false);
63
- expect(params.ProjectionExpression).toBe('name, age');
63
+ expect(params.ProjectionExpression).toBe('#name, #age');
64
+ expect(params.ExpressionAttributeNames).toEqual(
65
+ expect.objectContaining({
66
+ '#name': 'name',
67
+ '#age': 'age',
68
+ })
69
+ );
64
70
  });
65
71
 
66
72
  test('should not include ExclusiveStartKey if not set', () => {
@@ -227,6 +233,126 @@ describe('QueryBuilder - Pagination', () => {
227
233
  expect(builder2.dbParams().ExclusiveStartKey).toEqual(startKey);
228
234
  });
229
235
  });
236
+
237
+ describe('iterate method', () => {
238
+ test('walks every page transparently and yields items in order', async () => {
239
+ const page1 = [
240
+ { pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
241
+ { pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
242
+ ];
243
+ const page2 = [
244
+ { pk: 'USER#3', sk: 'USER#3', name: 'User 3' },
245
+ { pk: 'USER#4', sk: 'USER#4', name: 'User 4' },
246
+ ];
247
+ const page3 = [{ pk: 'USER#5', sk: 'USER#5', name: 'User 5' }];
248
+
249
+ ddbMock
250
+ .on(QueryCommand)
251
+ .resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
252
+ .resolvesOnce({ Items: page2, LastEvaluatedKey: { pk: 'USER#4', sk: 'USER#4' } })
253
+ .resolvesOnce({ Items: page3 });
254
+
255
+ const collected: TestModel[] = [];
256
+ for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
257
+ .where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
258
+ .iterate()) {
259
+ collected.push(item);
260
+ }
261
+
262
+ expect(collected).toHaveLength(5);
263
+ expect(collected.map((u) => u.name)).toEqual([
264
+ 'User 1',
265
+ 'User 2',
266
+ 'User 3',
267
+ 'User 4',
268
+ 'User 5',
269
+ ]);
270
+ expect(ddbMock.calls()).toHaveLength(3);
271
+ });
272
+
273
+ test('forwards ExclusiveStartKey from each response into the next call', async () => {
274
+ const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
275
+ const cursor2 = { pk: 'USER#4', sk: 'USER#4' };
276
+
277
+ ddbMock
278
+ .on(QueryCommand)
279
+ .resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
280
+ .resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }], LastEvaluatedKey: cursor2 })
281
+ .resolvesOnce({ Items: [{ pk: 'USER#5', sk: 'USER#5' }] });
282
+
283
+ const iterator = createQueryBuilder<TestModel>(tableName, client, testModel)
284
+ .where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
285
+ .iterate();
286
+
287
+ // Consume the iterator
288
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
289
+ for await (const _ of iterator) {
290
+ // drain
291
+ }
292
+
293
+ const calls = ddbMock.calls();
294
+ expect(calls).toHaveLength(3);
295
+ expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
296
+ expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
297
+ expect((calls[2]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor2);
298
+ });
299
+
300
+ test('starts from the user-provided cursor on the first call', async () => {
301
+ const startKey = { pk: 'USER#10', sk: 'USER#10' };
302
+
303
+ ddbMock.on(QueryCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
304
+
305
+ const iterator = createQueryBuilder<TestModel>(tableName, client, testModel)
306
+ .where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
307
+ .startFrom(startKey)
308
+ .iterate();
309
+
310
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
311
+ for await (const _ of iterator) {
312
+ // drain
313
+ }
314
+
315
+ const calls = ddbMock.calls();
316
+ expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
317
+ });
318
+
319
+ test('break out of the loop stops further DynamoDB calls', async () => {
320
+ ddbMock
321
+ .on(QueryCommand)
322
+ .resolvesOnce({
323
+ Items: [
324
+ { pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
325
+ { pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
326
+ ],
327
+ LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
328
+ });
329
+
330
+ const collected: TestModel[] = [];
331
+ for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
332
+ .where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
333
+ .iterate()) {
334
+ collected.push(item);
335
+ if (collected.length >= 1) break;
336
+ }
337
+
338
+ expect(collected).toHaveLength(1);
339
+ expect(ddbMock.calls()).toHaveLength(1);
340
+ });
341
+
342
+ test('handles an empty result without making extra calls', async () => {
343
+ ddbMock.on(QueryCommand).resolves({ Items: [] });
344
+
345
+ const collected: TestModel[] = [];
346
+ for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
347
+ .where((attr, op) => op.beginsWith(attr.pk, 'NONE#'))
348
+ .iterate()) {
349
+ collected.push(item);
350
+ }
351
+
352
+ expect(collected).toEqual([]);
353
+ expect(ddbMock.calls()).toHaveLength(1);
354
+ });
355
+ });
230
356
  });
231
357
 
232
358
  describe('QueryBuilder - GSI key recognition', () => {
@@ -502,3 +628,81 @@ describe('QueryBuilder - entity type auto filter', () => {
502
628
  expect(params.ExpressionAttributeValues?.[':_type']).toBe('AirportPersonnel');
503
629
  });
504
630
  });
631
+
632
+ describe('QueryBuilder - projection placeholders', () => {
633
+ const client = new DynamoDBClient({});
634
+ const tableName = 'TestTable';
635
+
636
+ interface UserModel {
637
+ pk: string;
638
+ sk: string;
639
+ name?: string;
640
+ status?: string;
641
+ age?: number;
642
+ type?: string;
643
+ }
644
+
645
+ const userModel: ModelDefinition = {
646
+ key: {
647
+ PK: { type: String, value: 'USER#${username}' },
648
+ SK: { type: String, value: 'USER#${username}' },
649
+ },
650
+ attributes: {
651
+ username: { type: String, required: true },
652
+ name: { type: String },
653
+ status: { type: String },
654
+ age: { type: Number },
655
+ type: { type: String },
656
+ },
657
+ };
658
+
659
+ test('projects reserved DynamoDB words via #-placeholders', () => {
660
+ const params = createQueryBuilder<UserModel>(tableName, client, userModel)
661
+ .where((attr, op) => op.eq(attr.pk, 'USER#alice'))
662
+ .select(['name', 'status', 'type'])
663
+ .dbParams();
664
+
665
+ expect(params.ProjectionExpression).toBe('#name, #status, #type');
666
+ expect(params.ExpressionAttributeNames).toEqual(
667
+ expect.objectContaining({
668
+ '#name': 'name',
669
+ '#status': 'status',
670
+ '#type': 'type',
671
+ })
672
+ );
673
+ });
674
+
675
+ test('merges projection names with key/filter names without clobbering', () => {
676
+ const params = createQueryBuilder<UserModel & { username: string }>(
677
+ tableName,
678
+ client,
679
+ userModel
680
+ )
681
+ .where((attr, op) => op.and(op.eq(attr.username, 'alice'), op.gt(attr.age, 18)))
682
+ .select(['name', 'status'])
683
+ .dbParams();
684
+
685
+ // Projection placeholders (#name, #status) merged with key (#PK from
686
+ // the username→PK template rewrite) and filter (#age) — none clobbered.
687
+ expect(params.ExpressionAttributeNames).toEqual(
688
+ expect.objectContaining({
689
+ '#name': 'name',
690
+ '#status': 'status',
691
+ '#PK': 'PK',
692
+ '#age': 'age',
693
+ })
694
+ );
695
+ });
696
+
697
+ test('shares the same placeholder when an attribute is both projected and filtered (no key duplication)', () => {
698
+ const params = createQueryBuilder<UserModel>(tableName, client, userModel)
699
+ .where((attr, op) => op.and(op.eq(attr.pk, 'USER#alice'), op.eq(attr.status, 'active')))
700
+ .select(['name', 'status'])
701
+ .dbParams();
702
+
703
+ // The map has #status exactly once, mapping to "status".
704
+ expect(params.ExpressionAttributeNames!['#status']).toBe('status');
705
+ expect(params.ProjectionExpression).toBe('#name, #status');
706
+ expect(params.FilterExpression).toMatch(/#status = :status_\d+/);
707
+ });
708
+ });
@@ -4,6 +4,7 @@ import { QueryCommand, QueryCommandInput } from '@aws-sdk/lib-dynamodb';
4
4
  import { Condition, AttrBuilder, AttrRef } from '../shared';
5
5
  import { createOpBuilder } from '../shared/operators';
6
6
  import { buildExpression } from '../shared/conditions';
7
+ import { buildProjectionExpression } from '../shared/projection';
7
8
  import { QueryBuilder, QueryExecutor, QueryResult } from './types';
8
9
  import { ModelDefinition } from '../../core/types';
9
10
  import { extractTemplateVars } from '../../utils/model-utils';
@@ -294,7 +295,7 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
294
295
  }
295
296
 
296
297
  // Merge attribute names and values from both expressions
297
- const allNames = {
298
+ const allNames: Record<string, string> = {
298
299
  ...(keyExpr.names ?? {}),
299
300
  ...(filterExpr.names ?? {}),
300
301
  };
@@ -303,6 +304,19 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
303
304
  ...(filterExpr.values ?? {}),
304
305
  };
305
306
 
307
+ // Build ProjectionExpression with placeholders so reserved words
308
+ // (name, date, status, type, …) don't blow up at DynamoDB.
309
+ // Idempotent merge: filter/key and projection placeholders both map
310
+ // `#name → name`, so collisions resolve to the same value.
311
+ let projectionExpression: string | undefined;
312
+ if (state.projection && state.projection.length > 0) {
313
+ const proj = buildProjectionExpression(
314
+ (state.projection as (keyof Model)[]).map((a) => String(a))
315
+ );
316
+ projectionExpression = proj.ProjectionExpression;
317
+ Object.assign(allNames, proj.ExpressionAttributeNames);
318
+ }
319
+
306
320
  return {
307
321
  TableName: state.tableName,
308
322
  ...(keyExpr.expression && {
@@ -322,10 +336,9 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
322
336
  ...(state.scanForward !== undefined && {
323
337
  ScanIndexForward: state.scanForward,
324
338
  }),
325
- ...(state.projection &&
326
- state.projection.length > 0 && {
327
- ProjectionExpression: state.projection.join(', '),
328
- }),
339
+ ...(projectionExpression && {
340
+ ProjectionExpression: projectionExpression,
341
+ }),
329
342
  ...(state.consistentReadEnabled && { ConsistentRead: true }),
330
343
  ...(state.exclusiveStartKey && {
331
344
  ExclusiveStartKey: state.exclusiveStartKey,
@@ -333,6 +346,14 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
333
346
  };
334
347
  },
335
348
 
349
+ /**
350
+ * ⚠️ Returns only the FIRST page of results. DynamoDB caps each Query
351
+ * response at ~1MB (or `Limit` if set). If the matching set is larger,
352
+ * the remaining items are silently dropped.
353
+ *
354
+ * Use {@link executeWithPagination} when you need to drive pagination
355
+ * yourself, or {@link iterate} to walk every matching item lazily.
356
+ */
336
357
  async execute(): Promise<Model[]> {
337
358
  const params = this.dbParams();
338
359
  const result = await state.client.send(new QueryCommand(params));
@@ -351,6 +372,20 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
351
372
  scannedCount: result.ScannedCount,
352
373
  };
353
374
  },
375
+
376
+ async *iterate(): AsyncIterableIterator<Model> {
377
+ const baseParams = this.dbParams();
378
+ let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
379
+ do {
380
+ const params = { ...baseParams, ExclusiveStartKey: cursor };
381
+ const result = await state.client.send(new QueryCommand(params));
382
+ state.logger?.log('QueryCommand', params, result);
383
+ for (const item of (result.Items ?? []) as unknown as Model[]) {
384
+ yield item;
385
+ }
386
+ cursor = result.LastEvaluatedKey;
387
+ } while (cursor);
388
+ },
354
389
  };
355
390
  }
356
391
 
@@ -74,6 +74,23 @@ export interface QueryExecutor<Model> extends ExecutableBuilder<Model[]> {
74
74
  * Executes the query and returns result with pagination metadata
75
75
  */
76
76
  executeWithPagination(): Promise<QueryResult<Model>>;
77
+
78
+ /**
79
+ * Returns an async iterator that paginates internally and yields one item at
80
+ * a time. Memory stays at one page; you can `break` out of the loop early.
81
+ *
82
+ * `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
83
+ * cap, not a total cap. To cap the total, count yourself inside the loop:
84
+ *
85
+ * ```ts
86
+ * let n = 0;
87
+ * for await (const item of builder.iterate()) {
88
+ * if (n++ >= 1000) break;
89
+ * process(item);
90
+ * }
91
+ * ```
92
+ */
93
+ iterate(): AsyncIterableIterator<Model>;
77
94
  }
78
95
 
79
96
  /**
@@ -1,7 +1,11 @@
1
1
  /* eslint-disable @typescript-eslint/no-explicit-any */
2
2
  import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
3
+ import { ScanCommand } from '@aws-sdk/lib-dynamodb';
4
+ import { mockClient } from 'aws-sdk-client-mock';
3
5
  import { createScanBuilder } from './create-scan-builder';
4
6
 
7
+ const ddbMock = mockClient(DynamoDBClient);
8
+
5
9
  describe('ScanBuilder', () => {
6
10
  const client = new DynamoDBClient({});
7
11
  const tableName = 'TestTable';
@@ -73,17 +77,52 @@ describe('ScanBuilder', () => {
73
77
  .select(['name', 'age', 'status'])
74
78
  .dbParams();
75
79
 
76
- expect(params.ProjectionExpression).toBe('name, age, status');
80
+ expect(params.ProjectionExpression).toBe('#name, #age, #status');
81
+ expect(params.ExpressionAttributeNames).toEqual({
82
+ '#name': 'name',
83
+ '#age': 'age',
84
+ '#status': 'status',
85
+ });
77
86
  });
78
87
 
79
- test('should build params with select and filter', () => {
88
+ test('should build params with select and filter, merging ExpressionAttributeNames', () => {
80
89
  const params = createScanBuilder<TestModel>(tableName, client)
81
90
  .select(['name', 'age'])
82
- .filter((attr, op) => op.gt(attr.age, 18))
91
+ .filter((attr, op) => op.gt(attr.score, 50))
92
+ .dbParams();
93
+
94
+ expect(params.ProjectionExpression).toBe('#name, #age');
95
+ expect(params.FilterExpression).toMatch(/#score > :score_\d+/);
96
+ // Merged names from filter (#score) + projection (#name, #age)
97
+ expect(params.ExpressionAttributeNames).toEqual({
98
+ '#name': 'name',
99
+ '#age': 'age',
100
+ '#score': 'score',
101
+ });
102
+ });
103
+
104
+ test('placeholders survive when projected attribute is also referenced in the filter (no key collision)', () => {
105
+ const params = createScanBuilder<TestModel>(tableName, client)
106
+ .select(['name', 'age'])
107
+ .filter((attr, op) => op.eq(attr.name, 'alice'))
108
+ .dbParams();
109
+
110
+ expect(params.ProjectionExpression).toBe('#name, #age');
111
+ // #name resolves to "name" — filter and projection share the placeholder.
112
+ expect(params.ExpressionAttributeNames!['#name']).toBe('name');
113
+ expect(params.ExpressionAttributeNames!['#age']).toBe('age');
114
+ });
115
+
116
+ test('reserved DynamoDB words like "name", "date", "status", "type" can be projected', () => {
117
+ const params = createScanBuilder<TestModel>(tableName, client)
118
+ .select(['name', 'status'])
83
119
  .dbParams();
84
120
 
85
- expect(params.ProjectionExpression).toBe('name, age');
86
- expect(params.FilterExpression).toMatch(/#age > :age_\d+/);
121
+ expect(params.ProjectionExpression).toBe('#name, #status');
122
+ expect(params.ExpressionAttributeNames).toEqual({
123
+ '#name': 'name',
124
+ '#status': 'status',
125
+ });
87
126
  });
88
127
  });
89
128
 
@@ -180,7 +219,7 @@ describe('ScanBuilder', () => {
180
219
 
181
220
  expect(params.TableName).toBe(tableName);
182
221
  expect(params.FilterExpression).toMatch(/\(#age > :age_\d+\) AND \(#status = :status_\d+\)/);
183
- expect(params.ProjectionExpression).toBe('name, age, status');
222
+ expect(params.ProjectionExpression).toBe('#name, #age, #status');
184
223
  expect(params.Limit).toBe(100);
185
224
  expect(params.ConsistentRead).toBe(true);
186
225
  expect(params.IndexName).toBe('GSI1');
@@ -257,4 +296,121 @@ describe('ScanBuilder', () => {
257
296
  expect(Object.values(params.ExpressionAttributeValues || {})).toContain(65);
258
297
  });
259
298
  });
299
+
300
+ describe('executeWithPagination', () => {
301
+ beforeEach(() => {
302
+ ddbMock.reset();
303
+ });
304
+
305
+ test('returns items, lastEvaluatedKey and counts', async () => {
306
+ const items = [
307
+ { pk: 'USER#alice', sk: 'USER#alice', name: 'Alice' },
308
+ { pk: 'USER#bob', sk: 'USER#bob', name: 'Bob' },
309
+ ];
310
+ const lastKey = { pk: 'USER#bob', sk: 'USER#bob' };
311
+
312
+ ddbMock.on(ScanCommand).resolves({
313
+ Items: items,
314
+ LastEvaluatedKey: lastKey,
315
+ Count: 2,
316
+ ScannedCount: 5,
317
+ });
318
+
319
+ const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
320
+
321
+ expect(result.items).toEqual(items);
322
+ expect(result.lastEvaluatedKey).toEqual(lastKey);
323
+ expect(result.count).toBe(2);
324
+ expect(result.scannedCount).toBe(5);
325
+ });
326
+
327
+ test('returns undefined lastEvaluatedKey on the final page', async () => {
328
+ ddbMock.on(ScanCommand).resolves({ Items: [], Count: 0, ScannedCount: 0 });
329
+
330
+ const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
331
+
332
+ expect(result.items).toEqual([]);
333
+ expect(result.lastEvaluatedKey).toBeUndefined();
334
+ });
335
+ });
336
+
337
+ describe('iterate', () => {
338
+ beforeEach(() => {
339
+ ddbMock.reset();
340
+ });
341
+
342
+ test('walks every page transparently and yields items in order', async () => {
343
+ const page1 = [
344
+ { pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
345
+ { pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
346
+ ];
347
+ const page2 = [{ pk: 'USER#3', sk: 'USER#3', name: 'User 3' }];
348
+
349
+ ddbMock
350
+ .on(ScanCommand)
351
+ .resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
352
+ .resolvesOnce({ Items: page2 });
353
+
354
+ const collected: TestModel[] = [];
355
+ for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
356
+ collected.push(item);
357
+ }
358
+
359
+ expect(collected.map((u) => u.name)).toEqual(['User 1', 'User 2', 'User 3']);
360
+ expect(ddbMock.calls()).toHaveLength(2);
361
+ });
362
+
363
+ test('forwards LastEvaluatedKey from each response into the next call', async () => {
364
+ const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
365
+
366
+ ddbMock
367
+ .on(ScanCommand)
368
+ .resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
369
+ .resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }] });
370
+
371
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
372
+ for await (const _ of createScanBuilder<TestModel>(tableName, client).iterate()) {
373
+ // drain
374
+ }
375
+
376
+ const calls = ddbMock.calls();
377
+ expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
378
+ expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
379
+ });
380
+
381
+ test('starts from the user-provided cursor on the first call', async () => {
382
+ const startKey = { pk: 'USER#10', sk: 'USER#10' };
383
+
384
+ ddbMock.on(ScanCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
385
+
386
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
387
+ for await (const _ of createScanBuilder<TestModel>(tableName, client)
388
+ .startFrom(startKey)
389
+ .iterate()) {
390
+ // drain
391
+ }
392
+
393
+ const calls = ddbMock.calls();
394
+ expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
395
+ });
396
+
397
+ test('break out of the loop stops further DynamoDB calls', async () => {
398
+ ddbMock.on(ScanCommand).resolvesOnce({
399
+ Items: [
400
+ { pk: 'USER#1', sk: 'USER#1' },
401
+ { pk: 'USER#2', sk: 'USER#2' },
402
+ ],
403
+ LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
404
+ });
405
+
406
+ const collected: TestModel[] = [];
407
+ for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
408
+ collected.push(item);
409
+ if (collected.length >= 1) break;
410
+ }
411
+
412
+ expect(collected).toHaveLength(1);
413
+ expect(ddbMock.calls()).toHaveLength(1);
414
+ });
415
+ });
260
416
  });
@@ -1,8 +1,14 @@
1
1
  /* eslint-disable @typescript-eslint/no-explicit-any */
2
2
  import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
3
3
  import { ScanCommand } from '@aws-sdk/lib-dynamodb';
4
- import { buildExpression, AttrBuilder, Condition, createOpBuilder } from '../shared';
5
- import { ScanBuilder } from './types';
4
+ import {
5
+ buildExpression,
6
+ AttrBuilder,
7
+ Condition,
8
+ createOpBuilder,
9
+ buildProjectionExpression,
10
+ } from '../shared';
11
+ import { ScanBuilder, ScanResult } from './types';
6
12
  import { DynamoDBLogger } from '../../utils/dynamodb-logger';
7
13
 
8
14
  /**
@@ -155,10 +161,18 @@ export function createScanBuilder<Model>(
155
161
  expressionAttributeValues = result.values;
156
162
  }
157
163
 
158
- // Build ProjectionExpression
164
+ // Build ProjectionExpression with placeholders so reserved words
165
+ // (name, date, status, type, …) don't blow up at DynamoDB.
159
166
  let projectionExpression = '';
160
167
  if (projectionAttrs.length > 0) {
161
- projectionExpression = projectionAttrs.map((attr) => String(attr)).join(', ');
168
+ const proj = buildProjectionExpression(projectionAttrs.map((attr) => String(attr)));
169
+ projectionExpression = proj.ProjectionExpression;
170
+ // Idempotent merge: filter and projection placeholders both map
171
+ // `#name → name`, so collisions resolve to the same value.
172
+ expressionAttributeNames = {
173
+ ...proj.ExpressionAttributeNames,
174
+ ...expressionAttributeNames,
175
+ };
162
176
  }
163
177
 
164
178
  const params: any = {
@@ -205,12 +219,46 @@ export function createScanBuilder<Model>(
205
219
  return params;
206
220
  },
207
221
 
222
+ /**
223
+ * ⚠️ Returns only the FIRST page of results. DynamoDB caps each Scan
224
+ * response at ~1MB (or `Limit` if set). If the matching set is larger,
225
+ * the remaining items are silently dropped.
226
+ *
227
+ * Use {@link executeWithPagination} when you need to drive pagination
228
+ * yourself, or {@link iterate} to walk every matching item lazily.
229
+ */
208
230
  async execute() {
209
231
  const params = build().dbParams();
210
232
  const response = await client.send(new ScanCommand(params));
211
233
  logger?.log('ScanCommand', params, response);
212
234
  return (response.Items || []) as Model[];
213
235
  },
236
+
237
+ async executeWithPagination(): Promise<ScanResult<Model>> {
238
+ const params = build().dbParams();
239
+ const response = await client.send(new ScanCommand(params));
240
+ logger?.log('ScanCommand', params, response);
241
+ return {
242
+ items: (response.Items ?? []) as Model[],
243
+ lastEvaluatedKey: response.LastEvaluatedKey,
244
+ count: response.Count,
245
+ scannedCount: response.ScannedCount,
246
+ };
247
+ },
248
+
249
+ async *iterate(): AsyncIterableIterator<Model> {
250
+ const baseParams = build().dbParams();
251
+ let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
252
+ do {
253
+ const params = { ...baseParams, ExclusiveStartKey: cursor };
254
+ const response = await client.send(new ScanCommand(params));
255
+ logger?.log('ScanCommand', params, response);
256
+ for (const item of (response.Items ?? []) as Model[]) {
257
+ yield item;
258
+ }
259
+ cursor = response.LastEvaluatedKey;
260
+ } while (cursor);
261
+ },
214
262
  });
215
263
 
216
264
  return build();
@@ -1,5 +1,20 @@
1
+ /* eslint-disable @typescript-eslint/no-explicit-any */
1
2
  import { ExecutableBuilder, AttrBuilder, OpBuilder, Condition } from '../shared';
2
3
 
4
+ /**
5
+ * A single page of scan results plus metadata for manual pagination.
6
+ */
7
+ export interface ScanResult<Model> {
8
+ /** Items returned in this page. */
9
+ items: Model[];
10
+ /** Cursor for the next page; `undefined` when there are no more results. */
11
+ lastEvaluatedKey?: Record<string, any>;
12
+ /** Number of items returned after applying the filter. */
13
+ count?: number;
14
+ /** Number of items examined before applying the filter. */
15
+ scannedCount?: number;
16
+ }
17
+
3
18
  /**
4
19
  * Builder for DynamoDB Scan operations.
5
20
  * Scans the entire table or index without requiring key conditions.
@@ -46,4 +61,28 @@ export interface ScanBuilder<Model> extends ExecutableBuilder<Model[]> {
46
61
  * Returns a new immutable builder.
47
62
  */
48
63
  segment(segmentNumber: number, totalSegments: number): ScanBuilder<Model>;
64
+
65
+ /**
66
+ * Executes a single Scan request and returns the page along with the
67
+ * `lastEvaluatedKey` cursor and counts. Use this when you want to drive
68
+ * pagination yourself (e.g. expose a cursor to a client).
69
+ */
70
+ executeWithPagination(): Promise<ScanResult<Model>>;
71
+
72
+ /**
73
+ * Returns an async iterator that paginates internally and yields one item at
74
+ * a time. Memory stays at one page; you can `break` out of the loop early.
75
+ *
76
+ * `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
77
+ * cap, not a total cap. To cap the total, count yourself inside the loop:
78
+ *
79
+ * ```ts
80
+ * let n = 0;
81
+ * for await (const item of builder.iterate()) {
82
+ * if (n++ >= 1000) break;
83
+ * process(item);
84
+ * }
85
+ * ```
86
+ */
87
+ iterate(): AsyncIterableIterator<Model>;
49
88
  }
@@ -1,3 +1,4 @@
1
1
  export * from './types';
2
2
  export * from './operators';
3
3
  export * from './conditions';
4
+ export * from './projection';