@ftschopp/dynatable-core 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,10 @@
1
+ # @ftschopp/dynatable-core [1.4.0](https://github.com/ftschopp/dynatable/compare/@ftschopp/dynatable-core@1.3.0...@ftschopp/dynatable-core@1.4.0) (2026-05-08)
2
+
3
+
4
+ ### Features
5
+
6
+ * **core:** add iterate() async iterator and Scan executeWithPagination ([#21](https://github.com/ftschopp/dynatable/issues/21)) ([c31d458](https://github.com/ftschopp/dynatable/commit/c31d45861e92ba0c52240128084939128a6e4acc)), closes [#4](https://github.com/ftschopp/dynatable/issues/4)
7
+
1
8
  # @ftschopp/dynatable-core [1.3.0](https://github.com/ftschopp/dynatable/compare/@ftschopp/dynatable-core@1.2.5...@ftschopp/dynatable-core@1.3.0) (2026-05-07)
2
9
 
3
10
 
@@ -1 +1 @@
1
- {"version":3,"file":"create-query-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/query/create-query-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAK1D,OAAO,EAAE,YAAY,EAA8B,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AA2V7D;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EACtC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,cAAc,EACtB,KAAK,CAAC,EAAE,eAAe,EACvB,MAAM,CAAC,EAAE,cAAc,EACvB,UAAU,CAAC,EAAE,MAAM,GAClB,YAAY,CAAC,KAAK,CAAC,CA6BrB"}
1
+ {"version":3,"file":"create-query-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/query/create-query-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAK1D,OAAO,EAAE,YAAY,EAA8B,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAiX7D;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EACtC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,cAAc,EACtB,KAAK,CAAC,EAAE,eAAe,EACvB,MAAM,CAAC,EAAE,cAAc,EACvB,UAAU,CAAC,EAAE,MAAM,GAClB,YAAY,CAAC,KAAK,CAAC,CA6BrB"}
@@ -255,6 +255,14 @@ function createQueryExecutor(state) {
255
255
  }),
256
256
  };
257
257
  },
258
+ /**
259
+ * ⚠️ Returns only the FIRST page of results. DynamoDB caps each Query
260
+ * response at ~1MB (or `Limit` if set). If the matching set is larger,
261
+ * the remaining items are silently dropped.
262
+ *
263
+ * Use {@link executeWithPagination} when you need to drive pagination
264
+ * yourself, or {@link iterate} to walk every matching item lazily.
265
+ */
258
266
  async execute() {
259
267
  const params = this.dbParams();
260
268
  const result = await state.client.send(new lib_dynamodb_1.QueryCommand(params));
@@ -272,6 +280,19 @@ function createQueryExecutor(state) {
272
280
  scannedCount: result.ScannedCount,
273
281
  };
274
282
  },
283
+ async *iterate() {
284
+ const baseParams = this.dbParams();
285
+ let cursor = baseParams.ExclusiveStartKey;
286
+ do {
287
+ const params = { ...baseParams, ExclusiveStartKey: cursor };
288
+ const result = await state.client.send(new lib_dynamodb_1.QueryCommand(params));
289
+ state.logger?.log('QueryCommand', params, result);
290
+ for (const item of (result.Items ?? [])) {
291
+ yield item;
292
+ }
293
+ cursor = result.LastEvaluatedKey;
294
+ } while (cursor);
295
+ },
275
296
  };
276
297
  }
277
298
  /**
@@ -60,6 +60,22 @@ export interface QueryExecutor<Model> extends ExecutableBuilder<Model[]> {
60
60
  * Executes the query and returns result with pagination metadata
61
61
  */
62
62
  executeWithPagination(): Promise<QueryResult<Model>>;
63
+ /**
64
+ * Returns an async iterator that paginates internally and yields one item at
65
+ * a time. Memory stays at one page; you can `break` out of the loop early.
66
+ *
67
+ * `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
68
+ * cap, not a total cap. To cap the total, count yourself inside the loop:
69
+ *
70
+ * ```ts
71
+ * let n = 0;
72
+ * for await (const item of builder.iterate()) {
73
+ * if (n++ >= 1000) break;
74
+ * process(item);
75
+ * }
76
+ * ```
77
+ */
78
+ iterate(): AsyncIterableIterator<Model>;
63
79
  }
64
80
  /**
65
81
  * Main Query Builder interface with type-safe where clause
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/query/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEjF;;GAEG;AACH,MAAM,WAAW,WAAW,CAAC,KAAK;IAChC;;OAEG;IACH,KAAK,EAAE,KAAK,EAAE,CAAC;IAEf;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEvC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,KAAK,CAAE,SAAQ,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACtE;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE3C;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAEzD;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAElD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAErD;;OAEG;IACH,cAAc,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;IAEvC;;OAEG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE1D;;OAEG;IACH,QAAQ,IAAI,GAAG,CAAC;IAEhB;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAE5B;;OAEG;IACH,qBAAqB,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;CACtD;AAED;;GAEG;AACH,MAAM,WAAW,YAAY,CAAC,KAAK;IACjC;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,SAAS,KAAK,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;CACzF"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/query/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEjF;;GAEG;AACH,MAAM,WAAW,WAAW,CAAC,KAAK;IAChC;;OAEG;IACH,KAAK,EAAE,KAAK,EAAE,CAAC;IAEf;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEvC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,KAAK,CAAE,SAAQ,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACtE;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE3C;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAEzD;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAElD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAErD;;OAEG;IACH,cAAc,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;IAEvC;;OAEG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE1D;;OAEG;IACH,QAAQ,IAAI,GAAG,CAAC;IAEhB;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAE5B;;OAEG;IACH,qBAAqB,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAErD;;;;;;;;;;;;;;OAcG;IACH,OAAO,IAAI,qBAAqB,CAAC,KAAK,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY,CAAC,KAAK;IACjC;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,SAAS,KAAK,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;CACzF"}
@@ -1 +1 @@
1
- {"version":3,"file":"create-scan-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/scan/create-scan-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAgC,SAAS,EAAmB,MAAM,WAAW,CAAC;AACrF,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EACrC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,SAAS,EAAO,EACzB,eAAe,GAAE,CAAC,MAAM,KAAK,CAAC,EAAO,EACrC,UAAU,CAAC,EAAE,MAAM,EACnB,gBAAgB,UAAQ,EACxB,SAAS,CAAC,EAAE,MAAM,EAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACvC,aAAa,CAAC,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,EAC1D,MAAM,CAAC,EAAE,cAAc,GACtB,WAAW,CAAC,KAAK,CAAC,CAmMpB"}
1
+ {"version":3,"file":"create-scan-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/scan/create-scan-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAgC,SAAS,EAAmB,MAAM,WAAW,CAAC;AACrF,OAAO,EAAE,WAAW,EAAc,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EACrC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,SAAS,EAAO,EACzB,eAAe,GAAE,CAAC,MAAM,KAAK,CAAC,EAAO,EACrC,UAAU,CAAC,EAAE,MAAM,EACnB,gBAAgB,UAAQ,EACxB,SAAS,CAAC,EAAE,MAAM,EAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACvC,aAAa,CAAC,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,EAC1D,MAAM,CAAC,EAAE,cAAc,GACtB,WAAW,CAAC,KAAK,CAAC,CAqOpB"}
@@ -92,12 +92,44 @@ function createScanBuilder(tableName, client, filters = [], projectionAttrs = []
92
92
  }
93
93
  return params;
94
94
  },
95
+ /**
96
+ * ⚠️ Returns only the FIRST page of results. DynamoDB caps each Scan
97
+ * response at ~1MB (or `Limit` if set). If the matching set is larger,
98
+ * the remaining items are silently dropped.
99
+ *
100
+ * Use {@link executeWithPagination} when you need to drive pagination
101
+ * yourself, or {@link iterate} to walk every matching item lazily.
102
+ */
95
103
  async execute() {
96
104
  const params = build().dbParams();
97
105
  const response = await client.send(new lib_dynamodb_1.ScanCommand(params));
98
106
  logger?.log('ScanCommand', params, response);
99
107
  return (response.Items || []);
100
108
  },
109
+ async executeWithPagination() {
110
+ const params = build().dbParams();
111
+ const response = await client.send(new lib_dynamodb_1.ScanCommand(params));
112
+ logger?.log('ScanCommand', params, response);
113
+ return {
114
+ items: (response.Items ?? []),
115
+ lastEvaluatedKey: response.LastEvaluatedKey,
116
+ count: response.Count,
117
+ scannedCount: response.ScannedCount,
118
+ };
119
+ },
120
+ async *iterate() {
121
+ const baseParams = build().dbParams();
122
+ let cursor = baseParams.ExclusiveStartKey;
123
+ do {
124
+ const params = { ...baseParams, ExclusiveStartKey: cursor };
125
+ const response = await client.send(new lib_dynamodb_1.ScanCommand(params));
126
+ logger?.log('ScanCommand', params, response);
127
+ for (const item of (response.Items ?? [])) {
128
+ yield item;
129
+ }
130
+ cursor = response.LastEvaluatedKey;
131
+ } while (cursor);
132
+ },
101
133
  });
102
134
  return build();
103
135
  }
@@ -1,4 +1,17 @@
1
1
  import { ExecutableBuilder, AttrBuilder, OpBuilder, Condition } from '../shared';
2
+ /**
3
+ * A single page of scan results plus metadata for manual pagination.
4
+ */
5
+ export interface ScanResult<Model> {
6
+ /** Items returned in this page. */
7
+ items: Model[];
8
+ /** Cursor for the next page; `undefined` when there are no more results. */
9
+ lastEvaluatedKey?: Record<string, any>;
10
+ /** Number of items returned after applying the filter. */
11
+ count?: number;
12
+ /** Number of items examined before applying the filter. */
13
+ scannedCount?: number;
14
+ }
2
15
  /**
3
16
  * Builder for DynamoDB Scan operations.
4
17
  * Scans the entire table or index without requiring key conditions.
@@ -39,5 +52,27 @@ export interface ScanBuilder<Model> extends ExecutableBuilder<Model[]> {
39
52
  * Returns a new immutable builder.
40
53
  */
41
54
  segment(segmentNumber: number, totalSegments: number): ScanBuilder<Model>;
55
+ /**
56
+ * Executes a single Scan request and returns the page along with the
57
+ * `lastEvaluatedKey` cursor and counts. Use this when you want to drive
58
+ * pagination yourself (e.g. expose a cursor to a client).
59
+ */
60
+ executeWithPagination(): Promise<ScanResult<Model>>;
61
+ /**
62
+ * Returns an async iterator that paginates internally and yields one item at
63
+ * a time. Memory stays at one page; you can `break` out of the loop early.
64
+ *
65
+ * `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
66
+ * cap, not a total cap. To cap the total, count yourself inside the loop:
67
+ *
68
+ * ```ts
69
+ * let n = 0;
70
+ * for await (const item of builder.iterate()) {
71
+ * if (n++ >= 1000) break;
72
+ * process(item);
73
+ * }
74
+ * ```
75
+ */
76
+ iterate(): AsyncIterableIterator<Model>;
42
77
  }
43
78
  //# sourceMappingURL=types.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/scan/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEjF;;;GAGG;AACH,MAAM,WAAW,WAAW,CAAC,KAAK,CAAE,SAAQ,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACpE;;;OAGG;IACH,MAAM,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,SAAS,KAAK,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEvF;;;OAGG;IACH,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEnD;;;OAGG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEzC;;;OAGG;IACH,cAAc,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC;IAErC;;;OAGG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAElD;;;OAGG;IACH,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEtE;;;OAGG;IACH,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;CAC3E"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/scan/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEjF;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,KAAK;IAC/B,mCAAmC;IACnC,KAAK,EAAE,KAAK,EAAE,CAAC;IACf,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACvC,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW,CAAC,KAAK,CAAE,SAAQ,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACpE;;;OAGG;IACH,MAAM,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,SAAS,KAAK,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEvF;;;OAGG;IACH,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEnD;;;OAGG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEzC;;;OAGG;IACH,cAAc,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC;IAErC;;;OAGG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAElD;;;OAGG;IACH,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEtE;;;OAGG;IACH,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAE1E;;;;OAIG;IACH,qBAAqB,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IAEpD;;;;;;;;;;;;;;OAcG;IACH,OAAO,IAAI,qBAAqB,CAAC,KAAK,CAAC,CAAC;CACzC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ftschopp/dynatable-core",
3
- "version": "1.3.0",
3
+ "version": "1.4.0",
4
4
  "description": "Core library for DynamoDB single table design",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -227,6 +227,126 @@ describe('QueryBuilder - Pagination', () => {
227
227
  expect(builder2.dbParams().ExclusiveStartKey).toEqual(startKey);
228
228
  });
229
229
  });
230
+
231
+ describe('iterate method', () => {
232
+ test('walks every page transparently and yields items in order', async () => {
233
+ const page1 = [
234
+ { pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
235
+ { pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
236
+ ];
237
+ const page2 = [
238
+ { pk: 'USER#3', sk: 'USER#3', name: 'User 3' },
239
+ { pk: 'USER#4', sk: 'USER#4', name: 'User 4' },
240
+ ];
241
+ const page3 = [{ pk: 'USER#5', sk: 'USER#5', name: 'User 5' }];
242
+
243
+ ddbMock
244
+ .on(QueryCommand)
245
+ .resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
246
+ .resolvesOnce({ Items: page2, LastEvaluatedKey: { pk: 'USER#4', sk: 'USER#4' } })
247
+ .resolvesOnce({ Items: page3 });
248
+
249
+ const collected: TestModel[] = [];
250
+ for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
251
+ .where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
252
+ .iterate()) {
253
+ collected.push(item);
254
+ }
255
+
256
+ expect(collected).toHaveLength(5);
257
+ expect(collected.map((u) => u.name)).toEqual([
258
+ 'User 1',
259
+ 'User 2',
260
+ 'User 3',
261
+ 'User 4',
262
+ 'User 5',
263
+ ]);
264
+ expect(ddbMock.calls()).toHaveLength(3);
265
+ });
266
+
267
+ test('forwards ExclusiveStartKey from each response into the next call', async () => {
268
+ const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
269
+ const cursor2 = { pk: 'USER#4', sk: 'USER#4' };
270
+
271
+ ddbMock
272
+ .on(QueryCommand)
273
+ .resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
274
+ .resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }], LastEvaluatedKey: cursor2 })
275
+ .resolvesOnce({ Items: [{ pk: 'USER#5', sk: 'USER#5' }] });
276
+
277
+ const iterator = createQueryBuilder<TestModel>(tableName, client, testModel)
278
+ .where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
279
+ .iterate();
280
+
281
+ // Consume the iterator
282
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
283
+ for await (const _ of iterator) {
284
+ // drain
285
+ }
286
+
287
+ const calls = ddbMock.calls();
288
+ expect(calls).toHaveLength(3);
289
+ expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
290
+ expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
291
+ expect((calls[2]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor2);
292
+ });
293
+
294
+ test('starts from the user-provided cursor on the first call', async () => {
295
+ const startKey = { pk: 'USER#10', sk: 'USER#10' };
296
+
297
+ ddbMock.on(QueryCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
298
+
299
+ const iterator = createQueryBuilder<TestModel>(tableName, client, testModel)
300
+ .where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
301
+ .startFrom(startKey)
302
+ .iterate();
303
+
304
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
305
+ for await (const _ of iterator) {
306
+ // drain
307
+ }
308
+
309
+ const calls = ddbMock.calls();
310
+ expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
311
+ });
312
+
313
+ test('break out of the loop stops further DynamoDB calls', async () => {
314
+ ddbMock
315
+ .on(QueryCommand)
316
+ .resolvesOnce({
317
+ Items: [
318
+ { pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
319
+ { pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
320
+ ],
321
+ LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
322
+ });
323
+
324
+ const collected: TestModel[] = [];
325
+ for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
326
+ .where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
327
+ .iterate()) {
328
+ collected.push(item);
329
+ if (collected.length >= 1) break;
330
+ }
331
+
332
+ expect(collected).toHaveLength(1);
333
+ expect(ddbMock.calls()).toHaveLength(1);
334
+ });
335
+
336
+ test('handles an empty result without making extra calls', async () => {
337
+ ddbMock.on(QueryCommand).resolves({ Items: [] });
338
+
339
+ const collected: TestModel[] = [];
340
+ for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
341
+ .where((attr, op) => op.beginsWith(attr.pk, 'NONE#'))
342
+ .iterate()) {
343
+ collected.push(item);
344
+ }
345
+
346
+ expect(collected).toEqual([]);
347
+ expect(ddbMock.calls()).toHaveLength(1);
348
+ });
349
+ });
230
350
  });
231
351
 
232
352
  describe('QueryBuilder - GSI key recognition', () => {
@@ -333,6 +333,14 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
333
333
  };
334
334
  },
335
335
 
336
+ /**
337
+ * ⚠️ Returns only the FIRST page of results. DynamoDB caps each Query
338
+ * response at ~1MB (or `Limit` if set). If the matching set is larger,
339
+ * the remaining items are silently dropped.
340
+ *
341
+ * Use {@link executeWithPagination} when you need to drive pagination
342
+ * yourself, or {@link iterate} to walk every matching item lazily.
343
+ */
336
344
  async execute(): Promise<Model[]> {
337
345
  const params = this.dbParams();
338
346
  const result = await state.client.send(new QueryCommand(params));
@@ -351,6 +359,20 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
351
359
  scannedCount: result.ScannedCount,
352
360
  };
353
361
  },
362
+
363
+ async *iterate(): AsyncIterableIterator<Model> {
364
+ const baseParams = this.dbParams();
365
+ let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
366
+ do {
367
+ const params = { ...baseParams, ExclusiveStartKey: cursor };
368
+ const result = await state.client.send(new QueryCommand(params));
369
+ state.logger?.log('QueryCommand', params, result);
370
+ for (const item of (result.Items ?? []) as unknown as Model[]) {
371
+ yield item;
372
+ }
373
+ cursor = result.LastEvaluatedKey;
374
+ } while (cursor);
375
+ },
354
376
  };
355
377
  }
356
378
 
@@ -74,6 +74,23 @@ export interface QueryExecutor<Model> extends ExecutableBuilder<Model[]> {
74
74
  * Executes the query and returns result with pagination metadata
75
75
  */
76
76
  executeWithPagination(): Promise<QueryResult<Model>>;
77
+
78
+ /**
79
+ * Returns an async iterator that paginates internally and yields one item at
80
+ * a time. Memory stays at one page; you can `break` out of the loop early.
81
+ *
82
+ * `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
83
+ * cap, not a total cap. To cap the total, count yourself inside the loop:
84
+ *
85
+ * ```ts
86
+ * let n = 0;
87
+ * for await (const item of builder.iterate()) {
88
+ * if (n++ >= 1000) break;
89
+ * process(item);
90
+ * }
91
+ * ```
92
+ */
93
+ iterate(): AsyncIterableIterator<Model>;
77
94
  }
78
95
 
79
96
  /**
@@ -1,7 +1,11 @@
1
1
  /* eslint-disable @typescript-eslint/no-explicit-any */
2
2
  import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
3
+ import { ScanCommand } from '@aws-sdk/lib-dynamodb';
4
+ import { mockClient } from 'aws-sdk-client-mock';
3
5
  import { createScanBuilder } from './create-scan-builder';
4
6
 
7
+ const ddbMock = mockClient(DynamoDBClient);
8
+
5
9
  describe('ScanBuilder', () => {
6
10
  const client = new DynamoDBClient({});
7
11
  const tableName = 'TestTable';
@@ -257,4 +261,121 @@ describe('ScanBuilder', () => {
257
261
  expect(Object.values(params.ExpressionAttributeValues || {})).toContain(65);
258
262
  });
259
263
  });
264
+
265
+ describe('executeWithPagination', () => {
266
+ beforeEach(() => {
267
+ ddbMock.reset();
268
+ });
269
+
270
+ test('returns items, lastEvaluatedKey and counts', async () => {
271
+ const items = [
272
+ { pk: 'USER#alice', sk: 'USER#alice', name: 'Alice' },
273
+ { pk: 'USER#bob', sk: 'USER#bob', name: 'Bob' },
274
+ ];
275
+ const lastKey = { pk: 'USER#bob', sk: 'USER#bob' };
276
+
277
+ ddbMock.on(ScanCommand).resolves({
278
+ Items: items,
279
+ LastEvaluatedKey: lastKey,
280
+ Count: 2,
281
+ ScannedCount: 5,
282
+ });
283
+
284
+ const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
285
+
286
+ expect(result.items).toEqual(items);
287
+ expect(result.lastEvaluatedKey).toEqual(lastKey);
288
+ expect(result.count).toBe(2);
289
+ expect(result.scannedCount).toBe(5);
290
+ });
291
+
292
+ test('returns undefined lastEvaluatedKey on the final page', async () => {
293
+ ddbMock.on(ScanCommand).resolves({ Items: [], Count: 0, ScannedCount: 0 });
294
+
295
+ const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
296
+
297
+ expect(result.items).toEqual([]);
298
+ expect(result.lastEvaluatedKey).toBeUndefined();
299
+ });
300
+ });
301
+
302
+ describe('iterate', () => {
303
+ beforeEach(() => {
304
+ ddbMock.reset();
305
+ });
306
+
307
+ test('walks every page transparently and yields items in order', async () => {
308
+ const page1 = [
309
+ { pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
310
+ { pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
311
+ ];
312
+ const page2 = [{ pk: 'USER#3', sk: 'USER#3', name: 'User 3' }];
313
+
314
+ ddbMock
315
+ .on(ScanCommand)
316
+ .resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
317
+ .resolvesOnce({ Items: page2 });
318
+
319
+ const collected: TestModel[] = [];
320
+ for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
321
+ collected.push(item);
322
+ }
323
+
324
+ expect(collected.map((u) => u.name)).toEqual(['User 1', 'User 2', 'User 3']);
325
+ expect(ddbMock.calls()).toHaveLength(2);
326
+ });
327
+
328
+ test('forwards LastEvaluatedKey from each response into the next call', async () => {
329
+ const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
330
+
331
+ ddbMock
332
+ .on(ScanCommand)
333
+ .resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
334
+ .resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }] });
335
+
336
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
337
+ for await (const _ of createScanBuilder<TestModel>(tableName, client).iterate()) {
338
+ // drain
339
+ }
340
+
341
+ const calls = ddbMock.calls();
342
+ expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
343
+ expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
344
+ });
345
+
346
+ test('starts from the user-provided cursor on the first call', async () => {
347
+ const startKey = { pk: 'USER#10', sk: 'USER#10' };
348
+
349
+ ddbMock.on(ScanCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
350
+
351
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
352
+ for await (const _ of createScanBuilder<TestModel>(tableName, client)
353
+ .startFrom(startKey)
354
+ .iterate()) {
355
+ // drain
356
+ }
357
+
358
+ const calls = ddbMock.calls();
359
+ expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
360
+ });
361
+
362
+ test('break out of the loop stops further DynamoDB calls', async () => {
363
+ ddbMock.on(ScanCommand).resolvesOnce({
364
+ Items: [
365
+ { pk: 'USER#1', sk: 'USER#1' },
366
+ { pk: 'USER#2', sk: 'USER#2' },
367
+ ],
368
+ LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
369
+ });
370
+
371
+ const collected: TestModel[] = [];
372
+ for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
373
+ collected.push(item);
374
+ if (collected.length >= 1) break;
375
+ }
376
+
377
+ expect(collected).toHaveLength(1);
378
+ expect(ddbMock.calls()).toHaveLength(1);
379
+ });
380
+ });
260
381
  });
@@ -2,7 +2,7 @@
2
2
  import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
3
3
  import { ScanCommand } from '@aws-sdk/lib-dynamodb';
4
4
  import { buildExpression, AttrBuilder, Condition, createOpBuilder } from '../shared';
5
- import { ScanBuilder } from './types';
5
+ import { ScanBuilder, ScanResult } from './types';
6
6
  import { DynamoDBLogger } from '../../utils/dynamodb-logger';
7
7
 
8
8
  /**
@@ -205,12 +205,46 @@ export function createScanBuilder<Model>(
205
205
  return params;
206
206
  },
207
207
 
208
+ /**
209
+ * ⚠️ Returns only the FIRST page of results. DynamoDB caps each Scan
210
+ * response at ~1MB (or `Limit` if set). If the matching set is larger,
211
+ * the remaining items are silently dropped.
212
+ *
213
+ * Use {@link executeWithPagination} when you need to drive pagination
214
+ * yourself, or {@link iterate} to walk every matching item lazily.
215
+ */
208
216
  async execute() {
209
217
  const params = build().dbParams();
210
218
  const response = await client.send(new ScanCommand(params));
211
219
  logger?.log('ScanCommand', params, response);
212
220
  return (response.Items || []) as Model[];
213
221
  },
222
+
223
+ async executeWithPagination(): Promise<ScanResult<Model>> {
224
+ const params = build().dbParams();
225
+ const response = await client.send(new ScanCommand(params));
226
+ logger?.log('ScanCommand', params, response);
227
+ return {
228
+ items: (response.Items ?? []) as Model[],
229
+ lastEvaluatedKey: response.LastEvaluatedKey,
230
+ count: response.Count,
231
+ scannedCount: response.ScannedCount,
232
+ };
233
+ },
234
+
235
+ async *iterate(): AsyncIterableIterator<Model> {
236
+ const baseParams = build().dbParams();
237
+ let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
238
+ do {
239
+ const params = { ...baseParams, ExclusiveStartKey: cursor };
240
+ const response = await client.send(new ScanCommand(params));
241
+ logger?.log('ScanCommand', params, response);
242
+ for (const item of (response.Items ?? []) as Model[]) {
243
+ yield item;
244
+ }
245
+ cursor = response.LastEvaluatedKey;
246
+ } while (cursor);
247
+ },
214
248
  });
215
249
 
216
250
  return build();
@@ -1,5 +1,20 @@
1
+ /* eslint-disable @typescript-eslint/no-explicit-any */
1
2
  import { ExecutableBuilder, AttrBuilder, OpBuilder, Condition } from '../shared';
2
3
 
4
+ /**
5
+ * A single page of scan results plus metadata for manual pagination.
6
+ */
7
+ export interface ScanResult<Model> {
8
+ /** Items returned in this page. */
9
+ items: Model[];
10
+ /** Cursor for the next page; `undefined` when there are no more results. */
11
+ lastEvaluatedKey?: Record<string, any>;
12
+ /** Number of items returned after applying the filter. */
13
+ count?: number;
14
+ /** Number of items examined before applying the filter. */
15
+ scannedCount?: number;
16
+ }
17
+
3
18
  /**
4
19
  * Builder for DynamoDB Scan operations.
5
20
  * Scans the entire table or index without requiring key conditions.
@@ -46,4 +61,28 @@ export interface ScanBuilder<Model> extends ExecutableBuilder<Model[]> {
46
61
  * Returns a new immutable builder.
47
62
  */
48
63
  segment(segmentNumber: number, totalSegments: number): ScanBuilder<Model>;
64
+
65
+ /**
66
+ * Executes a single Scan request and returns the page along with the
67
+ * `lastEvaluatedKey` cursor and counts. Use this when you want to drive
68
+ * pagination yourself (e.g. expose a cursor to a client).
69
+ */
70
+ executeWithPagination(): Promise<ScanResult<Model>>;
71
+
72
+ /**
73
+ * Returns an async iterator that paginates internally and yields one item at
74
+ * a time. Memory stays at one page; you can `break` out of the loop early.
75
+ *
76
+ * `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
77
+ * cap, not a total cap. To cap the total, count yourself inside the loop:
78
+ *
79
+ * ```ts
80
+ * let n = 0;
81
+ * for await (const item of builder.iterate()) {
82
+ * if (n++ >= 1000) break;
83
+ * process(item);
84
+ * }
85
+ * ```
86
+ */
87
+ iterate(): AsyncIterableIterator<Model>;
49
88
  }