@ftschopp/dynatable-core 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/builders/query/create-query-builder.d.ts.map +1 -1
- package/dist/builders/query/create-query-builder.js +21 -0
- package/dist/builders/query/types.d.ts +16 -0
- package/dist/builders/query/types.d.ts.map +1 -1
- package/dist/builders/scan/create-scan-builder.d.ts.map +1 -1
- package/dist/builders/scan/create-scan-builder.js +32 -0
- package/dist/builders/scan/types.d.ts +35 -0
- package/dist/builders/scan/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/builders/query/create-query-builder.test.ts +120 -0
- package/src/builders/query/create-query-builder.ts +22 -0
- package/src/builders/query/types.ts +17 -0
- package/src/builders/scan/create-scan-builder.test.ts +121 -0
- package/src/builders/scan/create-scan-builder.ts +35 -1
- package/src/builders/scan/types.ts +39 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
# @ftschopp/dynatable-core [1.4.0](https://github.com/ftschopp/dynatable/compare/@ftschopp/dynatable-core@1.3.0...@ftschopp/dynatable-core@1.4.0) (2026-05-08)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* **core:** add iterate() async iterator and Scan executeWithPagination ([#21](https://github.com/ftschopp/dynatable/issues/21)) ([c31d458](https://github.com/ftschopp/dynatable/commit/c31d45861e92ba0c52240128084939128a6e4acc)), closes [#4](https://github.com/ftschopp/dynatable/issues/4)
|
|
7
|
+
|
|
1
8
|
# @ftschopp/dynatable-core [1.3.0](https://github.com/ftschopp/dynatable/compare/@ftschopp/dynatable-core@1.2.5...@ftschopp/dynatable-core@1.3.0) (2026-05-07)
|
|
2
9
|
|
|
3
10
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-query-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/query/create-query-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAK1D,OAAO,EAAE,YAAY,EAA8B,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"create-query-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/query/create-query-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAK1D,OAAO,EAAE,YAAY,EAA8B,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAiX7D;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EACtC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,cAAc,EACtB,KAAK,CAAC,EAAE,eAAe,EACvB,MAAM,CAAC,EAAE,cAAc,EACvB,UAAU,CAAC,EAAE,MAAM,GAClB,YAAY,CAAC,KAAK,CAAC,CA6BrB"}
|
|
@@ -255,6 +255,14 @@ function createQueryExecutor(state) {
|
|
|
255
255
|
}),
|
|
256
256
|
};
|
|
257
257
|
},
|
|
258
|
+
/**
|
|
259
|
+
* ⚠️ Returns only the FIRST page of results. DynamoDB caps each Query
|
|
260
|
+
* response at ~1MB (or `Limit` if set). If the matching set is larger,
|
|
261
|
+
* the remaining items are silently dropped.
|
|
262
|
+
*
|
|
263
|
+
* Use {@link executeWithPagination} when you need to drive pagination
|
|
264
|
+
* yourself, or {@link iterate} to walk every matching item lazily.
|
|
265
|
+
*/
|
|
258
266
|
async execute() {
|
|
259
267
|
const params = this.dbParams();
|
|
260
268
|
const result = await state.client.send(new lib_dynamodb_1.QueryCommand(params));
|
|
@@ -272,6 +280,19 @@ function createQueryExecutor(state) {
|
|
|
272
280
|
scannedCount: result.ScannedCount,
|
|
273
281
|
};
|
|
274
282
|
},
|
|
283
|
+
async *iterate() {
|
|
284
|
+
const baseParams = this.dbParams();
|
|
285
|
+
let cursor = baseParams.ExclusiveStartKey;
|
|
286
|
+
do {
|
|
287
|
+
const params = { ...baseParams, ExclusiveStartKey: cursor };
|
|
288
|
+
const result = await state.client.send(new lib_dynamodb_1.QueryCommand(params));
|
|
289
|
+
state.logger?.log('QueryCommand', params, result);
|
|
290
|
+
for (const item of (result.Items ?? [])) {
|
|
291
|
+
yield item;
|
|
292
|
+
}
|
|
293
|
+
cursor = result.LastEvaluatedKey;
|
|
294
|
+
} while (cursor);
|
|
295
|
+
},
|
|
275
296
|
};
|
|
276
297
|
}
|
|
277
298
|
/**
|
|
@@ -60,6 +60,22 @@ export interface QueryExecutor<Model> extends ExecutableBuilder<Model[]> {
|
|
|
60
60
|
* Executes the query and returns result with pagination metadata
|
|
61
61
|
*/
|
|
62
62
|
executeWithPagination(): Promise<QueryResult<Model>>;
|
|
63
|
+
/**
|
|
64
|
+
* Returns an async iterator that paginates internally and yields one item at
|
|
65
|
+
* a time. Memory stays at one page; you can `break` out of the loop early.
|
|
66
|
+
*
|
|
67
|
+
* `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
|
|
68
|
+
* cap, not a total cap. To cap the total, count yourself inside the loop:
|
|
69
|
+
*
|
|
70
|
+
* ```ts
|
|
71
|
+
* let n = 0;
|
|
72
|
+
* for await (const item of builder.iterate()) {
|
|
73
|
+
* if (n++ >= 1000) break;
|
|
74
|
+
* process(item);
|
|
75
|
+
* }
|
|
76
|
+
* ```
|
|
77
|
+
*/
|
|
78
|
+
iterate(): AsyncIterableIterator<Model>;
|
|
63
79
|
}
|
|
64
80
|
/**
|
|
65
81
|
* Main Query Builder interface with type-safe where clause
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/query/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEjF;;GAEG;AACH,MAAM,WAAW,WAAW,CAAC,KAAK;IAChC;;OAEG;IACH,KAAK,EAAE,KAAK,EAAE,CAAC;IAEf;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEvC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,KAAK,CAAE,SAAQ,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACtE;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE3C;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAEzD;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAElD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAErD;;OAEG;IACH,cAAc,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;IAEvC;;OAEG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE1D;;OAEG;IACH,QAAQ,IAAI,GAAG,CAAC;IAEhB;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAE5B;;OAEG;IACH,qBAAqB,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/query/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEjF;;GAEG;AACH,MAAM,WAAW,WAAW,CAAC,KAAK;IAChC;;OAEG;IACH,KAAK,EAAE,KAAK,EAAE,CAAC;IAEf;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEvC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,KAAK,CAAE,SAAQ,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACtE;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE3C;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAEzD;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAElD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAErD;;OAEG;IACH,cAAc,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;IAEvC;;OAEG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAE1D;;OAEG;IACH,QAAQ,IAAI,GAAG,CAAC;IAEhB;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAE5B;;OAEG;IACH,qBAAqB,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAErD;;;;;;;;;;;;;;OAcG;IACH,OAAO,IAAI,qBAAqB,CAAC,KAAK,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY,CAAC,KAAK;IACjC;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,SAAS,KAAK,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;CACzF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-scan-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/scan/create-scan-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAgC,SAAS,EAAmB,MAAM,WAAW,CAAC;AACrF,OAAO,EAAE,WAAW,
|
|
1
|
+
{"version":3,"file":"create-scan-builder.d.ts","sourceRoot":"","sources":["../../../src/builders/scan/create-scan-builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE1D,OAAO,EAAgC,SAAS,EAAmB,MAAM,WAAW,CAAC;AACrF,OAAO,EAAE,WAAW,EAAc,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EACrC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,SAAS,EAAO,EACzB,eAAe,GAAE,CAAC,MAAM,KAAK,CAAC,EAAO,EACrC,UAAU,CAAC,EAAE,MAAM,EACnB,gBAAgB,UAAQ,EACxB,SAAS,CAAC,EAAE,MAAM,EAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACvC,aAAa,CAAC,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,EAC1D,MAAM,CAAC,EAAE,cAAc,GACtB,WAAW,CAAC,KAAK,CAAC,CAqOpB"}
|
|
@@ -92,12 +92,44 @@ function createScanBuilder(tableName, client, filters = [], projectionAttrs = []
|
|
|
92
92
|
}
|
|
93
93
|
return params;
|
|
94
94
|
},
|
|
95
|
+
/**
|
|
96
|
+
* ⚠️ Returns only the FIRST page of results. DynamoDB caps each Scan
|
|
97
|
+
* response at ~1MB (or `Limit` if set). If the matching set is larger,
|
|
98
|
+
* the remaining items are silently dropped.
|
|
99
|
+
*
|
|
100
|
+
* Use {@link executeWithPagination} when you need to drive pagination
|
|
101
|
+
* yourself, or {@link iterate} to walk every matching item lazily.
|
|
102
|
+
*/
|
|
95
103
|
async execute() {
|
|
96
104
|
const params = build().dbParams();
|
|
97
105
|
const response = await client.send(new lib_dynamodb_1.ScanCommand(params));
|
|
98
106
|
logger?.log('ScanCommand', params, response);
|
|
99
107
|
return (response.Items || []);
|
|
100
108
|
},
|
|
109
|
+
async executeWithPagination() {
|
|
110
|
+
const params = build().dbParams();
|
|
111
|
+
const response = await client.send(new lib_dynamodb_1.ScanCommand(params));
|
|
112
|
+
logger?.log('ScanCommand', params, response);
|
|
113
|
+
return {
|
|
114
|
+
items: (response.Items ?? []),
|
|
115
|
+
lastEvaluatedKey: response.LastEvaluatedKey,
|
|
116
|
+
count: response.Count,
|
|
117
|
+
scannedCount: response.ScannedCount,
|
|
118
|
+
};
|
|
119
|
+
},
|
|
120
|
+
async *iterate() {
|
|
121
|
+
const baseParams = build().dbParams();
|
|
122
|
+
let cursor = baseParams.ExclusiveStartKey;
|
|
123
|
+
do {
|
|
124
|
+
const params = { ...baseParams, ExclusiveStartKey: cursor };
|
|
125
|
+
const response = await client.send(new lib_dynamodb_1.ScanCommand(params));
|
|
126
|
+
logger?.log('ScanCommand', params, response);
|
|
127
|
+
for (const item of (response.Items ?? [])) {
|
|
128
|
+
yield item;
|
|
129
|
+
}
|
|
130
|
+
cursor = response.LastEvaluatedKey;
|
|
131
|
+
} while (cursor);
|
|
132
|
+
},
|
|
101
133
|
});
|
|
102
134
|
return build();
|
|
103
135
|
}
|
|
@@ -1,4 +1,17 @@
|
|
|
1
1
|
import { ExecutableBuilder, AttrBuilder, OpBuilder, Condition } from '../shared';
|
|
2
|
+
/**
|
|
3
|
+
* A single page of scan results plus metadata for manual pagination.
|
|
4
|
+
*/
|
|
5
|
+
export interface ScanResult<Model> {
|
|
6
|
+
/** Items returned in this page. */
|
|
7
|
+
items: Model[];
|
|
8
|
+
/** Cursor for the next page; `undefined` when there are no more results. */
|
|
9
|
+
lastEvaluatedKey?: Record<string, any>;
|
|
10
|
+
/** Number of items returned after applying the filter. */
|
|
11
|
+
count?: number;
|
|
12
|
+
/** Number of items examined before applying the filter. */
|
|
13
|
+
scannedCount?: number;
|
|
14
|
+
}
|
|
2
15
|
/**
|
|
3
16
|
* Builder for DynamoDB Scan operations.
|
|
4
17
|
* Scans the entire table or index without requiring key conditions.
|
|
@@ -39,5 +52,27 @@ export interface ScanBuilder<Model> extends ExecutableBuilder<Model[]> {
|
|
|
39
52
|
* Returns a new immutable builder.
|
|
40
53
|
*/
|
|
41
54
|
segment(segmentNumber: number, totalSegments: number): ScanBuilder<Model>;
|
|
55
|
+
/**
|
|
56
|
+
* Executes a single Scan request and returns the page along with the
|
|
57
|
+
* `lastEvaluatedKey` cursor and counts. Use this when you want to drive
|
|
58
|
+
* pagination yourself (e.g. expose a cursor to a client).
|
|
59
|
+
*/
|
|
60
|
+
executeWithPagination(): Promise<ScanResult<Model>>;
|
|
61
|
+
/**
|
|
62
|
+
* Returns an async iterator that paginates internally and yields one item at
|
|
63
|
+
* a time. Memory stays at one page; you can `break` out of the loop early.
|
|
64
|
+
*
|
|
65
|
+
* `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
|
|
66
|
+
* cap, not a total cap. To cap the total, count yourself inside the loop:
|
|
67
|
+
*
|
|
68
|
+
* ```ts
|
|
69
|
+
* let n = 0;
|
|
70
|
+
* for await (const item of builder.iterate()) {
|
|
71
|
+
* if (n++ >= 1000) break;
|
|
72
|
+
* process(item);
|
|
73
|
+
* }
|
|
74
|
+
* ```
|
|
75
|
+
*/
|
|
76
|
+
iterate(): AsyncIterableIterator<Model>;
|
|
42
77
|
}
|
|
43
78
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/scan/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/builders/scan/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEjF;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,KAAK;IAC/B,mCAAmC;IACnC,KAAK,EAAE,KAAK,EAAE,CAAC;IACf,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACvC,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW,CAAC,KAAK,CAAE,SAAQ,iBAAiB,CAAC,KAAK,EAAE,CAAC;IACpE;;;OAGG;IACH,MAAM,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,SAAS,KAAK,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEvF;;;OAGG;IACH,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEnD;;;OAGG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEzC;;;OAGG;IACH,cAAc,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC;IAErC;;;OAGG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAElD;;;OAGG;IACH,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAEtE;;;OAGG;IACH,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAE1E;;;;OAIG;IACH,qBAAqB,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IAEpD;;;;;;;;;;;;;;OAcG;IACH,OAAO,IAAI,qBAAqB,CAAC,KAAK,CAAC,CAAC;CACzC"}
|
package/package.json
CHANGED
|
@@ -227,6 +227,126 @@ describe('QueryBuilder - Pagination', () => {
|
|
|
227
227
|
expect(builder2.dbParams().ExclusiveStartKey).toEqual(startKey);
|
|
228
228
|
});
|
|
229
229
|
});
|
|
230
|
+
|
|
231
|
+
describe('iterate method', () => {
|
|
232
|
+
test('walks every page transparently and yields items in order', async () => {
|
|
233
|
+
const page1 = [
|
|
234
|
+
{ pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
|
|
235
|
+
{ pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
|
|
236
|
+
];
|
|
237
|
+
const page2 = [
|
|
238
|
+
{ pk: 'USER#3', sk: 'USER#3', name: 'User 3' },
|
|
239
|
+
{ pk: 'USER#4', sk: 'USER#4', name: 'User 4' },
|
|
240
|
+
];
|
|
241
|
+
const page3 = [{ pk: 'USER#5', sk: 'USER#5', name: 'User 5' }];
|
|
242
|
+
|
|
243
|
+
ddbMock
|
|
244
|
+
.on(QueryCommand)
|
|
245
|
+
.resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
|
|
246
|
+
.resolvesOnce({ Items: page2, LastEvaluatedKey: { pk: 'USER#4', sk: 'USER#4' } })
|
|
247
|
+
.resolvesOnce({ Items: page3 });
|
|
248
|
+
|
|
249
|
+
const collected: TestModel[] = [];
|
|
250
|
+
for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
251
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
|
|
252
|
+
.iterate()) {
|
|
253
|
+
collected.push(item);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
expect(collected).toHaveLength(5);
|
|
257
|
+
expect(collected.map((u) => u.name)).toEqual([
|
|
258
|
+
'User 1',
|
|
259
|
+
'User 2',
|
|
260
|
+
'User 3',
|
|
261
|
+
'User 4',
|
|
262
|
+
'User 5',
|
|
263
|
+
]);
|
|
264
|
+
expect(ddbMock.calls()).toHaveLength(3);
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
test('forwards ExclusiveStartKey from each response into the next call', async () => {
|
|
268
|
+
const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
|
|
269
|
+
const cursor2 = { pk: 'USER#4', sk: 'USER#4' };
|
|
270
|
+
|
|
271
|
+
ddbMock
|
|
272
|
+
.on(QueryCommand)
|
|
273
|
+
.resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
|
|
274
|
+
.resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }], LastEvaluatedKey: cursor2 })
|
|
275
|
+
.resolvesOnce({ Items: [{ pk: 'USER#5', sk: 'USER#5' }] });
|
|
276
|
+
|
|
277
|
+
const iterator = createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
278
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
|
|
279
|
+
.iterate();
|
|
280
|
+
|
|
281
|
+
// Consume the iterator
|
|
282
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
283
|
+
for await (const _ of iterator) {
|
|
284
|
+
// drain
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const calls = ddbMock.calls();
|
|
288
|
+
expect(calls).toHaveLength(3);
|
|
289
|
+
expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
|
|
290
|
+
expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
|
|
291
|
+
expect((calls[2]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor2);
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
test('starts from the user-provided cursor on the first call', async () => {
|
|
295
|
+
const startKey = { pk: 'USER#10', sk: 'USER#10' };
|
|
296
|
+
|
|
297
|
+
ddbMock.on(QueryCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
|
|
298
|
+
|
|
299
|
+
const iterator = createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
300
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
|
|
301
|
+
.startFrom(startKey)
|
|
302
|
+
.iterate();
|
|
303
|
+
|
|
304
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
305
|
+
for await (const _ of iterator) {
|
|
306
|
+
// drain
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const calls = ddbMock.calls();
|
|
310
|
+
expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
test('break out of the loop stops further DynamoDB calls', async () => {
|
|
314
|
+
ddbMock
|
|
315
|
+
.on(QueryCommand)
|
|
316
|
+
.resolvesOnce({
|
|
317
|
+
Items: [
|
|
318
|
+
{ pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
|
|
319
|
+
{ pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
|
|
320
|
+
],
|
|
321
|
+
LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
const collected: TestModel[] = [];
|
|
325
|
+
for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
326
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'USER#'))
|
|
327
|
+
.iterate()) {
|
|
328
|
+
collected.push(item);
|
|
329
|
+
if (collected.length >= 1) break;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
expect(collected).toHaveLength(1);
|
|
333
|
+
expect(ddbMock.calls()).toHaveLength(1);
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
test('handles an empty result without making extra calls', async () => {
|
|
337
|
+
ddbMock.on(QueryCommand).resolves({ Items: [] });
|
|
338
|
+
|
|
339
|
+
const collected: TestModel[] = [];
|
|
340
|
+
for await (const item of createQueryBuilder<TestModel>(tableName, client, testModel)
|
|
341
|
+
.where((attr, op) => op.beginsWith(attr.pk, 'NONE#'))
|
|
342
|
+
.iterate()) {
|
|
343
|
+
collected.push(item);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
expect(collected).toEqual([]);
|
|
347
|
+
expect(ddbMock.calls()).toHaveLength(1);
|
|
348
|
+
});
|
|
349
|
+
});
|
|
230
350
|
});
|
|
231
351
|
|
|
232
352
|
describe('QueryBuilder - GSI key recognition', () => {
|
|
@@ -333,6 +333,14 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
|
|
|
333
333
|
};
|
|
334
334
|
},
|
|
335
335
|
|
|
336
|
+
/**
|
|
337
|
+
* ⚠️ Returns only the FIRST page of results. DynamoDB caps each Query
|
|
338
|
+
* response at ~1MB (or `Limit` if set). If the matching set is larger,
|
|
339
|
+
* the remaining items are silently dropped.
|
|
340
|
+
*
|
|
341
|
+
* Use {@link executeWithPagination} when you need to drive pagination
|
|
342
|
+
* yourself, or {@link iterate} to walk every matching item lazily.
|
|
343
|
+
*/
|
|
336
344
|
async execute(): Promise<Model[]> {
|
|
337
345
|
const params = this.dbParams();
|
|
338
346
|
const result = await state.client.send(new QueryCommand(params));
|
|
@@ -351,6 +359,20 @@ function createQueryExecutor<Model>(state: QueryState<Model>): QueryExecutor<Mod
|
|
|
351
359
|
scannedCount: result.ScannedCount,
|
|
352
360
|
};
|
|
353
361
|
},
|
|
362
|
+
|
|
363
|
+
async *iterate(): AsyncIterableIterator<Model> {
|
|
364
|
+
const baseParams = this.dbParams();
|
|
365
|
+
let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
|
|
366
|
+
do {
|
|
367
|
+
const params = { ...baseParams, ExclusiveStartKey: cursor };
|
|
368
|
+
const result = await state.client.send(new QueryCommand(params));
|
|
369
|
+
state.logger?.log('QueryCommand', params, result);
|
|
370
|
+
for (const item of (result.Items ?? []) as unknown as Model[]) {
|
|
371
|
+
yield item;
|
|
372
|
+
}
|
|
373
|
+
cursor = result.LastEvaluatedKey;
|
|
374
|
+
} while (cursor);
|
|
375
|
+
},
|
|
354
376
|
};
|
|
355
377
|
}
|
|
356
378
|
|
|
@@ -74,6 +74,23 @@ export interface QueryExecutor<Model> extends ExecutableBuilder<Model[]> {
|
|
|
74
74
|
* Executes the query and returns result with pagination metadata
|
|
75
75
|
*/
|
|
76
76
|
executeWithPagination(): Promise<QueryResult<Model>>;
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Returns an async iterator that paginates internally and yields one item at
|
|
80
|
+
* a time. Memory stays at one page; you can `break` out of the loop early.
|
|
81
|
+
*
|
|
82
|
+
* `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
|
|
83
|
+
* cap, not a total cap. To cap the total, count yourself inside the loop:
|
|
84
|
+
*
|
|
85
|
+
* ```ts
|
|
86
|
+
* let n = 0;
|
|
87
|
+
* for await (const item of builder.iterate()) {
|
|
88
|
+
* if (n++ >= 1000) break;
|
|
89
|
+
* process(item);
|
|
90
|
+
* }
|
|
91
|
+
* ```
|
|
92
|
+
*/
|
|
93
|
+
iterate(): AsyncIterableIterator<Model>;
|
|
77
94
|
}
|
|
78
95
|
|
|
79
96
|
/**
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
2
|
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
|
|
3
|
+
import { ScanCommand } from '@aws-sdk/lib-dynamodb';
|
|
4
|
+
import { mockClient } from 'aws-sdk-client-mock';
|
|
3
5
|
import { createScanBuilder } from './create-scan-builder';
|
|
4
6
|
|
|
7
|
+
const ddbMock = mockClient(DynamoDBClient);
|
|
8
|
+
|
|
5
9
|
describe('ScanBuilder', () => {
|
|
6
10
|
const client = new DynamoDBClient({});
|
|
7
11
|
const tableName = 'TestTable';
|
|
@@ -257,4 +261,121 @@ describe('ScanBuilder', () => {
|
|
|
257
261
|
expect(Object.values(params.ExpressionAttributeValues || {})).toContain(65);
|
|
258
262
|
});
|
|
259
263
|
});
|
|
264
|
+
|
|
265
|
+
describe('executeWithPagination', () => {
|
|
266
|
+
beforeEach(() => {
|
|
267
|
+
ddbMock.reset();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test('returns items, lastEvaluatedKey and counts', async () => {
|
|
271
|
+
const items = [
|
|
272
|
+
{ pk: 'USER#alice', sk: 'USER#alice', name: 'Alice' },
|
|
273
|
+
{ pk: 'USER#bob', sk: 'USER#bob', name: 'Bob' },
|
|
274
|
+
];
|
|
275
|
+
const lastKey = { pk: 'USER#bob', sk: 'USER#bob' };
|
|
276
|
+
|
|
277
|
+
ddbMock.on(ScanCommand).resolves({
|
|
278
|
+
Items: items,
|
|
279
|
+
LastEvaluatedKey: lastKey,
|
|
280
|
+
Count: 2,
|
|
281
|
+
ScannedCount: 5,
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
|
|
285
|
+
|
|
286
|
+
expect(result.items).toEqual(items);
|
|
287
|
+
expect(result.lastEvaluatedKey).toEqual(lastKey);
|
|
288
|
+
expect(result.count).toBe(2);
|
|
289
|
+
expect(result.scannedCount).toBe(5);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
test('returns undefined lastEvaluatedKey on the final page', async () => {
|
|
293
|
+
ddbMock.on(ScanCommand).resolves({ Items: [], Count: 0, ScannedCount: 0 });
|
|
294
|
+
|
|
295
|
+
const result = await createScanBuilder<TestModel>(tableName, client).executeWithPagination();
|
|
296
|
+
|
|
297
|
+
expect(result.items).toEqual([]);
|
|
298
|
+
expect(result.lastEvaluatedKey).toBeUndefined();
|
|
299
|
+
});
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
describe('iterate', () => {
|
|
303
|
+
beforeEach(() => {
|
|
304
|
+
ddbMock.reset();
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
test('walks every page transparently and yields items in order', async () => {
|
|
308
|
+
const page1 = [
|
|
309
|
+
{ pk: 'USER#1', sk: 'USER#1', name: 'User 1' },
|
|
310
|
+
{ pk: 'USER#2', sk: 'USER#2', name: 'User 2' },
|
|
311
|
+
];
|
|
312
|
+
const page2 = [{ pk: 'USER#3', sk: 'USER#3', name: 'User 3' }];
|
|
313
|
+
|
|
314
|
+
ddbMock
|
|
315
|
+
.on(ScanCommand)
|
|
316
|
+
.resolvesOnce({ Items: page1, LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' } })
|
|
317
|
+
.resolvesOnce({ Items: page2 });
|
|
318
|
+
|
|
319
|
+
const collected: TestModel[] = [];
|
|
320
|
+
for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
|
|
321
|
+
collected.push(item);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
expect(collected.map((u) => u.name)).toEqual(['User 1', 'User 2', 'User 3']);
|
|
325
|
+
expect(ddbMock.calls()).toHaveLength(2);
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
test('forwards LastEvaluatedKey from each response into the next call', async () => {
|
|
329
|
+
const cursor1 = { pk: 'USER#2', sk: 'USER#2' };
|
|
330
|
+
|
|
331
|
+
ddbMock
|
|
332
|
+
.on(ScanCommand)
|
|
333
|
+
.resolvesOnce({ Items: [{ pk: 'USER#1', sk: 'USER#1' }], LastEvaluatedKey: cursor1 })
|
|
334
|
+
.resolvesOnce({ Items: [{ pk: 'USER#3', sk: 'USER#3' }] });
|
|
335
|
+
|
|
336
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
337
|
+
for await (const _ of createScanBuilder<TestModel>(tableName, client).iterate()) {
|
|
338
|
+
// drain
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const calls = ddbMock.calls();
|
|
342
|
+
expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toBeUndefined();
|
|
343
|
+
expect((calls[1]!.args[0].input as any).ExclusiveStartKey).toEqual(cursor1);
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
test('starts from the user-provided cursor on the first call', async () => {
|
|
347
|
+
const startKey = { pk: 'USER#10', sk: 'USER#10' };
|
|
348
|
+
|
|
349
|
+
ddbMock.on(ScanCommand).resolves({ Items: [{ pk: 'USER#11', sk: 'USER#11' }] });
|
|
350
|
+
|
|
351
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
352
|
+
for await (const _ of createScanBuilder<TestModel>(tableName, client)
|
|
353
|
+
.startFrom(startKey)
|
|
354
|
+
.iterate()) {
|
|
355
|
+
// drain
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const calls = ddbMock.calls();
|
|
359
|
+
expect((calls[0]!.args[0].input as any).ExclusiveStartKey).toEqual(startKey);
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
test('break out of the loop stops further DynamoDB calls', async () => {
|
|
363
|
+
ddbMock.on(ScanCommand).resolvesOnce({
|
|
364
|
+
Items: [
|
|
365
|
+
{ pk: 'USER#1', sk: 'USER#1' },
|
|
366
|
+
{ pk: 'USER#2', sk: 'USER#2' },
|
|
367
|
+
],
|
|
368
|
+
LastEvaluatedKey: { pk: 'USER#2', sk: 'USER#2' },
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
const collected: TestModel[] = [];
|
|
372
|
+
for await (const item of createScanBuilder<TestModel>(tableName, client).iterate()) {
|
|
373
|
+
collected.push(item);
|
|
374
|
+
if (collected.length >= 1) break;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
expect(collected).toHaveLength(1);
|
|
378
|
+
expect(ddbMock.calls()).toHaveLength(1);
|
|
379
|
+
});
|
|
380
|
+
});
|
|
260
381
|
});
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
|
|
3
3
|
import { ScanCommand } from '@aws-sdk/lib-dynamodb';
|
|
4
4
|
import { buildExpression, AttrBuilder, Condition, createOpBuilder } from '../shared';
|
|
5
|
-
import { ScanBuilder } from './types';
|
|
5
|
+
import { ScanBuilder, ScanResult } from './types';
|
|
6
6
|
import { DynamoDBLogger } from '../../utils/dynamodb-logger';
|
|
7
7
|
|
|
8
8
|
/**
|
|
@@ -205,12 +205,46 @@ export function createScanBuilder<Model>(
|
|
|
205
205
|
return params;
|
|
206
206
|
},
|
|
207
207
|
|
|
208
|
+
/**
|
|
209
|
+
* ⚠️ Returns only the FIRST page of results. DynamoDB caps each Scan
|
|
210
|
+
* response at ~1MB (or `Limit` if set). If the matching set is larger,
|
|
211
|
+
* the remaining items are silently dropped.
|
|
212
|
+
*
|
|
213
|
+
* Use {@link executeWithPagination} when you need to drive pagination
|
|
214
|
+
* yourself, or {@link iterate} to walk every matching item lazily.
|
|
215
|
+
*/
|
|
208
216
|
async execute() {
|
|
209
217
|
const params = build().dbParams();
|
|
210
218
|
const response = await client.send(new ScanCommand(params));
|
|
211
219
|
logger?.log('ScanCommand', params, response);
|
|
212
220
|
return (response.Items || []) as Model[];
|
|
213
221
|
},
|
|
222
|
+
|
|
223
|
+
async executeWithPagination(): Promise<ScanResult<Model>> {
|
|
224
|
+
const params = build().dbParams();
|
|
225
|
+
const response = await client.send(new ScanCommand(params));
|
|
226
|
+
logger?.log('ScanCommand', params, response);
|
|
227
|
+
return {
|
|
228
|
+
items: (response.Items ?? []) as Model[],
|
|
229
|
+
lastEvaluatedKey: response.LastEvaluatedKey,
|
|
230
|
+
count: response.Count,
|
|
231
|
+
scannedCount: response.ScannedCount,
|
|
232
|
+
};
|
|
233
|
+
},
|
|
234
|
+
|
|
235
|
+
async *iterate(): AsyncIterableIterator<Model> {
|
|
236
|
+
const baseParams = build().dbParams();
|
|
237
|
+
let cursor: Record<string, any> | undefined = baseParams.ExclusiveStartKey;
|
|
238
|
+
do {
|
|
239
|
+
const params = { ...baseParams, ExclusiveStartKey: cursor };
|
|
240
|
+
const response = await client.send(new ScanCommand(params));
|
|
241
|
+
logger?.log('ScanCommand', params, response);
|
|
242
|
+
for (const item of (response.Items ?? []) as Model[]) {
|
|
243
|
+
yield item;
|
|
244
|
+
}
|
|
245
|
+
cursor = response.LastEvaluatedKey;
|
|
246
|
+
} while (cursor);
|
|
247
|
+
},
|
|
214
248
|
});
|
|
215
249
|
|
|
216
250
|
return build();
|
|
@@ -1,5 +1,20 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
1
2
|
import { ExecutableBuilder, AttrBuilder, OpBuilder, Condition } from '../shared';
|
|
2
3
|
|
|
4
|
+
/**
|
|
5
|
+
* A single page of scan results plus metadata for manual pagination.
|
|
6
|
+
*/
|
|
7
|
+
export interface ScanResult<Model> {
|
|
8
|
+
/** Items returned in this page. */
|
|
9
|
+
items: Model[];
|
|
10
|
+
/** Cursor for the next page; `undefined` when there are no more results. */
|
|
11
|
+
lastEvaluatedKey?: Record<string, any>;
|
|
12
|
+
/** Number of items returned after applying the filter. */
|
|
13
|
+
count?: number;
|
|
14
|
+
/** Number of items examined before applying the filter. */
|
|
15
|
+
scannedCount?: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
3
18
|
/**
|
|
4
19
|
* Builder for DynamoDB Scan operations.
|
|
5
20
|
* Scans the entire table or index without requiring key conditions.
|
|
@@ -46,4 +61,28 @@ export interface ScanBuilder<Model> extends ExecutableBuilder<Model[]> {
|
|
|
46
61
|
* Returns a new immutable builder.
|
|
47
62
|
*/
|
|
48
63
|
segment(segmentNumber: number, totalSegments: number): ScanBuilder<Model>;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Executes a single Scan request and returns the page along with the
|
|
67
|
+
* `lastEvaluatedKey` cursor and counts. Use this when you want to drive
|
|
68
|
+
* pagination yourself (e.g. expose a cursor to a client).
|
|
69
|
+
*/
|
|
70
|
+
executeWithPagination(): Promise<ScanResult<Model>>;
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Returns an async iterator that paginates internally and yields one item at
|
|
74
|
+
* a time. Memory stays at one page; you can `break` out of the loop early.
|
|
75
|
+
*
|
|
76
|
+
* `Limit` set via `.limit()` is forwarded to DynamoDB as a *per-request*
|
|
77
|
+
* cap, not a total cap. To cap the total, count yourself inside the loop:
|
|
78
|
+
*
|
|
79
|
+
* ```ts
|
|
80
|
+
* let n = 0;
|
|
81
|
+
* for await (const item of builder.iterate()) {
|
|
82
|
+
* if (n++ >= 1000) break;
|
|
83
|
+
* process(item);
|
|
84
|
+
* }
|
|
85
|
+
* ```
|
|
86
|
+
*/
|
|
87
|
+
iterate(): AsyncIterableIterator<Model>;
|
|
49
88
|
}
|