effect-dynamodb 1.24.0 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Aggregate.d.ts +12 -2
- package/dist/Aggregate.d.ts.map +1 -1
- package/dist/Aggregate.js +284 -55
- package/dist/Aggregate.js.map +1 -1
- package/dist/Batch.d.ts +14 -5
- package/dist/Batch.d.ts.map +1 -1
- package/dist/Batch.js +184 -119
- package/dist/Batch.js.map +1 -1
- package/dist/Collection.d.ts +9 -1
- package/dist/Collection.d.ts.map +1 -1
- package/dist/Collection.js +5 -4
- package/dist/Collection.js.map +1 -1
- package/dist/DynamoClient.d.ts +80 -28
- package/dist/DynamoClient.d.ts.map +1 -1
- package/dist/DynamoClient.js +125 -25
- package/dist/DynamoClient.js.map +1 -1
- package/dist/Entity.d.ts +99 -40
- package/dist/Entity.d.ts.map +1 -1
- package/dist/Entity.js +693 -302
- package/dist/Entity.js.map +1 -1
- package/dist/EventStore.d.ts +598 -75
- package/dist/EventStore.d.ts.map +1 -1
- package/dist/EventStore.js +852 -221
- package/dist/EventStore.js.map +1 -1
- package/dist/Query.d.ts +100 -14
- package/dist/Query.d.ts.map +1 -1
- package/dist/Query.js +275 -71
- package/dist/Query.js.map +1 -1
- package/dist/StorageEncoding.d.ts +160 -0
- package/dist/StorageEncoding.d.ts.map +1 -0
- package/dist/StorageEncoding.js +258 -0
- package/dist/StorageEncoding.js.map +1 -0
- package/dist/Table.d.ts +56 -6
- package/dist/Table.d.ts.map +1 -1
- package/dist/Table.js +142 -60
- package/dist/Table.js.map +1 -1
- package/dist/Transaction.d.ts +8 -0
- package/dist/Transaction.d.ts.map +1 -1
- package/dist/Transaction.js +72 -22
- package/dist/Transaction.js.map +1 -1
- package/dist/VectorSearchEmulation.d.ts +3 -2
- package/dist/VectorSearchEmulation.d.ts.map +1 -1
- package/dist/VectorSearchEmulation.js +14 -2
- package/dist/VectorSearchEmulation.js.map +1 -1
- package/dist/compression/zlib.d.ts +75 -0
- package/dist/compression/zlib.d.ts.map +1 -0
- package/dist/compression/zlib.js +109 -0
- package/dist/compression/zlib.js.map +1 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/internal/BoundCrud.d.ts +39 -35
- package/dist/internal/BoundCrud.d.ts.map +1 -1
- package/dist/internal/BoundCrud.js.map +1 -1
- package/dist/internal/BoundQuery.d.ts +6 -5
- package/dist/internal/BoundQuery.d.ts.map +1 -1
- package/dist/internal/BoundQuery.js.map +1 -1
- package/dist/internal/BoundVectorQuery.d.ts +17 -1
- package/dist/internal/BoundVectorQuery.d.ts.map +1 -1
- package/dist/internal/BoundVectorQuery.js +31 -6
- package/dist/internal/BoundVectorQuery.js.map +1 -1
- package/dist/internal/EntityCombinators.d.ts +8 -5
- package/dist/internal/EntityCombinators.d.ts.map +1 -1
- package/dist/internal/EntityCombinators.js +24 -6
- package/dist/internal/EntityCombinators.js.map +1 -1
- package/dist/internal/EntityOps.d.ts +1 -1
- package/dist/internal/Expr.d.ts +9 -3
- package/dist/internal/Expr.d.ts.map +1 -1
- package/dist/internal/Expr.js +7 -5
- package/dist/internal/Expr.js.map +1 -1
- package/dist/internal/ExpressionLimits.d.ts +45 -0
- package/dist/internal/ExpressionLimits.d.ts.map +1 -0
- package/dist/internal/ExpressionLimits.js +89 -0
- package/dist/internal/ExpressionLimits.js.map +1 -0
- package/dist/internal/Hydrate.d.ts +62 -0
- package/dist/internal/Hydrate.d.ts.map +1 -0
- package/dist/internal/Hydrate.js +191 -0
- package/dist/internal/Hydrate.js.map +1 -0
- package/dist/internal/ItemSize.d.ts +60 -24
- package/dist/internal/ItemSize.d.ts.map +1 -1
- package/dist/internal/ItemSize.js +122 -39
- package/dist/internal/ItemSize.js.map +1 -1
- package/dist/internal/Packed.d.ts +289 -0
- package/dist/internal/Packed.d.ts.map +1 -0
- package/dist/internal/Packed.js +654 -0
- package/dist/internal/Packed.js.map +1 -0
- package/dist/internal/PathBuilder.d.ts +45 -2
- package/dist/internal/PathBuilder.d.ts.map +1 -1
- package/dist/internal/PathBuilder.js +4 -0
- package/dist/internal/PathBuilder.js.map +1 -1
- package/dist/internal/TransactWriteOps.d.ts +22 -46
- package/dist/internal/TransactWriteOps.d.ts.map +1 -1
- package/dist/internal/TransactWriteOps.js +26 -95
- package/dist/internal/TransactWriteOps.js.map +1 -1
- package/dist/internal/TransactableOps.d.ts +1 -1
- package/dist/internal/TransactableOps.d.ts.map +1 -1
- package/dist/internal/TransactableOps.js +2 -0
- package/dist/internal/TransactableOps.js.map +1 -1
- package/dist/internal/WriteDiagnosis.d.ts +353 -0
- package/dist/internal/WriteDiagnosis.d.ts.map +1 -0
- package/dist/internal/WriteDiagnosis.js +1028 -0
- package/dist/internal/WriteDiagnosis.js.map +1 -0
- package/dist/internal/sparse_check.js +43 -79
- package/dist/internal/sparse_check.js.map +1 -1
- package/package.json +6 -2
package/dist/Query.js
CHANGED
|
@@ -12,8 +12,29 @@ import * as Projection from "@effect-dynamodb/schema/Projection.js";
|
|
|
12
12
|
import { Effect, Function, Option, Pipeable, Stream } from "effect";
|
|
13
13
|
import { DynamoClient } from "./DynamoClient.js";
|
|
14
14
|
import { compileExpr, ExprTag, emptyPartProblem, isEmptyExpr } from "./internal/Expr.js";
|
|
15
|
+
import { batchGetRaw } from "./internal/Hydrate.js";
|
|
16
|
+
import { packedExprRefusal, packedProjectionRefusal, } from "./internal/Packed.js";
|
|
15
17
|
import { compilePath } from "./internal/PathBuilder.js";
|
|
18
|
+
import { diagnoseFailure, explainDiagnosis, withCause } from "./internal/WriteDiagnosis.js";
|
|
16
19
|
import { fromAttributeMap, toAttributeValue } from "./Marshaller.js";
|
|
20
|
+
/**
|
|
21
|
+
* @internal The raw-row hook a projection of `paths` runs: the query's
|
|
22
|
+
* {@link QueryState.rawRow}, unless it declares the attributes it rewrites
|
|
23
|
+
* ({@link QueryRawRow.attributes}) and the projection names none of them.
|
|
24
|
+
*/
|
|
25
|
+
const projectedRawRow = (hook, paths) => {
|
|
26
|
+
if (hook?.attributes === undefined)
|
|
27
|
+
return hook;
|
|
28
|
+
const rewrites = new Set(hook.attributes);
|
|
29
|
+
return paths.some((path) => rewrites.has(String(path[0]))) ? hook : undefined;
|
|
30
|
+
};
|
|
31
|
+
/** @internal The raw-row hook the projection of `state` runs, if any. */
|
|
32
|
+
const activeRawRow = (state) => projectedRawRow(state.rawRow, [
|
|
33
|
+
...(state.projection ?? []).map((attribute) => [attribute]),
|
|
34
|
+
...(state.projectionPaths ?? []),
|
|
35
|
+
]);
|
|
36
|
+
/** @internal The attributes the active raw-row hook of `state` reads (see {@link QueryRawRow.reads}). */
|
|
37
|
+
const rawRowReads = (state) => activeRawRow(state)?.reads ?? [];
|
|
17
38
|
/** @internal Dedupe + drop absent entries from a caller-supplied key field list. */
|
|
18
39
|
const normalizeKeyFields = (fields) => {
|
|
19
40
|
if (!fields)
|
|
@@ -43,9 +64,13 @@ class QueryImpl {
|
|
|
43
64
|
// Constructor (internal — used by Entity and Collection)
|
|
44
65
|
// ---------------------------------------------------------------------------
|
|
45
66
|
export const make = (config) => new QueryImpl({
|
|
67
|
+
rawRow: config.rawRow,
|
|
68
|
+
packed: config.packed ?? [],
|
|
46
69
|
liveRows: config.liveRows,
|
|
47
70
|
prepare: config.prepare,
|
|
48
71
|
keepRow: undefined,
|
|
72
|
+
hydration: config.hydration,
|
|
73
|
+
keyConditionDiagnosis: config.keyConditionDiagnosis,
|
|
49
74
|
tableName: config.tableName,
|
|
50
75
|
indexName: config.indexName,
|
|
51
76
|
pkField: config.pkField,
|
|
@@ -60,7 +85,6 @@ export const make = (config) => new QueryImpl({
|
|
|
60
85
|
maxPagesValue: undefined,
|
|
61
86
|
scanForward: true,
|
|
62
87
|
consistentRead: false,
|
|
63
|
-
globalIndex: config.globalIndex ?? false,
|
|
64
88
|
ignoreOwnershipFlag: false,
|
|
65
89
|
exclusiveStartKey: undefined,
|
|
66
90
|
isScan: false,
|
|
@@ -76,9 +100,13 @@ export const make = (config) => new QueryImpl({
|
|
|
76
100
|
* No key condition — only filter expressions and entity type filter.
|
|
77
101
|
*/
|
|
78
102
|
export const makeScan = (config) => new QueryImpl({
|
|
103
|
+
rawRow: config.rawRow,
|
|
104
|
+
packed: config.packed ?? [],
|
|
79
105
|
liveRows: config.liveRows,
|
|
80
106
|
prepare: config.prepare,
|
|
81
107
|
keepRow: undefined,
|
|
108
|
+
hydration: undefined,
|
|
109
|
+
keyConditionDiagnosis: undefined,
|
|
82
110
|
tableName: config.tableName,
|
|
83
111
|
indexName: config.indexName,
|
|
84
112
|
pkField: "",
|
|
@@ -93,7 +121,6 @@ export const makeScan = (config) => new QueryImpl({
|
|
|
93
121
|
maxPagesValue: undefined,
|
|
94
122
|
scanForward: true,
|
|
95
123
|
consistentRead: false,
|
|
96
|
-
globalIndex: config.globalIndex ?? false,
|
|
97
124
|
ignoreOwnershipFlag: false,
|
|
98
125
|
exclusiveStartKey: undefined,
|
|
99
126
|
isScan: true,
|
|
@@ -160,9 +187,10 @@ export const reverse = (self) => new QueryImpl({
|
|
|
160
187
|
scanForward: false,
|
|
161
188
|
});
|
|
162
189
|
/**
|
|
163
|
-
* Enable consistent reads for this query (or scan).
|
|
164
|
-
*
|
|
165
|
-
*
|
|
190
|
+
* Enable consistent reads for this query (or scan). Sent as asked: DynamoDB
|
|
191
|
+
* rejects it on a global secondary index, which it reads only eventually
|
|
192
|
+
* consistently, and the query fails with a `ValidationError`
|
|
193
|
+
* (`operation: "query.consistentRead"`) naming the index (#144).
|
|
166
194
|
*/
|
|
167
195
|
export const consistentRead = Function.dual((args) => isQuery(args[0]), (self) => new QueryImpl({
|
|
168
196
|
...self._state,
|
|
@@ -196,20 +224,35 @@ export const startFrom = Function.dual(2, (self, cursor) => cursor === ""
|
|
|
196
224
|
...self._state,
|
|
197
225
|
exclusiveStartKey: decodeCursor(cursor),
|
|
198
226
|
}));
|
|
227
|
+
/** @internal The decoder of a projected (raw) row: the raw-row hook, or the identity. */
|
|
228
|
+
const rawRowDecoder = (hook) => hook ?? ((raw) => Effect.succeed(raw));
|
|
229
|
+
/** @internal Throw `[EDD-9076]` for a projection inside a packed attribute. */
|
|
230
|
+
const refusePackedProjection = (state, paths) => {
|
|
231
|
+
const refused = packedProjectionRefusal(paths, state.packed);
|
|
232
|
+
if (refused !== undefined)
|
|
233
|
+
throw new Error(refused);
|
|
234
|
+
};
|
|
199
235
|
/**
|
|
200
236
|
* Select specific attributes to return (ProjectionExpression).
|
|
201
237
|
* Reduces read capacity and network transfer. When projection is active,
|
|
202
238
|
* items are returned as raw `Record<string, unknown>` instead of decoded
|
|
203
239
|
* schema instances, since projected items may be partial.
|
|
240
|
+
*
|
|
241
|
+
* An attribute stored packed (`DynamoModel.Packed` — an EventStore stream's
|
|
242
|
+
* `data`, for one) comes back as its parsed JSON, whatever its stored form
|
|
243
|
+
* (a native map, a JSON string, or compressed bytes), and may only be
|
|
244
|
+
* selected whole: a path inside it throws `[EDD-9076]`.
|
|
204
245
|
*/
|
|
205
246
|
export const select = Function.dual(2, (self, attributes) => {
|
|
206
247
|
if (self._state.predicates.length > 0) {
|
|
207
248
|
throw new Error(rejectPredicateWithProjection("select() after filterBy()"));
|
|
208
249
|
}
|
|
250
|
+
const paths = attributes.map((attribute) => [attribute]);
|
|
251
|
+
refusePackedProjection(self._state, paths);
|
|
209
252
|
return new QueryImpl({
|
|
210
253
|
...self._state,
|
|
211
254
|
projection: attributes,
|
|
212
|
-
decoder: (
|
|
255
|
+
decoder: rawRowDecoder(projectedRawRow(self._state.rawRow, paths)),
|
|
213
256
|
});
|
|
214
257
|
});
|
|
215
258
|
/**
|
|
@@ -222,11 +265,13 @@ export const selectProjected = (self, storedPaths, decode) => {
|
|
|
222
265
|
if (self._state.predicates.length > 0) {
|
|
223
266
|
throw new Error(rejectPredicateWithProjection("select() after filterBy()"));
|
|
224
267
|
}
|
|
268
|
+
refusePackedProjection(self._state, storedPaths);
|
|
269
|
+
const rawRow = rawRowDecoder(projectedRawRow(self._state.rawRow, storedPaths));
|
|
225
270
|
return new QueryImpl({
|
|
226
271
|
...self._state,
|
|
227
272
|
projection: undefined,
|
|
228
273
|
projectionPaths: storedPaths,
|
|
229
|
-
decoder: (raw) => Effect.
|
|
274
|
+
decoder: (raw) => Effect.map(rawRow(raw), (row) => decode(row)),
|
|
230
275
|
});
|
|
231
276
|
};
|
|
232
277
|
/**
|
|
@@ -250,6 +295,12 @@ export const selectRenamed = (self, paths, resolveDbName) => {
|
|
|
250
295
|
/**
|
|
251
296
|
* Add an Expr-based filter expression to the query.
|
|
252
297
|
* Multiple filterExpr calls are ANDed together.
|
|
298
|
+
*
|
|
299
|
+
* On a query whose items store an attribute packed (`DynamoModel.Packed` — an
|
|
300
|
+
* EventStore stream with `storedAs.data`, for one), a filter that addresses a
|
|
301
|
+
* path inside it, or applies anything but `attribute_exists` /
|
|
302
|
+
* `attribute_not_exists` to it, throws `[EDD-9076]`: DynamoDB sees one opaque
|
|
303
|
+
* value. Use {@link filterBy}, which runs on the decoded item.
|
|
253
304
|
*/
|
|
254
305
|
export const filterExpr = Function.dual(2, (self, expr) => {
|
|
255
306
|
// A filter that asserts nothing (`.filter({})`, `and()`) is no filter: it is
|
|
@@ -259,6 +310,9 @@ export const filterExpr = Function.dual(2, (self, expr) => {
|
|
|
259
310
|
// (`emptyPartProblem`).
|
|
260
311
|
if (isEmptyExpr(expr))
|
|
261
312
|
return self;
|
|
313
|
+
const refused = packedExprRefusal(expr, self._state.packed, "filter");
|
|
314
|
+
if (refused !== undefined)
|
|
315
|
+
throw new Error(refused);
|
|
262
316
|
return new QueryImpl({
|
|
263
317
|
...self._state,
|
|
264
318
|
exprFilters: [...self._state.exprFilters, expr],
|
|
@@ -333,10 +387,11 @@ export const selectPaths = Function.dual(2, (self, paths) => {
|
|
|
333
387
|
if (self._state.predicates.length > 0) {
|
|
334
388
|
throw new Error(rejectPredicateWithProjection("select() after filterBy()"));
|
|
335
389
|
}
|
|
390
|
+
refusePackedProjection(self._state, paths);
|
|
336
391
|
return new QueryImpl({
|
|
337
392
|
...self._state,
|
|
338
393
|
projectionPaths: paths,
|
|
339
|
-
decoder: (
|
|
394
|
+
decoder: rawRowDecoder(projectedRawRow(self._state.rawRow, paths)),
|
|
340
395
|
});
|
|
341
396
|
});
|
|
342
397
|
// ---------------------------------------------------------------------------
|
|
@@ -468,9 +523,100 @@ const buildDynamoCommand = (state, tableName, overrides, extraProjectionFields =
|
|
|
468
523
|
ConsistentRead: input.ConsistentRead,
|
|
469
524
|
Limit: computeRequestLimit(state, state.limitValue),
|
|
470
525
|
ScanIndexForward: state.isScan ? undefined : state.scanForward,
|
|
526
|
+
// An LSI that does not project the items: DynamoDB fetches them from the
|
|
527
|
+
// table. Not with a ProjectionExpression, which DynamoDB rejects beside
|
|
528
|
+
// `Select: ALL_ATTRIBUTES` (and which fetches the named attributes itself).
|
|
529
|
+
...(state.hydration?._tag === "Fetch" && input.ProjectionExpression === undefined
|
|
530
|
+
? { Select: "ALL_ATTRIBUTES" }
|
|
531
|
+
: {}),
|
|
471
532
|
...overrides,
|
|
472
533
|
};
|
|
473
534
|
};
|
|
535
|
+
/**
|
|
536
|
+
* @internal The state the index request is built from. A `BatchGet`-hydrated
|
|
537
|
+
* query sends its projection on the `BatchGetItem`, not on the index query
|
|
538
|
+
* (whose rows only carry keys).
|
|
539
|
+
*/
|
|
540
|
+
const indexQueryState = (state) => state.hydration?._tag === "BatchGet"
|
|
541
|
+
? { ...state, projection: undefined, projectionPaths: undefined }
|
|
542
|
+
: state;
|
|
543
|
+
/**
|
|
544
|
+
* @internal Replace index rows with the table items they refer to, for a
|
|
545
|
+
* `BatchGet`-hydrated query (see {@link QueryHydration}); any other query's
|
|
546
|
+
* rows are returned as they are. The result is in `rows` order, `undefined`
|
|
547
|
+
* for an item that no longer exists (it is dropped like an excluded row).
|
|
548
|
+
*/
|
|
549
|
+
const hydrateRows = (client, tableName, state, rows) => {
|
|
550
|
+
const hydration = state.hydration;
|
|
551
|
+
if (hydration?._tag !== "BatchGet" || rows.length === 0)
|
|
552
|
+
return Effect.succeed(rows);
|
|
553
|
+
return Effect.gen(function* () {
|
|
554
|
+
const [pkName, skName] = hydration.tableKey;
|
|
555
|
+
const requests = [];
|
|
556
|
+
for (const row of rows) {
|
|
557
|
+
const pk = row[pkName];
|
|
558
|
+
const sk = row[skName];
|
|
559
|
+
if (pk === undefined || sk === undefined) {
|
|
560
|
+
// Table keys are always projected into a secondary index.
|
|
561
|
+
return yield* Effect.die(new Error(`Query hydration: an index row of "${state.indexName}" lacks the table key ` +
|
|
562
|
+
`attribute "${pk === undefined ? pkName : skName}".`));
|
|
563
|
+
}
|
|
564
|
+
requests.push({ tableName, key: { [pkName]: pk, [skName]: sk } });
|
|
565
|
+
}
|
|
566
|
+
const ownership = !state.ignoreOwnershipFlag && state.entityTypes.length > 0;
|
|
567
|
+
// A `select` moves here: the projection, plus what matching (the table
|
|
568
|
+
// key) and the ownership check read — stripped again unless selected.
|
|
569
|
+
const names = {};
|
|
570
|
+
const projected = new Set([
|
|
571
|
+
...(state.projection ?? []),
|
|
572
|
+
...(state.projectionPaths ?? []).map((segments) => String(segments[0])),
|
|
573
|
+
]);
|
|
574
|
+
const borrowed = projected.size === 0
|
|
575
|
+
? []
|
|
576
|
+
: [
|
|
577
|
+
...new Set([
|
|
578
|
+
pkName,
|
|
579
|
+
skName,
|
|
580
|
+
...(ownership ? ["__edd_e__"] : []),
|
|
581
|
+
...rawRowReads(state),
|
|
582
|
+
]),
|
|
583
|
+
].filter((field) => !projected.has(field));
|
|
584
|
+
// What the raw-row hook reads survives until it has run (see `examinePage`).
|
|
585
|
+
const keptForRawRow = new Set(rawRowReads(state));
|
|
586
|
+
const expression = buildProjection(state, names, borrowed);
|
|
587
|
+
const fetched = yield* batchGetRaw(client, requests, {
|
|
588
|
+
projection: expression === undefined ? undefined : { expression, names },
|
|
589
|
+
rereadMissing: true,
|
|
590
|
+
});
|
|
591
|
+
const hydrated = [];
|
|
592
|
+
for (const item of fetched) {
|
|
593
|
+
if (item === undefined) {
|
|
594
|
+
hydrated.push(undefined);
|
|
595
|
+
continue;
|
|
596
|
+
}
|
|
597
|
+
if (ownership) {
|
|
598
|
+
const owner = item.__edd_e__?.S;
|
|
599
|
+
if (owner === undefined || !state.entityTypes.includes(owner)) {
|
|
600
|
+
// The index row passed the same `__edd_e__` filter: a defect.
|
|
601
|
+
return yield* Effect.die(new Error(`Query hydration: an item fetched through index "${state.indexName}" has ` +
|
|
602
|
+
`__edd_e__ ${JSON.stringify(owner)}, not ` +
|
|
603
|
+
`${state.entityTypes.map((t) => JSON.stringify(t)).join(" or ")}, although ` +
|
|
604
|
+
"its index row passed the same filter."));
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
if (borrowed.length === 0) {
|
|
608
|
+
hydrated.push(item);
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
const stripped = { ...item };
|
|
612
|
+
for (const field of borrowed)
|
|
613
|
+
if (!keptForRawRow.has(field))
|
|
614
|
+
delete stripped[field];
|
|
615
|
+
hydrated.push(stripped);
|
|
616
|
+
}
|
|
617
|
+
return hydrated;
|
|
618
|
+
});
|
|
619
|
+
};
|
|
474
620
|
// ---------------------------------------------------------------------------
|
|
475
621
|
// Internal: limit / pageSize execution helpers
|
|
476
622
|
// ---------------------------------------------------------------------------
|
|
@@ -546,15 +692,7 @@ const refuseEmptyFilterParts = (state) => {
|
|
|
546
692
|
*/
|
|
547
693
|
const prepared = (state, tableName) => {
|
|
548
694
|
// A consistent read of a GSI — which DynamoDB only reads eventually
|
|
549
|
-
// consistently — is
|
|
550
|
-
if (state.consistentRead && state.globalIndex) {
|
|
551
|
-
return Effect.fail(new ValidationError({
|
|
552
|
-
entityType: state.entityTypes.join(", ") || "unknown",
|
|
553
|
-
operation: "query.consistentRead",
|
|
554
|
-
cause: `consistentRead: index "${state.indexName}" is a global secondary index, which ` +
|
|
555
|
-
"DynamoDB reads only eventually consistently. Nothing was sent.",
|
|
556
|
-
}));
|
|
557
|
-
}
|
|
695
|
+
// consistently — is DynamoDB's to reject: diagnosed by `send` (#144).
|
|
558
696
|
// A filter with an empty part where none may be is refused before
|
|
559
697
|
// anything is sent (#133).
|
|
560
698
|
const filterRefusal = refuseEmptyFilterParts(state);
|
|
@@ -576,6 +714,38 @@ const prepared = (state, tableName) => {
|
|
|
576
714
|
};
|
|
577
715
|
});
|
|
578
716
|
};
|
|
717
|
+
/**
|
|
718
|
+
* @internal Send one request of a terminal. DynamoDB's rejection of what the
|
|
719
|
+
* library does not pre-check (#144) is diagnosed against the request sent, on
|
|
720
|
+
* failure only: a consistent read of a GSI, the key condition's rules (an
|
|
721
|
+
* empty key value, a reversed `BETWEEN`) and an `isIn()` with no values in the
|
|
722
|
+
* filter, named as {@link KeyConditionDiagnosis} says. Anything else passes
|
|
723
|
+
* through.
|
|
724
|
+
*/
|
|
725
|
+
const send = (client, state, cmd) => {
|
|
726
|
+
const request = state.isScan ? client.scan(cmd) : client.query(cmd);
|
|
727
|
+
return request.pipe(diagnoseFailure({
|
|
728
|
+
sent: () => ({ _tag: "Read", scan: state.isScan, input: cmd }),
|
|
729
|
+
only: ["ConsistentReadOnGsi", "EmptyKeyBound", "BetweenReversed", "EmptyIn"],
|
|
730
|
+
rewrite: (diagnosis, original) => {
|
|
731
|
+
const entityType = state.entityTypes.join(", ") || "unknown";
|
|
732
|
+
const naming = state.keyConditionDiagnosis ?? {
|
|
733
|
+
operation: "query",
|
|
734
|
+
label: `query of ${entityType}`,
|
|
735
|
+
};
|
|
736
|
+
const [operation, label] = diagnosis._tag === "ConsistentReadOnGsi"
|
|
737
|
+
? ["query.consistentRead", "query"]
|
|
738
|
+
: diagnosis._tag === "EmptyIn"
|
|
739
|
+
? ["query.filter", naming.label]
|
|
740
|
+
: [naming.operation, naming.label];
|
|
741
|
+
return new ValidationError({
|
|
742
|
+
entityType,
|
|
743
|
+
operation,
|
|
744
|
+
cause: withCause(explainDiagnosis(diagnosis, label), original),
|
|
745
|
+
});
|
|
746
|
+
},
|
|
747
|
+
}));
|
|
748
|
+
};
|
|
579
749
|
/** @internal Does the decoded item pass every client-side predicate? */
|
|
580
750
|
const accepts = (state, item) => {
|
|
581
751
|
for (const predicate of state.predicates) {
|
|
@@ -627,8 +797,74 @@ forCursor) => {
|
|
|
627
797
|
borrowed.add(state.skField);
|
|
628
798
|
for (const attr of state.liveRows?.reads ?? [])
|
|
629
799
|
borrowed.add(attr);
|
|
800
|
+
// …and a raw-row hook reads what it dispatches on (a collection's `__edd_e__`).
|
|
801
|
+
for (const attr of rawRowReads(state))
|
|
802
|
+
borrowed.add(attr);
|
|
630
803
|
return [...borrowed].filter((field) => !projected.has(field));
|
|
631
804
|
};
|
|
805
|
+
/**
|
|
806
|
+
* @internal Examine one response's rows in order, as `execute` and `paginate`
|
|
807
|
+
* both do: drop excluded rows, hydrate (see {@link QueryHydration}), decode,
|
|
808
|
+
* apply the client predicates, and stop once `budget` items are accepted.
|
|
809
|
+
*
|
|
810
|
+
* Without a client predicate every row is accepted, so only the rows the
|
|
811
|
+
* budget can still take are hydrated and decoded; a row dropped after
|
|
812
|
+
* hydration (its item gone) widens the window by one. With a predicate,
|
|
813
|
+
* acceptance is only known after decode, so the whole response is examined
|
|
814
|
+
* (#122). `stoppedAt` is the index of the row that filled the budget, or -1.
|
|
815
|
+
*/
|
|
816
|
+
const examinePage = (client, tableName, state, returned, budget, borrowedFields) => Effect.gen(function* () {
|
|
817
|
+
const hasPredicate = state.predicates.length > 0 || dropsRows(state);
|
|
818
|
+
// Borrowed attributes the raw-row hook reads are stripped after it ran:
|
|
819
|
+
// borrowed into the request's projection, or — on a `BatchGet`-hydrated
|
|
820
|
+
// query — into the hydrating fetch's (see `hydrateRows`).
|
|
821
|
+
const hookReads = new Set(rawRowReads(state));
|
|
822
|
+
const strippedAfterHook = new Set(borrowedFields.filter((field) => hookReads.has(field)));
|
|
823
|
+
if (state.hydration?._tag === "BatchGet") {
|
|
824
|
+
const projected = new Set([
|
|
825
|
+
...(state.projection ?? []),
|
|
826
|
+
...(state.projectionPaths ?? []).map((segments) => String(segments[0])),
|
|
827
|
+
]);
|
|
828
|
+
if (projected.size > 0) {
|
|
829
|
+
for (const field of hookReads)
|
|
830
|
+
if (!projected.has(field))
|
|
831
|
+
strippedAfterHook.add(field);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
const items = [];
|
|
835
|
+
let next = 0;
|
|
836
|
+
while (next < returned.length) {
|
|
837
|
+
const end = budget === undefined || hasPredicate
|
|
838
|
+
? returned.length
|
|
839
|
+
: Math.min(returned.length, next + (budget - items.length));
|
|
840
|
+
const positions = [];
|
|
841
|
+
for (let i = next; i < end; i++) {
|
|
842
|
+
if (!isExcludedRow(state, returned[i]))
|
|
843
|
+
positions.push(i);
|
|
844
|
+
}
|
|
845
|
+
const sources = yield* hydrateRows(client, tableName, state, positions.map((i) => returned[i]));
|
|
846
|
+
for (let k = 0; k < positions.length; k++) {
|
|
847
|
+
const source = sources[k];
|
|
848
|
+
if (source === undefined)
|
|
849
|
+
continue;
|
|
850
|
+
const raw = fromAttributeMap(source);
|
|
851
|
+
for (const field of borrowedFields)
|
|
852
|
+
if (!hookReads.has(field))
|
|
853
|
+
delete raw[field];
|
|
854
|
+
const item = yield* state.decoder(raw);
|
|
855
|
+
for (const field of strippedAfterHook)
|
|
856
|
+
delete raw[field];
|
|
857
|
+
if (hasPredicate && !accepts(state, item))
|
|
858
|
+
continue;
|
|
859
|
+
items.push(item);
|
|
860
|
+
if (budget !== undefined && items.length >= budget) {
|
|
861
|
+
return { items, stoppedAt: positions[k] };
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
next = end;
|
|
865
|
+
}
|
|
866
|
+
return { items, stoppedAt: -1 };
|
|
867
|
+
});
|
|
632
868
|
// ---------------------------------------------------------------------------
|
|
633
869
|
// Terminals
|
|
634
870
|
// ---------------------------------------------------------------------------
|
|
@@ -666,14 +902,14 @@ export const execute = (self) => Effect.gen(function* () {
|
|
|
666
902
|
? yield* self._state.resolveTableName
|
|
667
903
|
: self._state.tableName;
|
|
668
904
|
const state = yield* prepared(self._state, tableName);
|
|
905
|
+
const indexState = indexQueryState(state);
|
|
669
906
|
const limitValue = state.limitValue;
|
|
670
907
|
if (limitValue !== undefined && limitValue <= 0) {
|
|
671
908
|
return { items: [], cursor: null };
|
|
672
909
|
}
|
|
673
910
|
// Key attributes borrowed into an active projection so an over-reading
|
|
674
911
|
// request can still rebuild a cursor. Stripped again before decoding.
|
|
675
|
-
const borrowedFields = cursorProjectionFields(
|
|
676
|
-
const hasPredicate = state.predicates.length > 0 || dropsRows(state);
|
|
912
|
+
const borrowedFields = cursorProjectionFields(indexState, true);
|
|
677
913
|
const items = [];
|
|
678
914
|
let startKey = state.exclusiveStartKey;
|
|
679
915
|
let pageCount = 0;
|
|
@@ -681,34 +917,14 @@ export const execute = (self) => Effect.gen(function* () {
|
|
|
681
917
|
while (true) {
|
|
682
918
|
pageCount++;
|
|
683
919
|
const remaining = limitValue === undefined ? undefined : limitValue - items.length;
|
|
684
|
-
const cmd = buildDynamoCommand(
|
|
685
|
-
const result =
|
|
920
|
+
const cmd = buildDynamoCommand(indexState, tableName, { ExclusiveStartKey: startKey, Limit: computeRequestLimit(state, remaining, pageCount) }, borrowedFields);
|
|
921
|
+
const result = yield* send(client, state, cmd);
|
|
686
922
|
const returned = (result.Items ?? []);
|
|
687
923
|
const lastEvaluatedKey = result.LastEvaluatedKey;
|
|
688
|
-
//
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
const take = remaining === undefined || hasPredicate
|
|
693
|
-
? returned.length
|
|
694
|
-
: Math.min(remaining, returned.length);
|
|
695
|
-
/** Index of the last row examined when the limit was reached, else -1. */
|
|
696
|
-
let stoppedAt = -1;
|
|
697
|
-
for (let i = 0; i < take; i++) {
|
|
698
|
-
if (isExcludedRow(state, returned[i]))
|
|
699
|
-
continue;
|
|
700
|
-
const raw = fromAttributeMap(returned[i]);
|
|
701
|
-
for (const field of borrowedFields)
|
|
702
|
-
delete raw[field];
|
|
703
|
-
const item = yield* state.decoder(raw);
|
|
704
|
-
if (hasPredicate && !accepts(state, item))
|
|
705
|
-
continue;
|
|
706
|
-
items.push(item);
|
|
707
|
-
if (limitValue !== undefined && items.length >= limitValue) {
|
|
708
|
-
stoppedAt = i;
|
|
709
|
-
break;
|
|
710
|
-
}
|
|
711
|
-
}
|
|
924
|
+
// Index of the last row examined when the limit was reached, else -1.
|
|
925
|
+
const page = yield* examinePage(client, tableName, state, returned, remaining, borrowedFields);
|
|
926
|
+
items.push(...page.items);
|
|
927
|
+
const stoppedAt = page.stoppedAt;
|
|
712
928
|
// Stopped inside the page: the surplus was discarded, so the cursor has
|
|
713
929
|
// to be rebuilt from the last item handed back rather than from
|
|
714
930
|
// LastEvaluatedKey. `stoppedAt` is the last row ACCEPTED, which is what
|
|
@@ -759,30 +975,15 @@ const pageStream = (client, tableName, state) => {
|
|
|
759
975
|
}, (pageState) => Effect.gen(function* () {
|
|
760
976
|
const pageCount = pageState.pageCount + 1;
|
|
761
977
|
const remaining = limitValue === undefined ? undefined : limitValue - pageState.emitted;
|
|
762
|
-
const
|
|
763
|
-
const
|
|
978
|
+
const indexState = indexQueryState(state);
|
|
979
|
+
const borrowedFields = cursorProjectionFields(indexState, false);
|
|
980
|
+
const cmd = buildDynamoCommand(indexState, tableName, {
|
|
764
981
|
ExclusiveStartKey: pageState.key,
|
|
765
982
|
Limit: computeRequestLimit(state, remaining, pageCount),
|
|
766
983
|
}, borrowedFields);
|
|
767
|
-
const result =
|
|
984
|
+
const result = yield* send(client, state, cmd);
|
|
768
985
|
const returned = (result.Items ?? []);
|
|
769
|
-
|
|
770
|
-
// the budget cannot bound the examine window, only the accepted one.
|
|
771
|
-
const hasPredicate = state.predicates.length > 0 || dropsRows(state);
|
|
772
|
-
const take = remaining === undefined || hasPredicate
|
|
773
|
-
? returned.length
|
|
774
|
-
: Math.min(remaining, returned.length);
|
|
775
|
-
const examined = yield* Effect.forEach(returned
|
|
776
|
-
.slice(0, take)
|
|
777
|
-
.filter((item) => !isExcludedRow(state, item))
|
|
778
|
-
.map((item) => {
|
|
779
|
-
const raw = fromAttributeMap(item);
|
|
780
|
-
for (const field of borrowedFields)
|
|
781
|
-
delete raw[field];
|
|
782
|
-
return raw;
|
|
783
|
-
}), (raw) => state.decoder(raw));
|
|
784
|
-
const kept = hasPredicate ? examined.filter((item) => accepts(state, item)) : examined;
|
|
785
|
-
const decoded = remaining === undefined ? kept : kept.slice(0, remaining);
|
|
986
|
+
const { items: decoded } = yield* examinePage(client, tableName, state, returned, remaining, borrowedFields);
|
|
786
987
|
const emitted = pageState.emitted + decoded.length;
|
|
787
988
|
const hasMorePages = result.LastEvaluatedKey != null;
|
|
788
989
|
const maxPagesReached = state.maxPagesValue != null && pageCount >= state.maxPagesValue;
|
|
@@ -811,7 +1012,9 @@ export const count = (self) => Effect.gen(function* () {
|
|
|
811
1012
|
const tableName = self._state.resolveTableName
|
|
812
1013
|
? yield* self._state.resolveTableName
|
|
813
1014
|
: self._state.tableName;
|
|
814
|
-
|
|
1015
|
+
// A hydrated query counts index rows: nothing is fetched (and a `select`
|
|
1016
|
+
// projection, which hydration would send on the fetch, is not sent).
|
|
1017
|
+
const state = indexQueryState(yield* prepared(self._state, tableName));
|
|
815
1018
|
const limitValue = state.limitValue;
|
|
816
1019
|
if (limitValue !== undefined && limitValue <= 0)
|
|
817
1020
|
return 0;
|
|
@@ -846,7 +1049,7 @@ export const count = (self) => Effect.gen(function* () {
|
|
|
846
1049
|
ExclusiveStartKey: start,
|
|
847
1050
|
Limit: computeRequestLimit(state, remaining, pages),
|
|
848
1051
|
});
|
|
849
|
-
const result =
|
|
1052
|
+
const result = yield* send(client, state, cmd);
|
|
850
1053
|
const rows = (result.Items ?? []);
|
|
851
1054
|
counted += rows.filter((row) => !isExcludedRow(state, row)).length;
|
|
852
1055
|
start = result.LastEvaluatedKey;
|
|
@@ -868,7 +1071,7 @@ export const count = (self) => Effect.gen(function* () {
|
|
|
868
1071
|
Select: "COUNT",
|
|
869
1072
|
Limit: computeRequestLimit(state, remaining),
|
|
870
1073
|
});
|
|
871
|
-
const result =
|
|
1074
|
+
const result = yield* send(client, state, cmd);
|
|
872
1075
|
total += result.Count ?? 0;
|
|
873
1076
|
exclusiveStartKey = result.LastEvaluatedKey;
|
|
874
1077
|
if (limitValue !== undefined && total >= limitValue)
|
|
@@ -882,8 +1085,7 @@ export const count = (self) => Effect.gen(function* () {
|
|
|
882
1085
|
* Return the built DynamoDB command input without executing.
|
|
883
1086
|
* Useful for debugging, logging, or passing to DynamoClient directly.
|
|
884
1087
|
* Fails with a `ValidationError` for a filter the query would refuse to send
|
|
885
|
-
* (an empty part under `or` / `not`, an `or()` with no parts
|
|
886
|
-
* no values).
|
|
1088
|
+
* (an empty part under `or` / `not`, an `or()` with no parts).
|
|
887
1089
|
*/
|
|
888
1090
|
export const asParams = (self) => Effect.gen(function* () {
|
|
889
1091
|
const state = self._state;
|
|
@@ -891,7 +1093,9 @@ export const asParams = (self) => Effect.gen(function* () {
|
|
|
891
1093
|
if (refusal !== undefined)
|
|
892
1094
|
return yield* refusal;
|
|
893
1095
|
const tableName = state.resolveTableName ? yield* state.resolveTableName : state.tableName;
|
|
894
|
-
|
|
1096
|
+
// A `BatchGet`-hydrated query shows its index query; the `BatchGetItem`
|
|
1097
|
+
// that follows each page is not part of these parameters.
|
|
1098
|
+
return buildDynamoCommand(indexQueryState(state), tableName);
|
|
895
1099
|
});
|
|
896
1100
|
// ---------------------------------------------------------------------------
|
|
897
1101
|
// Type guard
|