@spine-event-engine/storage-datastore 2.0.0-snapshot.2 → 2.0.0-snapshot.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -9
- package/REFERENCE.md +9 -7
- package/dist/datastore/delivery-cleanup.d.ts +1 -1
- package/dist/datastore/delivery-cleanup.d.ts.map +1 -1
- package/dist/datastore/delivery-cleanup.js +1 -1
- package/dist/datastore/delivery-cleanup.js.map +1 -1
- package/dist/datastore/entity-history.d.ts +76 -4
- package/dist/datastore/entity-history.d.ts.map +1 -1
- package/dist/datastore/entity-history.js +266 -18
- package/dist/datastore/entity-history.js.map +1 -1
- package/dist/datastore/namespace.js +1 -1
- package/dist/datastore/namespace.js.map +1 -1
- package/dist/datastore/record-storage.d.ts +51 -0
- package/dist/datastore/record-storage.d.ts.map +1 -1
- package/dist/datastore/record-storage.js +351 -23
- package/dist/datastore/record-storage.js.map +1 -1
- package/dist/datastore/storage-factory.d.ts +3 -2
- package/dist/datastore/storage-factory.d.ts.map +1 -1
- package/dist/datastore/storage-factory.js +3 -2
- package/dist/datastore/storage-factory.js.map +1 -1
- package/dist/datastore/tenant-catalog.d.ts +1 -1
- package/dist/datastore/tenant-catalog.d.ts.map +1 -1
- package/dist/datastore/tenant-catalog.js +1 -1
- package/dist/datastore/tenant-catalog.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +7 -8
|
@@ -14,8 +14,9 @@
|
|
|
14
14
|
import { fromBinary, toBinary } from "@bufbuild/protobuf";
|
|
15
15
|
import { and, Datastore, or, PropertyFilter } from "@google-cloud/datastore";
|
|
16
16
|
import { StringifierRegistry } from "@spine-event-engine/core";
|
|
17
|
-
import { ColumnMappings, defaultQueryCandidateLimit, RecordStorage,
|
|
18
|
-
import {
|
|
17
|
+
import { ColumnMappings, defaultQueryCandidateLimit, RecordStorage, } from "@spine-event-engine/storage";
|
|
18
|
+
import { TenantBoundary } from "@spine-event-engine/storage/provider";
|
|
19
|
+
import { StorageQueryValues } from "@spine-event-engine/storage/provider";
|
|
19
20
|
import { DatastoreColumnMapping } from "./column-mapping.js";
|
|
20
21
|
import { DatastoreIdColumn } from "./id-column.js";
|
|
21
22
|
import { DefaultNamespaceConverter, NamespaceAssignments, } from "./namespace.js";
|
|
@@ -26,6 +27,9 @@ const casRetryDelayMs = 100;
|
|
|
26
27
|
const wrappedReadOptions = { wrapNumbers: true };
|
|
27
28
|
/**
|
|
28
29
|
* Private flat Datastore-entity codec for one record storage handle.
|
|
30
|
+
*
|
|
31
|
+
* @typeParam I Identifier type encoded in Datastore keys.
|
|
32
|
+
* @typeParam R Generated record message type encoded as payload bytes.
|
|
29
33
|
*/
|
|
30
34
|
class FlatEntityCodec {
|
|
31
35
|
recordSpec;
|
|
@@ -35,6 +39,18 @@ class FlatEntityCodec {
|
|
|
35
39
|
#namespace;
|
|
36
40
|
#idColumn;
|
|
37
41
|
#columnMapping;
|
|
42
|
+
/**
|
|
43
|
+
* Initializes the Datastore kind, tenant namespace, ID codec, and column mapping.
|
|
44
|
+
*
|
|
45
|
+
* @param context Storage tenant context.
|
|
46
|
+
* @param recordSpec Generated record and column contract.
|
|
47
|
+
* @param client Datastore client supplying the default namespace.
|
|
48
|
+
* @param maxClientSideScan Maximum local materialization budget.
|
|
49
|
+
* @param group Optional storage group used in the physical kind.
|
|
50
|
+
* @param kind Optional explicit physical kind.
|
|
51
|
+
* @param namespaceConverter Converts tenant IDs to Datastore namespaces.
|
|
52
|
+
* @param stringifiers Converts message-valued IDs and columns.
|
|
53
|
+
*/
|
|
38
54
|
constructor(context, recordSpec, client, maxClientSideScan, group, kind, namespaceConverter, stringifiers) {
|
|
39
55
|
this.recordSpec = recordSpec;
|
|
40
56
|
this.maxClientSideScan = maxClientSideScan;
|
|
@@ -53,12 +69,34 @@ class FlatEntityCodec {
|
|
|
53
69
|
this.#idColumn = new DatastoreIdColumn(recordSpec.idType, stringifiers);
|
|
54
70
|
this.#columnMapping = new DatastoreColumnMapping(stringifiers);
|
|
55
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Builds a namespaced Datastore key for a record identifier.
|
|
74
|
+
*
|
|
75
|
+
* @param client Datastore client that creates keys.
|
|
76
|
+
* @param id Typed record identifier.
|
|
77
|
+
* @returns Physical key for this kind and namespace.
|
|
78
|
+
*/
|
|
56
79
|
key(client, id) {
|
|
57
80
|
return client.key({
|
|
58
|
-
path: [this.#kind, this
|
|
81
|
+
path: [this.#kind, this.idName(id)],
|
|
59
82
|
...(this.#namespace === undefined ? {} : { namespace: this.#namespace }),
|
|
60
83
|
});
|
|
61
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* Returns the canonical Datastore key name for a logical record ID.
|
|
87
|
+
* @param id Typed record identifier.
|
|
88
|
+
* @returns Key name shared by equivalent logical IDs.
|
|
89
|
+
*/
|
|
90
|
+
idName(id) {
|
|
91
|
+
return this.#idColumn.value(id);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Encodes record bytes and declared materialized columns as entity properties.
|
|
95
|
+
*
|
|
96
|
+
* @param record Generated record to serialize.
|
|
97
|
+
* @param columns Materialized column values by declared name.
|
|
98
|
+
* @returns Datastore property map including the binary payload.
|
|
99
|
+
*/
|
|
62
100
|
encode(record, columns) {
|
|
63
101
|
const data = {
|
|
64
102
|
[payloadProperty]: Buffer.from(toBinary(this.recordSpec.recordType, record, { writeUnknownFields: false })),
|
|
@@ -68,6 +106,13 @@ class FlatEntityCodec {
|
|
|
68
106
|
}
|
|
69
107
|
return data;
|
|
70
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* Decodes a record ID from an entity's physical key.
|
|
111
|
+
*
|
|
112
|
+
* @param entity Raw Datastore entity.
|
|
113
|
+
* @param client Client whose key symbol identifies the entity key.
|
|
114
|
+
* @returns Typed record identifier.
|
|
115
|
+
*/
|
|
71
116
|
id(entity, client) {
|
|
72
117
|
const key = datastoreKey(entity, client.KEY);
|
|
73
118
|
const name = datastoreKeyName(key);
|
|
@@ -76,36 +121,85 @@ class FlatEntityCodec {
|
|
|
76
121
|
}
|
|
77
122
|
return this.#idColumn.read(name);
|
|
78
123
|
}
|
|
124
|
+
/**
|
|
125
|
+
* Reads declared column values from a decoded entity record.
|
|
126
|
+
*
|
|
127
|
+
* @param entity Raw Datastore entity.
|
|
128
|
+
* @returns Declared column values by name.
|
|
129
|
+
*/
|
|
79
130
|
columns(entity) {
|
|
80
131
|
return new Map(this.recordSpec.columns.map((column) => [column.name, column.valueIn(this.decode(entity))]));
|
|
81
132
|
}
|
|
133
|
+
/**
|
|
134
|
+
* Marks the binary record payload as excluded from Datastore indexes.
|
|
135
|
+
*
|
|
136
|
+
* @param data Entity property map being saved.
|
|
137
|
+
* @returns Payload property name to leave unindexed.
|
|
138
|
+
*/
|
|
82
139
|
unindexedProperties(data) {
|
|
83
140
|
void data;
|
|
84
141
|
return [payloadProperty];
|
|
85
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* Creates a provider query for this handle's kind and namespace.
|
|
145
|
+
*
|
|
146
|
+
* @param client Datastore client that creates queries.
|
|
147
|
+
* @returns Query scoped to this record family.
|
|
148
|
+
*/
|
|
86
149
|
createQuery(client) {
|
|
87
150
|
return this.#namespace === undefined
|
|
88
151
|
? client.createQuery(this.#kind)
|
|
89
152
|
: client.createQuery(this.#namespace, this.#kind);
|
|
90
153
|
}
|
|
154
|
+
/**
|
|
155
|
+
* Validates a declared column and returns its Datastore property name.
|
|
156
|
+
*
|
|
157
|
+
* @param column Requested record column name.
|
|
158
|
+
* @returns Datastore property name for the column.
|
|
159
|
+
*/
|
|
91
160
|
columnProperty(column) {
|
|
92
161
|
if (!this.recordSpec.columns.some((candidate) => candidate.name === column)) {
|
|
93
162
|
throw new Error(`Datastore record column "${column}" is not declared.`);
|
|
94
163
|
}
|
|
95
164
|
return column;
|
|
96
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* Returns a row limit with one sentinel beyond the local scan budget.
|
|
168
|
+
*
|
|
169
|
+
* @returns Provider row limit used to detect budget overflow.
|
|
170
|
+
*/
|
|
97
171
|
queryLimit() {
|
|
98
172
|
return this.maxClientSideScan + 1;
|
|
99
173
|
}
|
|
174
|
+
/**
|
|
175
|
+
* Returns the tenant namespace selected for this handle.
|
|
176
|
+
*
|
|
177
|
+
* @returns Datastore namespace, if tenant scoping requires one.
|
|
178
|
+
*/
|
|
100
179
|
namespace() {
|
|
101
180
|
return this.#namespace;
|
|
102
181
|
}
|
|
182
|
+
/**
|
|
183
|
+
* Converts a declared record column value to its provider representation.
|
|
184
|
+
*
|
|
185
|
+
* @param column Declared record column name.
|
|
186
|
+
* @param value Runtime column value.
|
|
187
|
+
* @returns Datastore-compatible column value.
|
|
188
|
+
*/
|
|
103
189
|
columnValue(column, value) {
|
|
104
190
|
const declared = this.recordSpec.columns.find((candidate) => candidate.name === column);
|
|
105
191
|
if (declared === undefined)
|
|
106
192
|
throw new Error(`Datastore record column "${column}" is not declared.`);
|
|
107
193
|
return ColumnMappings.value(this.#columnMapping, declared.type, value);
|
|
108
194
|
}
|
|
195
|
+
/**
|
|
196
|
+
* Captures ordered values and the physical key from the final page entity.
|
|
197
|
+
*
|
|
198
|
+
* @param entity Last raw Datastore entity in a page.
|
|
199
|
+
* @param order Provider ordering used for the page.
|
|
200
|
+
* @param keySymbol Client symbol for the physical entity key.
|
|
201
|
+
* @returns Stable keyset continuation for the next page.
|
|
202
|
+
*/
|
|
109
203
|
pageCursor(entity, order, keySymbol) {
|
|
110
204
|
const key = datastoreKey(entity, keySymbol);
|
|
111
205
|
const record = this.decode(entity);
|
|
@@ -121,6 +215,12 @@ class FlatEntityCodec {
|
|
|
121
215
|
throw new Error("Datastore provider page continuation is malformed.");
|
|
122
216
|
return { values, key };
|
|
123
217
|
}
|
|
218
|
+
/**
|
|
219
|
+
* Decodes a generated record from an entity's binary payload.
|
|
220
|
+
*
|
|
221
|
+
* @param entity Raw Datastore entity containing the payload.
|
|
222
|
+
* @returns Generated record without unknown fields.
|
|
223
|
+
*/
|
|
124
224
|
decode(entity) {
|
|
125
225
|
const payload = entity[payloadProperty];
|
|
126
226
|
if (!(payload instanceof Uint8Array)) {
|
|
@@ -168,6 +268,9 @@ function cursorJson(_key, value) {
|
|
|
168
268
|
}
|
|
169
269
|
/**
|
|
170
270
|
* Provides one Datastore-backed storage handle for a record family.
|
|
271
|
+
*
|
|
272
|
+
* @typeParam I Record identifier type used by this handle.
|
|
273
|
+
* @typeParam R Generated record message type stored by this handle.
|
|
171
274
|
*/
|
|
172
275
|
export class DatastoreRecordStorage extends RecordStorage {
|
|
173
276
|
client;
|
|
@@ -339,7 +442,7 @@ export class DatastoreRecordStorage extends RecordStorage {
|
|
|
339
442
|
queryCapabilities() {
|
|
340
443
|
return {
|
|
341
444
|
comparisons: ["equal", "greaterThan", "lessThan", "greaterOrEqual", "lessOrEqual"],
|
|
342
|
-
features: ["order", "
|
|
445
|
+
features: ["order", "limit"],
|
|
343
446
|
};
|
|
344
447
|
}
|
|
345
448
|
/**
|
|
@@ -348,12 +451,79 @@ export class DatastoreRecordStorage extends RecordStorage {
|
|
|
348
451
|
* @returns The candidate storage entries.
|
|
349
452
|
*/
|
|
350
453
|
async queryPlanRecordEntries(plan) {
|
|
351
|
-
|
|
454
|
+
const legal = DatastoreQueryPushdown.legal(plan);
|
|
455
|
+
const largeIds = plan.exhaustive && plan.predicate !== undefined
|
|
456
|
+
? DatastoreQueryPushdown.largeIdSet(plan.predicate, plan.order)
|
|
457
|
+
: undefined;
|
|
458
|
+
if (plan.predicate !== undefined && legal === undefined && largeIds === undefined)
|
|
352
459
|
throw new TypeError("Datastore normalized query has an illegal predicate or inequality ordering.");
|
|
353
460
|
for (const order of plan.order ?? [])
|
|
354
461
|
this.#codec.columnProperty(order.column);
|
|
355
|
-
|
|
356
|
-
|
|
462
|
+
if (plan.exhaustive) {
|
|
463
|
+
if (largeIds !== undefined)
|
|
464
|
+
return this.largeIdPlanEntries(largeIds);
|
|
465
|
+
return this.exhaustivePlanEntries(plan, true);
|
|
466
|
+
}
|
|
467
|
+
return this.boundedPlanEntries(plan);
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* Reads a mandatory large ID set through provider-legal key batches.
|
|
471
|
+
*
|
|
472
|
+
* @param ids Explicit IDs that every matching record must have.
|
|
473
|
+
* @returns Unique candidates for global predicate and ordering evaluation.
|
|
474
|
+
*/
|
|
475
|
+
async largeIdPlanEntries(ids) {
|
|
476
|
+
const unique = new Map();
|
|
477
|
+
for (const id of ids)
|
|
478
|
+
unique.set(this.#codec.idName(id), id);
|
|
479
|
+
const selected = [...unique.values()];
|
|
480
|
+
const entries = [];
|
|
481
|
+
for (let start = 0; start < selected.length; start += 30) {
|
|
482
|
+
const batch = selected.slice(start, start + 30);
|
|
483
|
+
entries.push(...(await this.exhaustivePlanEntries({ exhaustive: true, predicate: { kind: "ids", ids: batch } }, true)));
|
|
484
|
+
}
|
|
485
|
+
return entries;
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Reads every provider page before storage applies the global query semantics.
|
|
489
|
+
*
|
|
490
|
+
* @param plan Validated normalized query plan.
|
|
491
|
+
* @param pushdown Whether the provider can apply the predicate and ordering.
|
|
492
|
+
* @returns Every candidate entry in provider page order.
|
|
493
|
+
*/
|
|
494
|
+
async exhaustivePlanEntries(plan, pushdown) {
|
|
495
|
+
const entries = [];
|
|
496
|
+
let cursor;
|
|
497
|
+
const seen = new Set();
|
|
498
|
+
for (;;) {
|
|
499
|
+
const query = this.planQuery(plan, pushdown);
|
|
500
|
+
query.limit(128);
|
|
501
|
+
if (cursor !== undefined)
|
|
502
|
+
query.start(cursor);
|
|
503
|
+
const response = await this.provider(() => this.client.runQuery(query, wrappedReadOptions));
|
|
504
|
+
entries.push(...DatastoreResults.entities(response).map((entity) => ({
|
|
505
|
+
id: this.#codec.id(entity, this.client),
|
|
506
|
+
record: this.#codec.decode(entity),
|
|
507
|
+
})));
|
|
508
|
+
const info = DatastoreResults.queryInfo(response);
|
|
509
|
+
if (!info.more)
|
|
510
|
+
return entries;
|
|
511
|
+
const next = info.cursor;
|
|
512
|
+
const key = typeof next === "string" ? next : next.toString("base64");
|
|
513
|
+
if (seen.has(key))
|
|
514
|
+
throw new Error("Datastore query continuation did not advance.");
|
|
515
|
+
seen.add(key);
|
|
516
|
+
cursor = next;
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Reads one bounded provider result for public and Process Manager plans.
|
|
521
|
+
*
|
|
522
|
+
* @param plan Validated normalized query plan.
|
|
523
|
+
* @returns Candidate entries within the provider's configured bound.
|
|
524
|
+
*/
|
|
525
|
+
async boundedPlanEntries(plan) {
|
|
526
|
+
const query = this.planQuery(plan, true);
|
|
357
527
|
const providerLimit = Math.min(this.#codec.queryLimit(), plan.limit ?? Number.POSITIVE_INFINITY, (plan.candidateLimit ?? defaultQueryCandidateLimit) + 1);
|
|
358
528
|
query.limit(providerLimit);
|
|
359
529
|
const entities = DatastoreResults.entities(await this.provider(() => this.client.runQuery(query, wrappedReadOptions)));
|
|
@@ -365,6 +535,21 @@ export class DatastoreRecordStorage extends RecordStorage {
|
|
|
365
535
|
record: this.#codec.decode(entity),
|
|
366
536
|
}));
|
|
367
537
|
}
|
|
538
|
+
/**
|
|
539
|
+
* Builds one Datastore query with safe predicate and order pushdown.
|
|
540
|
+
*
|
|
541
|
+
* @param plan Normalized query plan to translate.
|
|
542
|
+
* @param pushdown Whether the plan is legal for provider filtering.
|
|
543
|
+
* @returns Query without a page bound or cursor.
|
|
544
|
+
*/
|
|
545
|
+
planQuery(plan, pushdown) {
|
|
546
|
+
const query = this.#codec.createQuery(this.client);
|
|
547
|
+
if (pushdown)
|
|
548
|
+
DatastoreQueryPushdown.plan(query, plan, (id) => this.#codec.key(this.client, id), (column) => this.#codec.columnProperty(column), (column, value) => this.#codec.columnValue(column, value));
|
|
549
|
+
if (plan.exhaustive && (!pushdown || (plan.order?.length ?? 0) === 0))
|
|
550
|
+
query.order("__key__");
|
|
551
|
+
return query;
|
|
552
|
+
}
|
|
368
553
|
/**
|
|
369
554
|
* Reads the record at one storage slot.
|
|
370
555
|
* @param id The storage slot identifier.
|
|
@@ -396,6 +581,14 @@ export class DatastoreRecordStorage extends RecordStorage {
|
|
|
396
581
|
}
|
|
397
582
|
throw new Error("Datastore compare-and-set retry limit was reached.");
|
|
398
583
|
}
|
|
584
|
+
/**
|
|
585
|
+
* Executes one transactional compare-and-set attempt against the stored payload.
|
|
586
|
+
*
|
|
587
|
+
* @param id Record slot selected for mutation.
|
|
588
|
+
* @param expected Required prior materialized record, or absence.
|
|
589
|
+
* @param next Replacement materialized record, or deletion.
|
|
590
|
+
* @returns True when the expected payload matched and commit completed.
|
|
591
|
+
*/
|
|
399
592
|
async attemptCompareAndSet(id, expected, next) {
|
|
400
593
|
const key = this.#codec.key(this.client, id);
|
|
401
594
|
const transaction = this.transaction();
|
|
@@ -457,6 +650,13 @@ export class DatastoreRecordStorage extends RecordStorage {
|
|
|
457
650
|
excludeFromIndexes: this.#codec.unindexedProperties(data),
|
|
458
651
|
}));
|
|
459
652
|
}
|
|
653
|
+
/**
|
|
654
|
+
* Executes a provider operation and hides provider error details from callers.
|
|
655
|
+
*
|
|
656
|
+
* @typeParam T Result produced by the provider operation.
|
|
657
|
+
* @param operation Provider request to execute.
|
|
658
|
+
* @returns Provider result after successful execution.
|
|
659
|
+
*/
|
|
460
660
|
async provider(operation) {
|
|
461
661
|
try {
|
|
462
662
|
return await operation();
|
|
@@ -480,24 +680,36 @@ export class DatastoreRecordStorage extends RecordStorage {
|
|
|
480
680
|
}
|
|
481
681
|
}
|
|
482
682
|
/**
|
|
483
|
-
*
|
|
683
|
+
* Compares decoded query values and persisted payload bytes.
|
|
484
684
|
*/
|
|
485
|
-
|
|
685
|
+
class RecordValueMethods {
|
|
486
686
|
// prettier-ignore
|
|
487
687
|
/**
|
|
488
688
|
* Compares canonical local values for equality.
|
|
689
|
+
*
|
|
690
|
+
* @param left First decoded query value.
|
|
691
|
+
* @param right Second decoded query value.
|
|
692
|
+
* @returns True when the query evaluator considers the values equal.
|
|
489
693
|
*/
|
|
490
694
|
equal(left, right) {
|
|
491
695
|
return StorageQueryValues.equal(left, right);
|
|
492
696
|
}
|
|
493
697
|
/**
|
|
494
698
|
* Compares canonical local values for ordering.
|
|
699
|
+
*
|
|
700
|
+
* @param left First decoded query value.
|
|
701
|
+
* @param right Second decoded query value.
|
|
702
|
+
* @returns Signed query sort comparison.
|
|
495
703
|
*/
|
|
496
704
|
compare(left, right) {
|
|
497
705
|
return StorageQueryValues.compare(left, right);
|
|
498
706
|
}
|
|
499
707
|
/**
|
|
500
708
|
* Compares two persisted payload byte arrays.
|
|
709
|
+
*
|
|
710
|
+
* @param left First persisted payload candidate.
|
|
711
|
+
* @param right Second persisted payload candidate.
|
|
712
|
+
* @returns True only when both are byte arrays with identical bytes.
|
|
501
713
|
*/
|
|
502
714
|
payloadEqual(left, right) {
|
|
503
715
|
return (left instanceof Uint8Array &&
|
|
@@ -505,7 +717,11 @@ const RecordValues = Object.freeze(new (class {
|
|
|
505
717
|
left.byteLength === right.byteLength &&
|
|
506
718
|
left.every((value, index) => value === right[index]));
|
|
507
719
|
}
|
|
508
|
-
}
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* Exposes comparisons for decoded query values and persisted payload bytes.
|
|
723
|
+
*/
|
|
724
|
+
const RecordValues = Object.freeze(new RecordValueMethods());
|
|
509
725
|
/**
|
|
510
726
|
* Raised when a query would exceed the adapter's configured finite client-side scan budget.
|
|
511
727
|
*/
|
|
@@ -530,6 +746,13 @@ class DatastoreQueryPushdownHelper {
|
|
|
530
746
|
// prettier-ignore
|
|
531
747
|
/**
|
|
532
748
|
* Translates the legacy record-query surface.
|
|
749
|
+
*
|
|
750
|
+
* @typeParam I Record identifier type used by the query.
|
|
751
|
+
* @param query Mutable Datastore query receiving filters and ordering.
|
|
752
|
+
* @param recordQuery Legacy record query to translate.
|
|
753
|
+
* @param keyFor Converts a record ID to a Datastore key.
|
|
754
|
+
* @param columnProperty Resolves a declared column to a property name.
|
|
755
|
+
* @param columnValue Converts a column value for the provider.
|
|
533
756
|
*/
|
|
534
757
|
translate(query, recordQuery, keyFor, columnProperty, columnValue) {
|
|
535
758
|
if (recordQuery.ids !== undefined && recordQuery.ids.length > 0) {
|
|
@@ -559,6 +782,13 @@ class DatastoreQueryPushdownHelper {
|
|
|
559
782
|
}
|
|
560
783
|
/**
|
|
561
784
|
* Applies the all-or-nothing legal portion of a normalized query plan.
|
|
785
|
+
*
|
|
786
|
+
* @typeParam I Record identifier type used by the plan.
|
|
787
|
+
* @param query Mutable Datastore query receiving safe clauses.
|
|
788
|
+
* @param plan Normalized plan whose predicate and ordering are inspected.
|
|
789
|
+
* @param keyFor Converts a record ID to a Datastore key.
|
|
790
|
+
* @param columnProperty Resolves a declared column to a property name.
|
|
791
|
+
* @param columnValue Converts a column value for the provider.
|
|
562
792
|
*/
|
|
563
793
|
plan(query, plan, keyFor, columnProperty, columnValue) {
|
|
564
794
|
const legal = this.legal(plan);
|
|
@@ -575,6 +805,10 @@ class DatastoreQueryPushdownHelper {
|
|
|
575
805
|
}
|
|
576
806
|
/**
|
|
577
807
|
* Returns the predicate only when every clause satisfies provider restrictions.
|
|
808
|
+
*
|
|
809
|
+
* @typeParam I Record identifier type in ID predicates.
|
|
810
|
+
* @param plan Normalized plan checked for supported predicate shape and ordering.
|
|
811
|
+
* @returns Predicate when Datastore can execute it completely.
|
|
578
812
|
*/
|
|
579
813
|
legal(plan) {
|
|
580
814
|
const predicate = plan.predicate;
|
|
@@ -604,8 +838,40 @@ class DatastoreQueryPushdownHelper {
|
|
|
604
838
|
}
|
|
605
839
|
return predicate;
|
|
606
840
|
}
|
|
841
|
+
/**
|
|
842
|
+
* Finds a mandatory ID set too wide for one Datastore IN filter.
|
|
843
|
+
*
|
|
844
|
+
* @typeParam I Identifier type in the predicate.
|
|
845
|
+
* @param predicate Predicate checked for a mandatory large ID set.
|
|
846
|
+
* @param order Ordering checked against any inequality comparison.
|
|
847
|
+
* @returns A required large ID set, or undefined for other shapes.
|
|
848
|
+
*/
|
|
849
|
+
largeIdSet(predicate, order) {
|
|
850
|
+
const leaves = predicate.kind === "all" ? predicate.predicates : [predicate];
|
|
851
|
+
if (leaves.some((leaf) => leaf.kind !== "ids" && leaf.kind !== "comparison"))
|
|
852
|
+
return undefined;
|
|
853
|
+
const large = leaves.find((leaf) => leaf.kind === "ids" && leaf.ids.length > 30);
|
|
854
|
+
if (large?.kind !== "ids")
|
|
855
|
+
return undefined;
|
|
856
|
+
const narrowed = leaves.map((leaf) => leaf.kind === "ids" && leaf.ids.length > 30 ? { ...leaf, ids: leaf.ids.slice(0, 1) } : leaf);
|
|
857
|
+
const representative = predicate.kind === "all"
|
|
858
|
+
? { kind: "all", predicates: narrowed }
|
|
859
|
+
: { kind: "ids", ids: large.ids.slice(0, 1) };
|
|
860
|
+
const legal = this.legal({
|
|
861
|
+
predicate: representative,
|
|
862
|
+
...(order === undefined ? {} : { order }),
|
|
863
|
+
});
|
|
864
|
+
return legal === undefined ? undefined : large.ids;
|
|
865
|
+
}
|
|
607
866
|
/**
|
|
608
867
|
* Adds one already-legal normalized predicate to the provider query.
|
|
868
|
+
*
|
|
869
|
+
* @typeParam I Record identifier type in ID predicates.
|
|
870
|
+
* @param query Mutable Datastore query receiving provider filters.
|
|
871
|
+
* @param predicate Normalized predicate approved for provider execution.
|
|
872
|
+
* @param keyFor Converts a record ID to a Datastore key.
|
|
873
|
+
* @param columnProperty Resolves a declared column to a property name.
|
|
874
|
+
* @param columnValue Converts a comparison value for the provider.
|
|
609
875
|
*/
|
|
610
876
|
predicate(query, predicate, keyFor, columnProperty, columnValue) {
|
|
611
877
|
if (predicate.kind === "all") {
|
|
@@ -644,13 +910,14 @@ class DatastoreQueryPushdownHelper {
|
|
|
644
910
|
*/
|
|
645
911
|
const DatastoreQueryPushdown = Object.freeze(new DatastoreQueryPushdownHelper());
|
|
646
912
|
/**
|
|
647
|
-
* Selects safe provider bounds for direct record queries.
|
|
913
|
+
* Selects safe provider bounds for direct legacy record queries.
|
|
648
914
|
*/
|
|
649
|
-
|
|
915
|
+
class DatastoreRecordQueryMethods {
|
|
650
916
|
// prettier-ignore
|
|
651
917
|
/**
|
|
652
918
|
* Returns a caller limit only when it cannot change local query semantics.
|
|
653
919
|
*
|
|
920
|
+
* @typeParam I Record identifier type in the legacy query.
|
|
654
921
|
* @param query Supplies the requested record query.
|
|
655
922
|
* @param scanLimit Supplies the configured materialization limit.
|
|
656
923
|
* @returns The finite provider row bound.
|
|
@@ -671,6 +938,7 @@ const DatastoreRecordQuery = Object.freeze(new (class {
|
|
|
671
938
|
/**
|
|
672
939
|
* Returns whether a legacy query is exactly an ascending identifier keyset page.
|
|
673
940
|
*
|
|
941
|
+
* @typeParam I Record identifier type in the legacy query.
|
|
674
942
|
* @param query The record query to classify.
|
|
675
943
|
* @returns `true` when Datastore can apply its limit and continuation completely.
|
|
676
944
|
*/
|
|
@@ -690,7 +958,11 @@ const DatastoreRecordQuery = Object.freeze(new (class {
|
|
|
690
958
|
continuationValue?.field === "id" &&
|
|
691
959
|
RecordValues.equal(continuationValue.value, continuation.id))));
|
|
692
960
|
}
|
|
693
|
-
}
|
|
961
|
+
}
|
|
962
|
+
/**
|
|
963
|
+
* Provides the bounded legacy record-query checks used by this adapter.
|
|
964
|
+
*/
|
|
965
|
+
const DatastoreRecordQuery = Object.freeze(new DatastoreRecordQueryMethods());
|
|
694
966
|
/**
|
|
695
967
|
* Applies record-query filtering, ordering, continuation, and paging locally.
|
|
696
968
|
*/
|
|
@@ -698,6 +970,12 @@ class LocalQueryResultsHelper {
|
|
|
698
970
|
// prettier-ignore
|
|
699
971
|
/**
|
|
700
972
|
* Applies one legacy record query to materialized candidate entries.
|
|
973
|
+
*
|
|
974
|
+
* @typeParam I Record identifier type in the candidates.
|
|
975
|
+
* @typeParam R Generated record message type in the candidates.
|
|
976
|
+
* @param entries Materialized candidates with declared column values.
|
|
977
|
+
* @param query Legacy filter, ordering, continuation, and paging request.
|
|
978
|
+
* @returns Selected record entries after local query evaluation.
|
|
701
979
|
*/
|
|
702
980
|
apply(entries, query) {
|
|
703
981
|
const matching = entries.filter((entry) => this.matches(entry, query));
|
|
@@ -713,6 +991,12 @@ class LocalQueryResultsHelper {
|
|
|
713
991
|
}
|
|
714
992
|
/**
|
|
715
993
|
* Tests whether a candidate satisfies the ID and equality filters.
|
|
994
|
+
*
|
|
995
|
+
* @typeParam I Record identifier type in the candidate.
|
|
996
|
+
* @typeParam R Generated record message type in the candidate.
|
|
997
|
+
* @param entry Candidate record with materialized columns.
|
|
998
|
+
* @param query Legacy ID and column filters.
|
|
999
|
+
* @returns True when the candidate satisfies every requested filter.
|
|
716
1000
|
*/
|
|
717
1001
|
matches(entry, query) {
|
|
718
1002
|
return ((query.ids === undefined ||
|
|
@@ -725,7 +1009,14 @@ class LocalQueryResultsHelper {
|
|
|
725
1009
|
}));
|
|
726
1010
|
}
|
|
727
1011
|
/**
|
|
728
|
-
*
|
|
1012
|
+
* Compares candidates by requested sort fields and canonical ID tie-breaker.
|
|
1013
|
+
*
|
|
1014
|
+
* @typeParam I Record identifier type in the candidates.
|
|
1015
|
+
* @typeParam R Generated record message type in the candidates.
|
|
1016
|
+
* @param left First candidate entry.
|
|
1017
|
+
* @param right Second candidate entry.
|
|
1018
|
+
* @param orders Requested field order in priority order.
|
|
1019
|
+
* @returns Signed sort comparison with ID as final tie-breaker.
|
|
729
1020
|
*/
|
|
730
1021
|
order(left, right, orders) {
|
|
731
1022
|
for (const order of orders) {
|
|
@@ -737,6 +1028,13 @@ class LocalQueryResultsHelper {
|
|
|
737
1028
|
}
|
|
738
1029
|
/**
|
|
739
1030
|
* Compares a candidate with a normalized continuation cursor.
|
|
1031
|
+
*
|
|
1032
|
+
* @typeParam I Record identifier type in the candidate and cursor.
|
|
1033
|
+
* @typeParam R Generated record message type in the candidate.
|
|
1034
|
+
* @param entry Candidate record with materialized columns.
|
|
1035
|
+
* @param orders Requested field order used by the cursor.
|
|
1036
|
+
* @param after Continuation cursor from the preceding page.
|
|
1037
|
+
* @returns Signed comparison after applying order directions and ID tie-breaker.
|
|
740
1038
|
*/
|
|
741
1039
|
continuation(entry, orders, after) {
|
|
742
1040
|
for (let index = 0; index < orders.length; index += 1) {
|
|
@@ -750,7 +1048,13 @@ class LocalQueryResultsHelper {
|
|
|
750
1048
|
return RecordValues.compare(entry.id, after.id);
|
|
751
1049
|
}
|
|
752
1050
|
/**
|
|
753
|
-
*
|
|
1051
|
+
* Reads an ID or indexed column value from a candidate.
|
|
1052
|
+
*
|
|
1053
|
+
* @typeParam I Record identifier type in the candidate.
|
|
1054
|
+
* @typeParam R Generated record message type in the candidate.
|
|
1055
|
+
* @param entry Candidate record with materialized columns.
|
|
1056
|
+
* @param field ID or declared column name.
|
|
1057
|
+
* @returns Candidate ID or stored column value.
|
|
754
1058
|
*/
|
|
755
1059
|
value(entry, field) {
|
|
756
1060
|
return field === "id" ? entry.id : entry.columns.get(field);
|
|
@@ -766,14 +1070,20 @@ const LocalQueryResults = Object.freeze(new LocalQueryResultsHelper());
|
|
|
766
1070
|
class DatastoreTransactionsHelper {
|
|
767
1071
|
// prettier-ignore
|
|
768
1072
|
/**
|
|
769
|
-
*
|
|
1073
|
+
* Replaces transaction errors with a stable public failure message.
|
|
1074
|
+
*
|
|
1075
|
+
* @param error Provider failure whose details must be hidden.
|
|
1076
|
+
* @returns New error without provider details.
|
|
770
1077
|
*/
|
|
771
1078
|
redact(error) {
|
|
772
1079
|
void error;
|
|
773
1080
|
return new Error("Datastore transaction failed.");
|
|
774
1081
|
}
|
|
775
1082
|
/**
|
|
776
|
-
*
|
|
1083
|
+
* Checks for a safe-to-retry Datastore transaction conflict.
|
|
1084
|
+
*
|
|
1085
|
+
* @param error Provider failure to classify.
|
|
1086
|
+
* @returns True for non-sensitive Datastore conflict code 10.
|
|
777
1087
|
*/
|
|
778
1088
|
retry(error) {
|
|
779
1089
|
return (error instanceof Error &&
|
|
@@ -782,6 +1092,9 @@ class DatastoreTransactionsHelper {
|
|
|
782
1092
|
}
|
|
783
1093
|
/**
|
|
784
1094
|
* Waits using exponential jitter before a bounded CAS retry.
|
|
1095
|
+
*
|
|
1096
|
+
* @param attempt Zero-based retry attempt used for backoff.
|
|
1097
|
+
* @returns Completes after the randomized delay.
|
|
785
1098
|
*/
|
|
786
1099
|
async wait(attempt) {
|
|
787
1100
|
const exponentialDelay = casRetryDelayMs * 2 ** attempt;
|
|
@@ -799,7 +1112,10 @@ const DatastoreTransactions = Object.freeze(new DatastoreTransactionsHelper());
|
|
|
799
1112
|
class DatastoreResultsHelper {
|
|
800
1113
|
// prettier-ignore
|
|
801
1114
|
/**
|
|
802
|
-
*
|
|
1115
|
+
* Reads the optional first entity from a Datastore read response.
|
|
1116
|
+
*
|
|
1117
|
+
* @param response Raw provider read response.
|
|
1118
|
+
* @returns First validated entity, if a row was returned.
|
|
803
1119
|
*/
|
|
804
1120
|
first(response) {
|
|
805
1121
|
if (!Array.isArray(response))
|
|
@@ -808,7 +1124,10 @@ class DatastoreResultsHelper {
|
|
|
808
1124
|
return entity === undefined ? undefined : this.entity(entity);
|
|
809
1125
|
}
|
|
810
1126
|
/**
|
|
811
|
-
*
|
|
1127
|
+
* Reads entity rows from a Datastore query response.
|
|
1128
|
+
*
|
|
1129
|
+
* @param response Raw provider query response.
|
|
1130
|
+
* @returns Validated entity objects in provider order.
|
|
812
1131
|
*/
|
|
813
1132
|
entities(response) {
|
|
814
1133
|
if (!Array.isArray(response) || !Array.isArray(response[0])) {
|
|
@@ -817,7 +1136,10 @@ class DatastoreResultsHelper {
|
|
|
817
1136
|
return response[0].map((value) => this.entity(value));
|
|
818
1137
|
}
|
|
819
1138
|
/**
|
|
820
|
-
*
|
|
1139
|
+
* Reads the provider continuation and the remaining-row indication.
|
|
1140
|
+
*
|
|
1141
|
+
* @param response Raw provider query response.
|
|
1142
|
+
* @returns Validated cursor and whether another page exists.
|
|
821
1143
|
*/
|
|
822
1144
|
queryInfo(response) {
|
|
823
1145
|
if (!Array.isArray(response) || typeof response[1] !== "object" || response[1] === null)
|
|
@@ -830,15 +1152,21 @@ class DatastoreResultsHelper {
|
|
|
830
1152
|
moreResults !== "MORE_RESULTS_AFTER_CURSOR")
|
|
831
1153
|
throw new Error("Datastore returned an invalid query response.");
|
|
832
1154
|
const more = moreResults === "MORE_RESULTS_AFTER_LIMIT" || moreResults === "MORE_RESULTS_AFTER_CURSOR";
|
|
833
|
-
if (more
|
|
834
|
-
|
|
1155
|
+
if (more) {
|
|
1156
|
+
if (!(cursor instanceof Buffer || typeof cursor === "string"))
|
|
1157
|
+
throw new Error("Datastore returned an invalid query response.");
|
|
1158
|
+
return { cursor, more: true };
|
|
1159
|
+
}
|
|
835
1160
|
return {
|
|
836
1161
|
cursor: cursor instanceof Buffer || typeof cursor === "string" ? cursor : undefined,
|
|
837
|
-
more,
|
|
1162
|
+
more: false,
|
|
838
1163
|
};
|
|
839
1164
|
}
|
|
840
1165
|
/**
|
|
841
1166
|
* Validates one provider entity object.
|
|
1167
|
+
*
|
|
1168
|
+
* @param value Candidate entity returned by Datastore.
|
|
1169
|
+
* @returns Record-like entity object.
|
|
842
1170
|
*/
|
|
843
1171
|
entity(value) {
|
|
844
1172
|
if (typeof value !== "object" || value === null) {
|