@procedures/mongo 0.14.1 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/README.md +8 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/storage/document-storage.d.ts +53 -7
- package/dist/storage/document-storage.d.ts.map +1 -1
- package/dist/storage/document-storage.js +132 -11
- package/dist/storage/document-storage.js.map +1 -1
- package/dist/storage/index.d.ts +2 -0
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +2 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/mongo-storage.d.ts +9 -4
- package/dist/storage/mongo-storage.d.ts.map +1 -1
- package/dist/storage/mongo-storage.js +15 -3
- package/dist/storage/mongo-storage.js.map +1 -1
- package/dist/storage/view-storage.d.ts +99 -0
- package/dist/storage/view-storage.d.ts.map +1 -0
- package/dist/storage/view-storage.js +189 -0
- package/dist/storage/view-storage.js.map +1 -0
- package/package.json +3 -3
- package/src/index.ts +3 -0
- package/src/storage/document-storage.ts +144 -14
- package/src/storage/index.ts +3 -0
- package/src/storage/mongo-storage.ts +15 -4
- package/src/storage/view-storage.ts +243 -0
|
@@ -38,21 +38,35 @@
|
|
|
38
38
|
*
|
|
39
39
|
* The aggregations: `aggregate()` runs a read-only pipeline on the items of the partition, with
|
|
40
40
|
* the guard of `scopePipeline()` and the limits of time and size.
|
|
41
|
+
*
|
|
42
|
+
* The preconditions (roadmap CLI-16): with the option `versionOf`, `set`, `delete` and `setBatch`
|
|
43
|
+
* take an expected version. On a replica set, the test reads the items in the transaction of the
|
|
44
|
+
* write, so the test and the write are one step. On a single server, each write of an item with a
|
|
45
|
+
* precondition has the `_seq` that the test read in its filter (a compare-and-set). Another writer
|
|
46
|
+
* between the test and the write makes the write fail with `VersionConflictError`. Then a batch of
|
|
47
|
+
* more than one item can keep a part of its writes: only a transaction makes a batch all or nothing.
|
|
41
48
|
*/
|
|
42
49
|
|
|
43
50
|
import { randomUUID } from "node:crypto";
|
|
44
51
|
import type { AnyBulkWriteOperation, ClientSession, Collection, Db, Document, Filter, MongoClient } from "mongodb";
|
|
45
52
|
import {
|
|
53
|
+
VersionConflictError,
|
|
46
54
|
applyPatch,
|
|
55
|
+
preconditionNotSupported,
|
|
47
56
|
queryEntries,
|
|
57
|
+
versionConflict,
|
|
48
58
|
type CollectionStorage,
|
|
59
|
+
type CurrentItem,
|
|
60
|
+
type StorageBatchWriteOptions,
|
|
49
61
|
type StorageChanges,
|
|
62
|
+
type StorageItemWriteOptions,
|
|
50
63
|
type StorageMetadata,
|
|
51
64
|
type StoragePatch,
|
|
52
65
|
type StorageQueryOptions,
|
|
53
66
|
type StorageQueryPolicy,
|
|
54
67
|
type StorageSelector,
|
|
55
68
|
type StorageWriteOptions,
|
|
69
|
+
type VersionConflict,
|
|
56
70
|
} from "@procedures/collections";
|
|
57
71
|
import { optionalMs, toMs, type Duration } from "@procedures/client";
|
|
58
72
|
import { ensureConnection } from "../connection.js";
|
|
@@ -73,7 +87,7 @@ const STORAGE_FIELDS: ReadonlySet<string> = new Set(["_id", KEY, PART, SEQ, ORD,
|
|
|
73
87
|
export const SEPARATOR = "\u001f";
|
|
74
88
|
|
|
75
89
|
/** The options of a `MongoDocumentStorage`. */
|
|
76
|
-
export interface MongoDocumentStorageOptions {
|
|
90
|
+
export interface MongoDocumentStorageOptions<T = unknown> {
|
|
77
91
|
/** The MongoDB collection. */
|
|
78
92
|
collection: string;
|
|
79
93
|
/** The database. The default is the database of the default connection, for each operation. */
|
|
@@ -101,6 +115,21 @@ export interface MongoDocumentStorageOptions {
|
|
|
101
115
|
* the watermark on a single server. `never`: each write uses the watermark.
|
|
102
116
|
*/
|
|
103
117
|
transactions?: "auto" | "never" | undefined;
|
|
118
|
+
/**
|
|
119
|
+
* The version of an item, for example `(item) => item.rev` (roadmap CLI-16). With it, `set`,
|
|
120
|
+
* `delete` and `setBatch` take an expected version: a write whose current item has another
|
|
121
|
+
* version changes nothing and rejects with `VersionConflictError`. Without it, a write with an
|
|
122
|
+
* expected version rejects with a `TypeError`. `versionOf(id)` of the storage is another thing:
|
|
123
|
+
* the sequence number of the last write.
|
|
124
|
+
*/
|
|
125
|
+
versionOf?: ((item: T) => string | undefined) | undefined;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** True for the error of a duplicate key: an insert of an upsert whose filter found no document. */
|
|
129
|
+
function isDuplicateKey(error: unknown): boolean {
|
|
130
|
+
// A bulk write error has the code of its first write error, and the list of the write errors
|
|
131
|
+
const { code, writeErrors } = (error ?? {}) as { code?: unknown; writeErrors?: unknown };
|
|
132
|
+
return code === 11000 || [writeErrors].flat().some((entry) => (entry as { code?: unknown } | null)?.code === 11000);
|
|
104
133
|
}
|
|
105
134
|
|
|
106
135
|
/** A stored document. */
|
|
@@ -257,7 +286,7 @@ interface Counter {
|
|
|
257
286
|
* ```
|
|
258
287
|
*/
|
|
259
288
|
export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
260
|
-
private readonly options: MongoDocumentStorageOptions
|
|
289
|
+
private readonly options: MongoDocumentStorageOptions<T>;
|
|
261
290
|
private readonly collectionName: string;
|
|
262
291
|
private readonly partition: string | undefined;
|
|
263
292
|
private readonly maxTombstones: number;
|
|
@@ -268,7 +297,7 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
268
297
|
private indexes: Promise<void> | undefined;
|
|
269
298
|
private closed = false;
|
|
270
299
|
|
|
271
|
-
constructor(options: MongoDocumentStorageOptions
|
|
300
|
+
constructor(options: MongoDocumentStorageOptions<T>, partitions?: Map<string, MongoDocumentStorage<T>>) {
|
|
272
301
|
if (options.partition !== undefined && (options.partition === "" || options.partition.includes(SEPARATOR))) {
|
|
273
302
|
throw new Error("A partition must be a text without the character U+001F");
|
|
274
303
|
}
|
|
@@ -673,11 +702,16 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
673
702
|
return enqueue(db, this.collectionName, () => task(db, items));
|
|
674
703
|
}
|
|
675
704
|
|
|
676
|
-
async set(id: string, value: T,
|
|
677
|
-
await this.setBatch([[id, value]]);
|
|
705
|
+
async set(id: string, value: T, options?: StorageItemWriteOptions): Promise<void> {
|
|
706
|
+
await this.setBatch([[id, value]], options?.expectedVersion === undefined ? undefined : { expectedVersions: [[id, options.expectedVersion]] });
|
|
678
707
|
}
|
|
679
708
|
|
|
680
|
-
|
|
709
|
+
/**
|
|
710
|
+
* This method writes several items. With `expectedVersions`, a conflict of one id rejects with
|
|
711
|
+
* `VersionConflictError` (roadmap CLI-16): the class comment tells when a batch is all or nothing.
|
|
712
|
+
*/
|
|
713
|
+
async setBatch(items: Array<[string, T]>, options?: StorageBatchWriteOptions): Promise<void> {
|
|
714
|
+
const expected = this.preconditions(options?.expectedVersions ?? []);
|
|
681
715
|
if (items.length === 0) {
|
|
682
716
|
this.checkOpen();
|
|
683
717
|
return;
|
|
@@ -690,14 +724,23 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
690
724
|
await this.write(async (db, collection) => {
|
|
691
725
|
const places = await this.placesOf(collection, items.map(([id]) => id));
|
|
692
726
|
await this.withNumbers(db, collection, items.length, async (first, session) => {
|
|
727
|
+
const guards = await this.checkVersions(collection, expected, session);
|
|
693
728
|
const placed = new Map(places);
|
|
694
729
|
const operations: AnyBulkWriteOperation<Stored>[] = items.map(([id, value], index) => {
|
|
695
730
|
const seq = first + index;
|
|
696
731
|
// A new id takes its place at its first write; a later write of the batch keeps it
|
|
697
732
|
if (!placed.has(id)) placed.set(id, seq);
|
|
698
|
-
|
|
733
|
+
// The guard of an id with a precondition: the write fails when another writer came first
|
|
734
|
+
const guard = guards.get(id);
|
|
735
|
+
guards.delete(id);
|
|
736
|
+
return { replaceOne: { filter: { ...this.writeFilter(id), ...guard }, replacement: this.encode(id, value, seq, placed.get(id)!), upsert: true } };
|
|
699
737
|
});
|
|
700
|
-
|
|
738
|
+
try {
|
|
739
|
+
await collection.bulkWrite(operations, { ordered: true, ...(session ? { session } : {}) });
|
|
740
|
+
} catch (error) {
|
|
741
|
+
if (expected.length > 0 && !session && isDuplicateKey(error)) throw await this.lateConflict(collection, expected);
|
|
742
|
+
throw error;
|
|
743
|
+
}
|
|
701
744
|
});
|
|
702
745
|
});
|
|
703
746
|
}
|
|
@@ -714,8 +757,23 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
714
757
|
});
|
|
715
758
|
}
|
|
716
759
|
|
|
717
|
-
|
|
718
|
-
|
|
760
|
+
/**
|
|
761
|
+
* This method deletes an item. With `expectedVersion`, a conflict rejects with
|
|
762
|
+
* `VersionConflictError` and deletes nothing (roadmap CLI-16).
|
|
763
|
+
*/
|
|
764
|
+
async delete(id: string, options?: StorageItemWriteOptions): Promise<boolean> {
|
|
765
|
+
if (options?.expectedVersion === undefined) return (await this.deleteBatch([id], options)) > 0;
|
|
766
|
+
const expected = this.preconditions([[id, options.expectedVersion]]);
|
|
767
|
+
return this.write(async (db, collection) => {
|
|
768
|
+
const live = await collection.findOne(this.liveFilter({ _id: this.storedId(id) }), { projection: { _id: 1 } });
|
|
769
|
+
if (!live) {
|
|
770
|
+
// No item to delete: the precondition holds only for the expected version null
|
|
771
|
+
await this.checkVersions(collection, expected);
|
|
772
|
+
return false;
|
|
773
|
+
}
|
|
774
|
+
await this.tombstone(db, collection, [id], expected);
|
|
775
|
+
return true;
|
|
776
|
+
});
|
|
719
777
|
}
|
|
720
778
|
|
|
721
779
|
async deleteBatch(ids: string[], _options?: StorageWriteOptions): Promise<number> {
|
|
@@ -748,19 +806,91 @@ export class MongoDocumentStorage<T> implements CollectionStorage<T> {
|
|
|
748
806
|
});
|
|
749
807
|
}
|
|
750
808
|
|
|
751
|
-
/**
|
|
752
|
-
|
|
809
|
+
/**
|
|
810
|
+
* This method replaces the items of the ids with tombstones, with new numbers, in the queue of the
|
|
811
|
+
* collection. With preconditions, it tests them first (roadmap CLI-16).
|
|
812
|
+
*/
|
|
813
|
+
private async tombstone(db: Db, collection: Collection<Stored>, ids: string[], expected: ReadonlyArray<readonly [string, string | null]> = []): Promise<void> {
|
|
753
814
|
await this.withNumbers(db, collection, ids.length, async (first, session) => {
|
|
815
|
+
const guards = await this.checkVersions(collection, expected, session);
|
|
754
816
|
const operations: AnyBulkWriteOperation<Stored>[] = ids.map((id, index) => {
|
|
755
817
|
const tomb: Stored = { _id: this.storedId(id), [KEY]: id, [SEQ]: first + index, [TOMB]: true };
|
|
756
818
|
if (this.partition !== undefined) tomb[PART] = this.partition;
|
|
757
|
-
return { replaceOne: { filter: this.writeFilter(id), replacement: tomb } };
|
|
819
|
+
return { replaceOne: { filter: { ...this.writeFilter(id), ...guards.get(id) }, replacement: tomb } };
|
|
758
820
|
});
|
|
759
|
-
await collection.bulkWrite(operations, { ordered: true, ...(session ? { session } : {}) });
|
|
821
|
+
const result = await collection.bulkWrite(operations, { ordered: true, ...(session ? { session } : {}) });
|
|
822
|
+
// A guard that found no document: another writer came between the test and the write
|
|
823
|
+
if (expected.length > 0 && !session && result.matchedCount < operations.length) throw await this.lateConflict(collection, expected);
|
|
760
824
|
});
|
|
761
825
|
await this.pruneTombstones(db, collection);
|
|
762
826
|
}
|
|
763
827
|
|
|
828
|
+
/**
|
|
829
|
+
* The preconditions of a write (roadmap CLI-16), after a check of their ids. A storage without
|
|
830
|
+
* `versionOf` throws a `TypeError` before the write starts.
|
|
831
|
+
*/
|
|
832
|
+
private preconditions(expected: ReadonlyArray<readonly [string, string | null]>): ReadonlyArray<readonly [string, string | null]> {
|
|
833
|
+
if (expected.length === 0) return expected;
|
|
834
|
+
if (!this.options.versionOf) throw preconditionNotSupported(`The MongoDocumentStorage of the collection ${this.collectionName}`);
|
|
835
|
+
for (const [id] of expected) this.storedId(id);
|
|
836
|
+
return expected;
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/** The current state of each id of the preconditions: the item, or no item for a tombstone or a missing document. */
|
|
840
|
+
private async currentDocs(collection: Collection<Stored>, ids: readonly string[], session?: ClientSession): Promise<Map<string, Document>> {
|
|
841
|
+
const docs = await collection
|
|
842
|
+
.find({ ...this.partitionFilter(), _id: { $in: [...new Set(ids)].map((id) => this.storedId(id)) } } as Filter<Stored>, session ? { session } : {})
|
|
843
|
+
.toArray();
|
|
844
|
+
return new Map(docs.map((doc) => [String(doc[KEY]), doc]));
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
/** The current item of a stored document: no item for a tombstone or a missing document. */
|
|
848
|
+
private currentOf(doc: Document | undefined): CurrentItem<T> {
|
|
849
|
+
return doc && doc[TOMB] !== true ? { found: true, value: this.decode(doc)[1] } : { found: false };
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/**
|
|
853
|
+
* This method tests the expected versions (roadmap CLI-16). A conflict throws
|
|
854
|
+
* `VersionConflictError`. It gives the guard of each id for a write without a transaction: the
|
|
855
|
+
* `_seq` that the test read, or no document. In a transaction, the guards are empty.
|
|
856
|
+
*/
|
|
857
|
+
private async checkVersions(collection: Collection<Stored>, expected: ReadonlyArray<readonly [string, string | null]>, session?: ClientSession): Promise<Map<string, Document>> {
|
|
858
|
+
const guards = new Map<string, Document>();
|
|
859
|
+
if (expected.length === 0 || !this.options.versionOf) return guards;
|
|
860
|
+
const versionOf = this.options.versionOf;
|
|
861
|
+
const docs = await this.currentDocs(collection, expected.map(([id]) => id), session);
|
|
862
|
+
const conflicts: VersionConflict[] = [];
|
|
863
|
+
for (const [id, version] of expected) {
|
|
864
|
+
const conflict = versionConflict(id, version, this.currentOf(docs.get(id)), versionOf);
|
|
865
|
+
if (conflict) conflicts.push(conflict);
|
|
866
|
+
}
|
|
867
|
+
if (conflicts.length > 0) throw new VersionConflictError(conflicts);
|
|
868
|
+
if (session) return guards;
|
|
869
|
+
for (const [id] of expected) {
|
|
870
|
+
const doc = docs.get(id);
|
|
871
|
+
guards.set(id, doc ? { [SEQ]: doc[SEQ] } : { [SEQ]: { $exists: false } });
|
|
872
|
+
}
|
|
873
|
+
return guards;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* The error of a guard that failed: another writer changed an id between the test and the write.
|
|
878
|
+
* It has the current item of each id of the preconditions.
|
|
879
|
+
*/
|
|
880
|
+
private async lateConflict(collection: Collection<Stored>, expected: ReadonlyArray<readonly [string, string | null]>): Promise<VersionConflictError> {
|
|
881
|
+
const versionOf = this.options.versionOf!;
|
|
882
|
+
const docs = await this.currentDocs(collection, expected.map(([id]) => id));
|
|
883
|
+
const conflicts = expected.map(([id, version]): VersionConflict => {
|
|
884
|
+
const current = this.currentOf(docs.get(id));
|
|
885
|
+
const conflict = versionConflict(id, version, current, versionOf);
|
|
886
|
+
if (conflict) return conflict;
|
|
887
|
+
// The current item has the expected version again, after a write of another writer
|
|
888
|
+
const now = current.found ? versionOf(current.value) : null;
|
|
889
|
+
return current.found ? { id, expected: version, found: true, value: current.value, ...(now !== undefined && { version: now }) } : { id, expected: version, found: false, version: null };
|
|
890
|
+
});
|
|
891
|
+
return new VersionConflictError(conflicts, "Another writer changed an item between the test of its version and the write");
|
|
892
|
+
}
|
|
893
|
+
|
|
764
894
|
// ===========================================================================
|
|
765
895
|
// The life of the storage
|
|
766
896
|
// ===========================================================================
|
package/src/storage/index.ts
CHANGED
|
@@ -20,3 +20,6 @@ export type { PipelineLimits, PipelineScope } from "./pipeline-scope.js";
|
|
|
20
20
|
// Materialized views driven by change streams (Graph's split plan, P4)
|
|
21
21
|
export { ViewMaterializer, identityPartition, partitionTargetId, viewFilter, VIEW_GENERATION_FIELD, VIEW_HASH_FIELD } from "./materializer.js";
|
|
22
22
|
export type { MaterializedView, MaterializedViewStatus, ViewMaterializerOptions, ViewPartitionOf, ViewPipelineScope, ViewTargetId } from "./materializer.js";
|
|
23
|
+
// A read-only storage of a view for the collection procedures: one generation for each read (roadmap CLI-38)
|
|
24
|
+
export { MongoViewStorage } from "./view-storage.js";
|
|
25
|
+
export type { MongoViewStorageOptions } from "./view-storage.js";
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { Collection, Document, Filter } from "mongodb";
|
|
9
|
-
import type
|
|
9
|
+
import { preconditionNotSupported, type CollectionStorage, type StorageBatchWriteOptions, type StorageItemWriteOptions, type StorageMetadata } from "@procedures/collections";
|
|
10
10
|
import { ensureConnection } from "../connection.js";
|
|
11
11
|
|
|
12
12
|
/**
|
|
@@ -85,7 +85,8 @@ export class MongoStorage<T> implements CollectionStorage<T> {
|
|
|
85
85
|
return (await this.getCollection()).countDocuments({});
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
-
async set(id: string, value: T): Promise<void> {
|
|
88
|
+
async set(id: string, value: T, options?: StorageItemWriteOptions): Promise<void> {
|
|
89
|
+
this.refusePrecondition(options?.expectedVersion !== undefined);
|
|
89
90
|
await (await this.getCollection()).updateOne(
|
|
90
91
|
{ _id: id } as Filter<MongoDoc<T>>,
|
|
91
92
|
{ $set: { _id: id, data: value } as unknown as Document },
|
|
@@ -93,7 +94,8 @@ export class MongoStorage<T> implements CollectionStorage<T> {
|
|
|
93
94
|
);
|
|
94
95
|
}
|
|
95
96
|
|
|
96
|
-
async delete(id: string): Promise<boolean> {
|
|
97
|
+
async delete(id: string, options?: StorageItemWriteOptions): Promise<boolean> {
|
|
98
|
+
this.refusePrecondition(options?.expectedVersion !== undefined);
|
|
97
99
|
const result = await (await this.getCollection()).deleteOne({ _id: id } as Filter<MongoDoc<T>>);
|
|
98
100
|
return result.deletedCount > 0;
|
|
99
101
|
}
|
|
@@ -102,8 +104,9 @@ export class MongoStorage<T> implements CollectionStorage<T> {
|
|
|
102
104
|
await (await this.getCollection()).deleteMany({});
|
|
103
105
|
}
|
|
104
106
|
|
|
105
|
-
async setBatch(items: Array<[string, T]
|
|
107
|
+
async setBatch(items: Array<[string, T]>, options?: StorageBatchWriteOptions): Promise<void> {
|
|
106
108
|
this.checkOpen();
|
|
109
|
+
this.refusePrecondition((options?.expectedVersions?.length ?? 0) > 0);
|
|
107
110
|
if (items.length === 0) return;
|
|
108
111
|
|
|
109
112
|
const operations = items.map(([id, value]) => ({
|
|
@@ -151,6 +154,14 @@ export class MongoStorage<T> implements CollectionStorage<T> {
|
|
|
151
154
|
this.closed = true;
|
|
152
155
|
}
|
|
153
156
|
|
|
157
|
+
/**
|
|
158
|
+
* This method refuses a write with an expected version (roadmap CLI-16). The `{ _id, data }`
|
|
159
|
+
* documents have no version: `MongoDocumentStorage` with the option `versionOf` tests it.
|
|
160
|
+
*/
|
|
161
|
+
private refusePrecondition(has: boolean): void {
|
|
162
|
+
if (has) throw preconditionNotSupported(`The MongoStorage of the collection ${this.collectionName}`, "use MongoDocumentStorage with the option versionOf");
|
|
163
|
+
}
|
|
164
|
+
|
|
154
165
|
/** This method throws after `close()`, so the promise of the operation rejects. */
|
|
155
166
|
private checkOpen(): void {
|
|
156
167
|
if (this.closed) throw new Error(`The MongoStorage of the collection ${this.collectionName} is closed`);
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `MongoViewStorage`: the rows of one materialized view as a read-only `CollectionStorage`
|
|
3
|
+
* (roadmap CLI-38). The collection procedures can then serve a view, and each reader of them gets
|
|
4
|
+
* one whole generation.
|
|
5
|
+
*
|
|
6
|
+
* A `ViewMaterializer` writes a big rebuild as a new generation in groups. Then it changes the
|
|
7
|
+
* generation in the status of the view. During the rebuild, the target has the rows of two
|
|
8
|
+
* generations, and a plain query reads both. This storage reads the status of the view one time
|
|
9
|
+
* for each operation, and reads only the rows of that generation (`viewFilter`). Thus `get`,
|
|
10
|
+
* `getAll`, `find`, `changes` and the other reads of the procedures do not mix two generations.
|
|
11
|
+
*
|
|
12
|
+
* - The key of a row is the `_id` of the row in the result of the pipeline. It must be a text.
|
|
13
|
+
* - The value of a row is the row of the result: its `_id` and its fields, without the fields of
|
|
14
|
+
* the materializer (the partition, `_viewGen` and `_viewHash`).
|
|
15
|
+
* - The storage has no change log. The procedure `changes` thus gives all the rows of the current
|
|
16
|
+
* generation.
|
|
17
|
+
* - A read that starts before a change of generation ends on the previous generation. The
|
|
18
|
+
* materializer keeps its rows for `generationGrace`, so a read must end in that time.
|
|
19
|
+
* - Before the first pass of the view, and without the view document, the storage has no row.
|
|
20
|
+
* - Only the materializer writes a target. Each write of this storage rejects with `FORBIDDEN`.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { Collection, Db, Document, ObjectId } from "mongodb";
|
|
24
|
+
import type { CollectionStorage, StorageMetadata } from "@procedures/collections";
|
|
25
|
+
import { ensureConnection } from "../connection.js";
|
|
26
|
+
import { partitionTargetId, viewFilter, VIEW_GENERATION_FIELD, VIEW_HASH_FIELD, type MaterializedView, type MaterializedViewStatus, type ViewTargetId } from "./materializer.js";
|
|
27
|
+
|
|
28
|
+
/** The options of a `MongoViewStorage`. Give the same names as to the `ViewMaterializer` of the view. */
|
|
29
|
+
export interface MongoViewStorageOptions {
|
|
30
|
+
/** The `_id` of the document of the view. The document names the partition and the target. */
|
|
31
|
+
view: string;
|
|
32
|
+
/** The database of the views and of the target. The default is the database of the default connection, for each operation. */
|
|
33
|
+
db?: (() => Db | Promise<Db>) | undefined;
|
|
34
|
+
/** The collection of the view documents (default `_aggregations`). */
|
|
35
|
+
views?: string | undefined;
|
|
36
|
+
/** The field of a document that names its partition (default `_part`). */
|
|
37
|
+
partitionField?: string | undefined;
|
|
38
|
+
/** The field of a view document that keeps its status (default `_materializerStatus`). */
|
|
39
|
+
statusField?: string | undefined;
|
|
40
|
+
/**
|
|
41
|
+
* The stored `_id` of a row from its key, the partition and the generation (default
|
|
42
|
+
* `partitionTargetId`). It is the option `targetId` of the materializer. `get`, `has` and
|
|
43
|
+
* `getBatch` find a row with it.
|
|
44
|
+
*/
|
|
45
|
+
targetId?: ViewTargetId | undefined;
|
|
46
|
+
/**
|
|
47
|
+
* The `_id` of the row in the result, from its stored `_id`. It reverses `targetId`. The default
|
|
48
|
+
* reads the field `id` of the form of `partitionTargetId`.
|
|
49
|
+
*/
|
|
50
|
+
rowIdOf?: ((storedId: unknown) => unknown) | undefined;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The current generation of a view: its target, and the filter of its rows. */
|
|
54
|
+
interface Generation {
|
|
55
|
+
readonly target: Collection<Document>;
|
|
56
|
+
readonly filter: Document;
|
|
57
|
+
readonly partition: string;
|
|
58
|
+
readonly generation: ObjectId;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The `_id` of a row in the result, from the stored `_id` of `partitionTargetId`. */
|
|
62
|
+
function defaultRowId(storedId: unknown): unknown {
|
|
63
|
+
return typeof storedId === "object" && storedId !== null && "id" in storedId ? (storedId as { id: unknown }).id : storedId;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* A read-only `CollectionStorage` over the target of one materialized view (roadmap CLI-38). Each
|
|
68
|
+
* read gives the rows of the current generation of the view only.
|
|
69
|
+
*
|
|
70
|
+
* @example
|
|
71
|
+
* ```typescript
|
|
72
|
+
* const server = new ProcedureServer({ collections: { totals: new MongoViewStorage({ view: "totals", db: () => db }) } });
|
|
73
|
+
* server.registerCollectionProcedures("totals");
|
|
74
|
+
* ```
|
|
75
|
+
*/
|
|
76
|
+
export class MongoViewStorage<T = Document> implements CollectionStorage<T> {
|
|
77
|
+
private readonly options: MongoViewStorageOptions;
|
|
78
|
+
private readonly viewsName: string;
|
|
79
|
+
private readonly partitionField: string;
|
|
80
|
+
private readonly statusField: string;
|
|
81
|
+
private readonly targetId: ViewTargetId;
|
|
82
|
+
private readonly rowIdOf: (storedId: unknown) => unknown;
|
|
83
|
+
private closed = false;
|
|
84
|
+
|
|
85
|
+
constructor(options: MongoViewStorageOptions) {
|
|
86
|
+
if (typeof options.view !== "string" || options.view === "") throw new Error("A MongoViewStorage needs the _id of its view");
|
|
87
|
+
this.options = options;
|
|
88
|
+
this.viewsName = options.views ?? "_aggregations";
|
|
89
|
+
this.partitionField = options.partitionField ?? "_part";
|
|
90
|
+
this.statusField = options.statusField ?? "_materializerStatus";
|
|
91
|
+
this.targetId = options.targetId ?? partitionTargetId;
|
|
92
|
+
this.rowIdOf = options.rowIdOf ?? defaultRowId;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ===========================================================================
|
|
96
|
+
// The generation of a read
|
|
97
|
+
// ===========================================================================
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The current generation of the view: one read of the view document. Each read of the storage
|
|
101
|
+
* uses one generation, so it gives the rows of one generation only. The result is undefined
|
|
102
|
+
* before the first pass of the view, or without the view document.
|
|
103
|
+
*/
|
|
104
|
+
private async current(): Promise<Generation | undefined> {
|
|
105
|
+
this.checkOpen();
|
|
106
|
+
const db = this.options.db ? await this.options.db() : (await ensureConnection()).getDb();
|
|
107
|
+
const view = (await db.collection(this.viewsName).findOne({ _id: this.options.view } as Document)) as unknown as MaterializedView | null;
|
|
108
|
+
if (!view || typeof view.targetCollection !== "string") return undefined;
|
|
109
|
+
const partition = view[this.partitionField];
|
|
110
|
+
const generation = (view[this.statusField] as Partial<MaterializedViewStatus> | undefined)?.generation;
|
|
111
|
+
if (typeof partition !== "string" || !generation) return undefined;
|
|
112
|
+
return {
|
|
113
|
+
target: db.collection(view.targetCollection),
|
|
114
|
+
filter: viewFilter(view, { partitionField: this.partitionField, statusField: this.statusField }),
|
|
115
|
+
partition,
|
|
116
|
+
generation,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** The stored `_id` of a key in a generation. */
|
|
121
|
+
private storedId(id: string, generation: Generation): unknown {
|
|
122
|
+
return this.targetId(id, generation.partition, generation.generation);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** The key and the value of a stored row. A row whose `_id` is not a text cannot have a key. */
|
|
126
|
+
private decode(doc: Document): [string, T] {
|
|
127
|
+
const { _id: storedId, [this.partitionField]: _partition, [VIEW_GENERATION_FIELD]: _generation, [VIEW_HASH_FIELD]: _hash, ...content } = doc;
|
|
128
|
+
const id = this.rowIdOf(storedId);
|
|
129
|
+
if (typeof id !== "string") {
|
|
130
|
+
throw new Error(`A row of the view ${this.options.view} has an _id that is not a text: give each row of its result a text _id`);
|
|
131
|
+
}
|
|
132
|
+
const value: Record<string, unknown> = { _id: id, ...content };
|
|
133
|
+
return [id, value as T];
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The rows of the current generation, in the order of their stored `_id`. */
|
|
137
|
+
private async entriesOf(extra?: Document): Promise<Array<[string, T]>> {
|
|
138
|
+
const generation = await this.current();
|
|
139
|
+
if (!generation) return [];
|
|
140
|
+
const docs = await generation.target
|
|
141
|
+
.find({ ...generation.filter, ...extra } as Document)
|
|
142
|
+
.sort({ _id: 1 })
|
|
143
|
+
.toArray();
|
|
144
|
+
return docs.map((doc) => this.decode(doc));
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// ===========================================================================
|
|
148
|
+
// Reads
|
|
149
|
+
// ===========================================================================
|
|
150
|
+
|
|
151
|
+
async get(id: string): Promise<T | undefined> {
|
|
152
|
+
const generation = await this.current();
|
|
153
|
+
if (!generation) return undefined;
|
|
154
|
+
const doc = await generation.target.findOne({ ...generation.filter, _id: this.storedId(id, generation) } as Document);
|
|
155
|
+
return doc ? this.decode(doc)[1] : undefined;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async getAll(): Promise<T[]> {
|
|
159
|
+
return (await this.entriesOf()).map(([, value]) => value);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async getEntries(): Promise<Array<[string, T]>> {
|
|
163
|
+
return this.entriesOf();
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async getIds(): Promise<string[]> {
|
|
167
|
+
return (await this.entriesOf()).map(([id]) => id);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async find(predicate: (item: T) => boolean): Promise<T[]> {
|
|
171
|
+
return (await this.getAll()).filter(predicate);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async has(id: string): Promise<boolean> {
|
|
175
|
+
const generation = await this.current();
|
|
176
|
+
if (!generation) return false;
|
|
177
|
+
return (await generation.target.countDocuments({ ...generation.filter, _id: this.storedId(id, generation) } as Document, { limit: 1 })) > 0;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async size(): Promise<number> {
|
|
181
|
+
const generation = await this.current();
|
|
182
|
+
if (!generation) return 0;
|
|
183
|
+
return generation.target.countDocuments(generation.filter);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async getBatch(ids: string[]): Promise<Map<string, T>> {
|
|
187
|
+
const result = new Map<string, T>();
|
|
188
|
+
const generation = await this.current();
|
|
189
|
+
if (!generation || ids.length === 0) return result;
|
|
190
|
+
const byId = new Map(await this.entriesOf({ _id: { $in: ids.map((id) => this.storedId(id, generation)) } }));
|
|
191
|
+
// The order of the ids of the call
|
|
192
|
+
for (const id of ids) if (byId.has(id)) result.set(id, byId.get(id)!);
|
|
193
|
+
return result;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// ===========================================================================
|
|
197
|
+
// Writes: only the materializer writes a target
|
|
198
|
+
// ===========================================================================
|
|
199
|
+
|
|
200
|
+
private readOnly(): Promise<never> {
|
|
201
|
+
this.checkOpen();
|
|
202
|
+
return Promise.reject(
|
|
203
|
+
Object.assign(new Error(`The view ${this.options.view} is read-only: only its materializer writes its target`), { code: "FORBIDDEN", retryable: false }),
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async set(_id: string, _value: T): Promise<void> {
|
|
208
|
+
return this.readOnly();
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
async setBatch(_items: Array<[string, T]>): Promise<void> {
|
|
212
|
+
return this.readOnly();
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async delete(_id: string): Promise<boolean> {
|
|
216
|
+
return this.readOnly();
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
async deleteBatch(_ids: string[]): Promise<number> {
|
|
220
|
+
return this.readOnly();
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async clear(): Promise<void> {
|
|
224
|
+
return this.readOnly();
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// ===========================================================================
|
|
228
|
+
// The life of the storage
|
|
229
|
+
// ===========================================================================
|
|
230
|
+
|
|
231
|
+
/** This method closes the storage. After it, each operation rejects, and a second `close()` does nothing. */
|
|
232
|
+
async close(): Promise<void> {
|
|
233
|
+
this.closed = true;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
private checkOpen(): void {
|
|
237
|
+
if (this.closed) throw new Error(`The MongoViewStorage of the view ${this.options.view} is closed`);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async getMetadata(): Promise<StorageMetadata> {
|
|
241
|
+
return { type: "custom", size: await this.size(), stats: { view: this.options.view } };
|
|
242
|
+
}
|
|
243
|
+
}
|