@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/storage-denokv/rx-storage-instance-denokv.ts

@@ -0,0 +1,426 @@
+
+import {
+    Subject
+} from 'rxjs';
+import type {
+    RxStorageInstance,
+    RxStorageChangeEvent,
+    RxDocumentData,
+    BulkWriteRow,
+    RxStorageBulkWriteResponse,
+    RxStorageQueryResult,
+    RxJsonSchema,
+    RxStorageInstanceCreationParams,
+    EventBulk,
+    StringKeys,
+    RxStorageDefaultCheckpoint,
+    RxStorageCountResult,
+    PreparedQuery
+} from '../../types/index.d.ts';
+import { getPrimaryFieldOfPrimaryKey } from '../../rx-schema-helper.ts';
+import { addRxStorageMultiInstanceSupport } from '../../rx-storage-multiinstance.ts';
+import type { DenoKVIndexMeta, DenoKVSettings, DenoKVStorageInternals } from './denokv-types.ts';
+import { RxStorageDenoKV } from './index.ts';
+import { CLEANUP_INDEX, DENOKV_DOCUMENT_ROOT_PATH, RX_STORAGE_NAME_DENOKV, commitWithRetry, getDenoGlobal, getDenoKVIndexName } from "./denokv-helper.ts";
+import { getIndexableStringMonad, getStartIndexStringFromLowerBound } from "../../custom-index.ts";
+import { batchArray, toArray } from "../utils/utils-array.ts";
+import { ensureNotFalsy } from "../utils/utils-other.ts";
+import { categorizeBulkWriteRows } from "../../rx-storage-helper.ts";
+import { now } from "../utils/utils-time.ts";
+import { promiseWait } from "../utils/utils-promise.ts";
+import { queryDenoKV } from "./denokv-query.ts";
+import { INDEX_MAX } from "../../query-planner.ts";
+import { flatClone } from "../utils/utils-object.ts";
+import { PROMISE_RESOLVE_VOID } from 'rxdb-old';
+
+
+
+export class RxStorageInstanceDenoKV<RxDocType> implements RxStorageInstance<
+    RxDocType,
+    DenoKVStorageInternals<RxDocType>,
+    DenoKVSettings,
+    RxStorageDefaultCheckpoint
+> {
+    public readonly primaryPath: StringKeys<RxDocumentData<RxDocType>>;
+    private changes$: Subject<EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, RxStorageDefaultCheckpoint>> = new Subject();
+    public closed?: Promise<void>;
+    public readonly kvPromise: Promise<any>;
+    public writeQueue: Promise<any> = PROMISE_RESOLVE_VOID;
+
+    constructor(
+        public readonly storage: RxStorageDenoKV,
+        public readonly databaseName: string,
+        public readonly collectionName: string,
+        public readonly schema: Readonly<RxJsonSchema<RxDocumentData<RxDocType>>>,
+        public readonly internals: DenoKVStorageInternals<RxDocType>,
+        public readonly options: Readonly<DenoKVSettings>,
+        public readonly settings: DenoKVSettings,
+        public readonly keySpace = ['rxdb', databaseName, collectionName, schema.version].join('|'),
+        public readonly kvOptions = { consistency: settings.consistencyLevel }
+    ) {
+        this.primaryPath = getPrimaryFieldOfPrimaryKey(this.schema.primaryKey);
+        this.kvPromise = getDenoGlobal().openKv(settings.openKvPath).then(async (kv: any) => {
+            // insert writeBlockKey
+            await kv.set([this.keySpace], 1);
+            return kv;
+        });
+    }
+
+    /**
+     * DenoKV has no transactions
+     * so we have to ensure that there is no write in between our queries
+     * which would confuse RxDB and return wrong query results.
+     */
+    async retryUntilNoWriteInBetween<T>(
+        fn: () => Promise<T>
+    ): Promise<T> {
+        const kv = await this.kvPromise;
+        let retryCount = 0;
+        while (true) {
+            const writeBlockKeyBefore = await kv.get([this.keySpace], this.kvOptions);
+            const writeBlockValueBefore = writeBlockKeyBefore ? writeBlockKeyBefore.value : -1;
+            const result = await fn();
+            const writeBlockKeyAfter = await kv.get([this.keySpace], this.kvOptions);
+            const writeBlockValueAfter = writeBlockKeyAfter ? writeBlockKeyAfter.value : -1;
+
+            if (writeBlockValueBefore === writeBlockValueAfter) {
+                return result;
+            }
+            retryCount++;
+            await promiseWait(retryCount * 5);
+        }
+    }
+
+    async bulkWrite(documentWrites: BulkWriteRow<RxDocType>[], context: string): Promise<RxStorageBulkWriteResponse<RxDocType>> {
+        this.writeQueue = this.writeQueue.then(() => this._bulkWrite(documentWrites, context));
+        const ret = this.writeQueue;
+        this.writeQueue = this.writeQueue.catch(() => { });
+        return ret;
+    }
+
+    async _bulkWrite(documentWrites: BulkWriteRow<RxDocType>[], context: string): Promise<RxStorageBulkWriteResponse<RxDocType>> {
+        const kv = await this.kvPromise;
+        const primaryPath = this.primaryPath;
+        const ret: RxStorageBulkWriteResponse<RxDocType> = {
+            error: []
+        };
+
+        const batches = batchArray(documentWrites, ensureNotFalsy(this.settings.batchSize));
+
+        /**
+         * DenoKV does not have transactions
+         * so we use a special writeBlock row to ensure
+         * atomic writes (per document)
+         * and so that we can do bulkWrites
+         */
+        for (const writeBatch of batches) {
+            let writeRetryCount = 0;
+            while (true) {
+                const writeBlockKey = await kv.get([this.keySpace], this.kvOptions);
+                const docsInDB = new Map<string, RxDocumentData<RxDocType>>();
+
+                /**
+                 * The max amount for .getMany() is 10 which is defined by deno itself:
+                 * @link https://docs.deno.com/deploy/kv/manual/transactions/
+                 * @link https://github.com/denoland/deno/issues/19284
+                 */
+                const readManyBatches = batchArray(writeBatch, 10);
+                await Promise.all(
+                    readManyBatches.map(async (readManyBatch) => {
+                        const docsResult = await kv.getMany(
+                            readManyBatch.map(writeRow => {
+                                const docId: string = writeRow.document[primaryPath] as any;
+                                return [this.keySpace, DENOKV_DOCUMENT_ROOT_PATH, docId];
+                            })
+                        );
+                        docsResult.map((row: any) => {
+                            const docData = row.value;
+                            if (!docData) {
+                                return;
+                            }
+                            const docId: string = docData[primaryPath] as any;
+                            docsInDB.set(docId, docData);
+                        });
+                    })
+                );
+                const categorized = categorizeBulkWriteRows<RxDocType>(
+                    this,
+                    this.primaryPath as any,
+                    docsInDB,
+                    writeBatch,
+                    context
+                );
+
+                let tx = kv.atomic();
+                tx = tx.set([this.keySpace], ensureNotFalsy(writeBlockKey.value) + 1);
+                tx = tx.check(writeBlockKey);
+
+                // INSERTS
+                categorized.bulkInsertDocs.forEach(writeRow => {
+                    const docId: string = writeRow.document[this.primaryPath] as any;
+
+                    // insert document data
+                    tx = tx.set([this.keySpace, DENOKV_DOCUMENT_ROOT_PATH, docId], writeRow.document);
+
+                    // insert secondary indexes
+                    Object.values(this.internals.indexes).forEach(indexMeta => {
+                        const indexString = indexMeta.getIndexableString(writeRow.document as any);
+                        tx = tx.set([this.keySpace, indexMeta.indexId, indexString], docId);
+                    });
+                });
+                // UPDATES
+                categorized.bulkUpdateDocs.forEach((writeRow: BulkWriteRow<RxDocType>) => {
+                    const docId: string = writeRow.document[this.primaryPath] as any;
+
+                    // insert document data
+                    tx = tx.set([this.keySpace, DENOKV_DOCUMENT_ROOT_PATH, docId], writeRow.document);
+
+                    // insert secondary indexes
+                    Object.values(this.internals.indexes).forEach(indexMeta => {
+                        const oldIndexString = indexMeta.getIndexableString(ensureNotFalsy(writeRow.previous));
+                        const newIndexString = indexMeta.getIndexableString(writeRow.document as any);
+                        if (oldIndexString !== newIndexString) {
+                            tx = tx.delete([this.keySpace, indexMeta.indexId, oldIndexString]);
+                            tx = tx.set([this.keySpace, indexMeta.indexId, newIndexString], docId);
+                        }
+                    });
+                });
+
+                let txResult;
+                try {
+                    txResult = await tx.commit();
+                } catch (err: any) {
+                    if (
+                        err.message.includes('Error code 5:') ||
+                        err.message.includes('Error code 517:') ||
+                        err.message.includes('database is locked')
+                    ) {
+                        // retry
+                    } else {
+                        throw err;
+                    }
+                }
+                if (txResult && txResult.ok) {
+                    ret.error = ret.error.concat(categorized.errors);
+                    if (categorized.eventBulk.events.length > 0) {
+                        const lastState = ensureNotFalsy(categorized.newestRow).document;
+                        categorized.eventBulk.checkpoint = {
+                            id: lastState[primaryPath],
+                            lwt: lastState._meta.lwt
+                        };
+                        this.changes$.next(categorized.eventBulk);
+                    }
+                    break;
+                }
+                writeRetryCount++;
+                await promiseWait(writeRetryCount * 5);
+            }
+        }
+        return ret;
+    }
+    async findDocumentsById(ids: string[], withDeleted: boolean): Promise<RxDocumentData<RxDocType>[]> {
+        const kv = await this.kvPromise;
+        const ret: RxDocumentData<RxDocType>[] = [];
+        await Promise.all(
+            ids.map(async (docId) => {
+                const kvKey = [this.keySpace, DENOKV_DOCUMENT_ROOT_PATH, docId];
+                const findSingleResult = await kv.get(kvKey, this.kvOptions);
+                const docInDb = findSingleResult.value;
+                if (
+                    docInDb &&
+                    (
+                        !docInDb._deleted ||
+                        withDeleted
+                    )
+                ) {
+                    ret.push(docInDb);
+                }
+            })
+        );
+        return ret;
+    }
+    query(preparedQuery: PreparedQuery<RxDocType>): Promise<RxStorageQueryResult<RxDocType>> {
+        return this.retryUntilNoWriteInBetween(
+            () => queryDenoKV(this, preparedQuery)
+        );
+    }
+    async count(preparedQuery: PreparedQuery<RxDocType>): Promise<RxStorageCountResult> {
+        /**
+         * At this point in time (end 2023), DenoKV does not support
+         * range counts. So we have to run a normal query and use the result set length.
+         * @link https://github.com/denoland/deno/issues/18965
+         */
+        const result = await this.retryUntilNoWriteInBetween(
+            () => queryDenoKV(this, preparedQuery)
+        );
+        return {
+            count: result.documents.length,
+            mode: 'fast'
+        };
+    }
+    getAttachmentData(documentId: string, attachmentId: string, digest: string): Promise<Blob> {
+        throw new Error("Method not implemented.");
+    }
+    changeStream() {
+        return this.changes$.asObservable();
+    }
+    async cleanup(minimumDeletedTime: number): Promise<boolean> {
+        const maxDeletionTime = now() - minimumDeletedTime;
+        const kv = await this.kvPromise;
+        const index = CLEANUP_INDEX;
+        const indexName = getDenoKVIndexName(index);
+        const indexMeta = this.internals.indexes[indexName];
+        const lowerBoundString = getStartIndexStringFromLowerBound(
+            this.schema,
+            index,
+            [
+                true,
+                /**
+                 * Do not use 0 here,
+                 * because 1 is the minimum value for _meta.lwt
+                 */
+                1
+            ]
+        );
+        const upperBoundString = getStartIndexStringFromLowerBound(
+            this.schema,
+            index,
+            [
+                true,
+                maxDeletionTime
+            ]
+        );
+        let noMoreUndeleted: boolean = true;
+
+        const range = kv.list({
+            start: [this.keySpace, indexMeta.indexId, lowerBoundString],
+            end: [this.keySpace, indexMeta.indexId, upperBoundString]
+        }, {
+            consistency: this.settings.consistencyLevel,
+            batchSize: this.settings.batchSize,
+            limit: this.settings.batchSize
+        });
+
+        let rangeCount = 0;
+        for await (const row of range) {
+            rangeCount = rangeCount + 1;
+            const docId = row.value;
+            const docDataResult = await kv.get([this.keySpace, DENOKV_DOCUMENT_ROOT_PATH, docId], this.kvOptions);
+            if (!docDataResult.value) {
+                continue;
+            }
+            const docData = ensureNotFalsy(docDataResult.value);
+            if (
+                !docData._deleted ||
+                docData._meta.lwt > maxDeletionTime
+            ) {
+                continue;
+            }
+
+            await commitWithRetry(() => {
+                let tx = kv.atomic();
+                tx = tx.check(docDataResult);
+                tx = tx.delete([this.keySpace, DENOKV_DOCUMENT_ROOT_PATH, docId]);
+                Object.values(this.internals.indexes).forEach(indexMetaInner => {
+                    const indexString = indexMetaInner.getIndexableString(docData as any);
+                    tx = tx.delete([this.keySpace, indexMetaInner.indexId, indexString]);
+                });
+                return tx;
+            });
+        }
+        return noMoreUndeleted;
+    }
+    async close(): Promise<void> {
+        if (this.closed) {
+            return this.closed;
+        }
+        this.closed = (async () => {
+            this.changes$.complete();
+            const kv = await this.kvPromise;
+            await kv.close();
+        })();
+        return this.closed;
+    }
+    async remove(): Promise<void> {
+        ensureNotClosed(this);
+        const kv = await this.kvPromise;
+        const range = kv.list({
+            start: [this.keySpace],
+            end: [this.keySpace, INDEX_MAX]
+        }, {
+            consistency: this.settings.consistencyLevel,
+            batchSize: this.settings.batchSize
+        });
+        let promises: Promise<any>[] = [];
+        for await (const row of range) {
+            promises.push(kv.delete(row.key));
+        }
+
+        await Promise.all(promises);
+        return this.close();
+    }
+}
+
+
+
+export async function createDenoKVStorageInstance<RxDocType>(
+    storage: RxStorageDenoKV,
+    params: RxStorageInstanceCreationParams<RxDocType, DenoKVSettings>,
+    settings: DenoKVSettings
+): Promise<RxStorageInstanceDenoKV<RxDocType>> {
+    settings = flatClone(settings);
+    if (!settings.batchSize) {
+        settings.batchSize = 100;
+    }
+
+    const primaryPath = getPrimaryFieldOfPrimaryKey(params.schema.primaryKey);
+
+    const indexDBs: { [indexName: string]: DenoKVIndexMeta<RxDocType>; } = {};
+    const useIndexes = params.schema.indexes ? params.schema.indexes.slice(0) : [];
+    useIndexes.push([primaryPath]);
+    const useIndexesFinal = useIndexes.map(index => {
+        const indexAr = toArray(index);
+        return indexAr;
+    });
+    useIndexesFinal.push(CLEANUP_INDEX);
+    useIndexesFinal.forEach((indexAr, indexId) => {
+        const indexName = getDenoKVIndexName(indexAr);
+        indexDBs[indexName] = {
+            indexId: '|' + indexId + '|',
+            indexName,
+            getIndexableString: getIndexableStringMonad(params.schema, indexAr),
+            index: indexAr
+        };
+    });
+
+    const internals = {
+        indexes: indexDBs
+    };
+    const instance = new RxStorageInstanceDenoKV(
+        storage,
+        params.databaseName,
+        params.collectionName,
+        params.schema,
+        internals,
+        params.options,
+        settings
+    );
+
+    await addRxStorageMultiInstanceSupport(
+        RX_STORAGE_NAME_DENOKV,
+        params,
+        instance
+    );
+
+    return Promise.resolve(instance);
+}
+
+
+
+function ensureNotClosed(
+    instance: RxStorageInstanceDenoKV<any>
+) {
+    if (instance.closed) {
+        throw new Error('RxStorageInstanceDenoKV is closed ' + instance.databaseName + '-' + instance.collectionName);
+    }
+}