@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/state/rx-state.ts

@@ -0,0 +1,386 @@
+import {
+    Observable,
+    Subject,
+    distinctUntilChanged,
+    map,
+    merge,
+    shareReplay,
+    startWith,
+    tap
+} from 'rxjs';
+import { overwritable } from '../../overwritable.ts';
+import { getChangedDocumentsSince } from '../../rx-storage-helper.ts';
+import type {
+    RxChangeEventBulk,
+    RxCollection,
+    RxDatabase,
+    RxQuery,
+    RxDocument,
+    RxError,
+    Paths
+} from '../../types';
+import {
+    RXJS_SHARE_REPLAY_DEFAULTS,
+    getProperty,
+    setProperty,
+    PROMISE_RESOLVE_VOID,
+    clone,
+    randomToken,
+    deepEqual,
+    getFromMapOrCreate
+} from '../utils/index.ts';
+import {
+    RX_STATE_COLLECTION_SCHEMA,
+    isValidWeakMapKey,
+    nextRxStateId
+} from './helpers.ts';
+import {
+    RxStateDocument,
+    RxStateOperation,
+    RxStateModifier
+} from './types.ts';
+import { newRxError } from '../../rx-error.ts';
+import { runPluginHooks } from '../../hooks.ts';
+
+
+let debugId = 0;
+
+
+const deepFrozenCache = new WeakMap<any, any>();
+
+/**
+ * RxDB internally used properties are
+ * prefixed with lodash _ to make them less
+ * likely to clash with actual state properties
+ * from the user.
+ */
+export class RxStateBase<T, Reactivity = unknown> {
+    // used for debugging
+    public _id: number = debugId++;
+    public _state: T | any = {};
+    public $: Observable<T>;
+    public _lastIdQuery: RxQuery<RxStateDocument, RxDocument<RxStateDocument, {}> | null>;
+    public _nonPersisted: {
+        path: string;
+        modifier: RxStateModifier;
+    }[] = [];
+    public _writeQueue = PROMISE_RESOLVE_VOID;
+    public _initDone = false;
+    public _instanceId = randomToken(RX_STATE_COLLECTION_SCHEMA.properties.sId.maxLength);
+    public _ownEmits$ = new Subject<T>();
+
+    constructor(
+        public readonly prefix: string,
+        public readonly collection: RxCollection<RxStateDocument>
+    ) {
+        this.collection.onClose.push(() => this._writeQueue);
+        this._lastIdQuery = this.collection.findOne({
+            sort: [
+                { id: 'desc' }
+            ]
+        });
+        // make it "hot" for better write performance
+        this._lastIdQuery.$.subscribe();
+
+        this.$ = merge(
+            this._ownEmits$,
+            this.collection.eventBulks$.pipe(
+                tap((eventBulk: RxChangeEventBulk<RxStateDocument>) => {
+                    if (!this._initDone) {
+                        return;
+                    }
+                    const events = eventBulk.events;
+                    for (let index = 0; index < events.length; index++) {
+                        const event = events[index];
+                        if (
+                            event.operation === 'INSERT' &&
+                            event.documentData.sId !== this._instanceId
+                        ) {
+                            this.mergeOperationsIntoState(event.documentData.ops);
+                        }
+                    }
+                })
+            )
+        ).pipe(
+            shareReplay(RXJS_SHARE_REPLAY_DEFAULTS),
+            map(() => this._state)
+        );
+        // directly subscribe because of the tap() side effect
+        this.$.subscribe();
+    }
+
+    async set(
+        path: Paths<T> | '',
+        modifier: RxStateModifier
+    ) {
+        this._nonPersisted.push({
+            path,
+            modifier
+        });
+        return this._triggerWrite();
+    }
+
+    /**
+     * To have deterministic writes,
+     * and to ensure that multiple js realms do not overwrite
+     * each other, the write happens with incremental ids
+     * that would throw conflict errors and trigger a retry.
+     */
+    _triggerWrite() {
+        this._writeQueue = this._writeQueue.then(async () => {
+            if (this._nonPersisted.length === 0) {
+                return;
+            }
+            let useWrites: typeof this._nonPersisted = [];
+            let done = false;
+            while (!done) {
+                const lastIdDoc = await this._lastIdQuery.exec();
+                useWrites = useWrites.concat(this._nonPersisted);
+                this._nonPersisted = [];
+                const nextId = nextRxStateId(lastIdDoc ? lastIdDoc.id : undefined);
+                try {
+                    /**
+                     * TODO instead of a deep-clone we should
+                     * only clone the parts where we know that they
+                     * will be changed. This would improve performance.
+                     */
+                    let newState = clone(this._state);
+                    const ops: RxStateOperation[] = [];
+                    for (let index = 0; index < useWrites.length; index++) {
+                        const writeRow = useWrites[index];
+                        const value = getProperty(newState, writeRow.path);
+                        const newValue = writeRow.modifier(value);
+                        /**
+                         * Here we have to clone the value because
+                         * some storages like the memory storage
+                         * make input data deep-frozen in dev-mode.
+                         */
+                        if (writeRow.path === '') {
+                            newState = clone(newValue);
+                        } else {
+                            setProperty(newState, writeRow.path, clone(newValue));
+                        }
+                        ops.push({
+                            k: writeRow.path,
+                            /**
+                             * Here we have to clone the value because
+                             * some storages like the memory storage
+                             * make input data deep-frozen in dev-mode.
+                             */
+                            v: clone(newValue)
+                        });
+                    }
+                    await this.collection.insert({
+                        id: nextId,
+                        sId: this._instanceId,
+                        ops
+                    });
+                    this._state = newState;
+                    this._ownEmits$.next(this._state);
+                    done = true;
+                } catch (err) {
+                    if ((err as RxError).code !== 'CONFLICT') {
+                        throw err;
+                    }
+                }
+            }
+        }).catch(error => {
+            throw newRxError('SNH', {
+                name: 'RxState WRITE QUEUE ERROR',
+                error
+            });
+        });
+        return this._writeQueue;
+    }
+
+    mergeOperationsIntoState(
+        operations: RxStateOperation[]
+    ) {
+        let state = clone(this._state);
+        for (let index = 0; index < operations.length; index++) {
+            const operation = operations[index];
+            if (operation.k === '') {
+                state = clone(operation.v);
+            } else {
+                setProperty(state, operation.k, clone(operation.v));
+            }
+        }
+        this._state = state;
+    }
+    get(path?: Paths<T>) {
+        let ret;
+        if (!path) {
+            ret = this._state;
+        } else {
+            ret = getProperty(this._state, path);
+        }
+
+        /**
+         * In dev-mode we have to clone the value before deep-freezing
+         * it to not have an immutable subobject in the state value.
+         * But calling .get() with the same path multiple times,
+         * should return exactly the same object instance
+         * so it does not cause re-renders on react.
+         * So in dev-mode we have to 
+         */
+        if (overwritable.isDevMode() && isValidWeakMapKey(ret)) {
+            const frozen = getFromMapOrCreate(
+                deepFrozenCache,
+                ret,
+                () => overwritable.deepFreezeWhenDevMode(clone(ret))
+            );
+            return frozen;
+        }
+
+        return ret;
+    }
+    get$(path?: Paths<T>): Observable<any> {
+        return this.$.pipe(
+            map(() => this.get(path)),
+            startWith(this.get(path)),
+            distinctUntilChanged(deepEqual),
+            shareReplay(RXJS_SHARE_REPLAY_DEFAULTS),
+        );
+    }
+    get$$(path?: Paths<T>): Reactivity {
+        const obs = this.get$(path);
+        const reactivity = this.collection.database.getReactivityFactory();
+        return reactivity.fromObservable(
+            obs,
+            this.get(path),
+            this.collection.database
+        ) as any;
+    }
+
+    /**
+     * Merges the state operations into a single write row
+     * to store space and make recreating the state from
+     * disc faster.
+     */
+    async _cleanup() {
+        const firstWrite = await this.collection.findOne({
+            sort: [{ id: 'asc' }]
+        }).exec();
+        const lastWrite = await this._lastIdQuery.exec();
+
+        if (!firstWrite || !lastWrite) {
+            return;
+        }
+
+        const firstNr = parseInt(firstWrite.id, 10);
+        const lastNr = parseInt(lastWrite.id, 10);
+        if ((lastNr - 5) < firstNr) {
+            // only run if more than 5 write rows
+            return;
+        }
+
+        // update whole state object
+        await this._writeQueue;
+        await this.set('', () => this._state);
+
+        // delete old ones
+        await this.collection.find({
+            selector: {
+                id: {
+                    $lte: lastWrite.id
+                }
+            }
+        }).remove();
+    }
+}
+
+
+export async function createRxState<T>(
+    database: RxDatabase,
+    prefix: string
+): Promise<RxStateBase<T>> {
+    const collectionName = 'rx-state-' + prefix;
+    await database.addCollections({
+        [collectionName]: {
+            schema: RX_STATE_COLLECTION_SCHEMA as any
+        }
+    });
+    const collection: RxCollection<RxStateDocument> = database.collections[collectionName];
+
+    const rxState = new RxStateBase<T>(
+        prefix,
+        collection
+    );
+
+
+    /**
+     * Directly get the state and put it into memory.
+     * This ensures we can do non-async accesses to the
+     * correct state.
+     */
+    let done = false;
+    let checkpoint: any = undefined;
+    while (!done) {
+        const result = await getChangedDocumentsSince<RxStateDocument, any>(
+            collection.storageInstance,
+            1000,
+            checkpoint
+        );
+        checkpoint = result.checkpoint;
+        const documents = result.documents;
+        if (documents.length === 0) {
+            done = true;
+        } else {
+            for (let index = 0; index < documents.length; index++) {
+                const document = documents[index];
+                mergeOperationsIntoState(rxState._state, document.ops);
+            }
+        }
+    }
+    rxState._initDone = true;
+
+    const proxy = new Proxy(
+        rxState as any,
+        {
+            get(target, property: any) {
+                if (typeof property !== 'string') {
+                    return target[property];
+                }
+                if ((rxState as any)[property]) {
+                    const ret = (rxState as any)[property];
+                    if (typeof ret === 'function') {
+                        return ret.bind(rxState);
+                    } else {
+                        return ret;
+                    }
+                }
+                const lastChar = property.charAt(property.length - 1);
+                if (property.endsWith('$$')) {
+                    const key = property.slice(0, -2);
+                    return rxState.get$$(key as any);
+                } else if (lastChar === '$') {
+                    const key = property.slice(0, -1);
+                    return rxState.get$(key as any);
+                } else {
+                    return rxState.get(property as any);
+                }
+            },
+            set(target, newValue, receiver) {
+                throw new Error('Do not write to RxState');
+            }
+        }
+    );
+
+    runPluginHooks('createRxState', {
+        collection,
+        state: proxy
+    });
+
+    return proxy;
+}
+
+
+export function mergeOperationsIntoState<T>(
+    state: T,
+    operations: RxStateOperation[]
+) {
+    for (let index = 0; index < operations.length; index++) {
+        const operation = operations[index];
+        setProperty(state, operation.k, clone(operation.v));
+    }
+}

package/src/plugins/state/types.ts

@@ -0,0 +1,32 @@
+import { DeepReadonly } from '../../types';
+
+/**
+ * 
+ */
+export type RxStateDocument = {
+    /**
+     * Ensures that when multiple
+     * javascript realms write at the same time,
+     * we do not overwrite each other but instead
+     * one write must conflict-error and retry.
+     * The clock value is also the primary key.
+     * The clock value contains incremental numbers
+     * in a string format like '0001', '0123'...
+     */
+    id: string;
+    /**
+     * Id of the JavaScript Instance of RxState
+     * that did the write. Used to optimise performance
+     * by not running these modifiers twice.
+     */
+    sId: string;
+    ops: RxStateOperation[]
+};
+
+
+export type RxStateOperation = {
+    k: string;
+    v: any;
+};
+
+export type RxStateModifier = (preValue: DeepReadonly<any>) => any;

package/src/plugins/storage-denokv/denokv-helper.ts

@@ -0,0 +1,46 @@
+import { promiseWait } from '../utils/utils-promise.ts';
+
+export const RX_STORAGE_NAME_DENOKV = 'denokv';
+
+export function getDenoKVIndexName(index: string[]): string {
+    return index.join('|');
+}
+
+/**
+ * Used for non-index rows that contain the document data,
+ * not just a documentId
+ */
+export const DENOKV_DOCUMENT_ROOT_PATH = '||';
+
+export const CLEANUP_INDEX: string[] = ['_deleted', '_meta.lwt'];
+
+
+/**
+ * Get the global Deno variable from globalThis.Deno
+ * so that compiling with plain typescript does not fail.
+ * Deno has no way to just "download" the deno typings,
+ * so we have to use the "any" type here.
+ */
+export function getDenoGlobal(): any {
+    return (globalThis as any).Deno;
+}
+
+
+export async function commitWithRetry(buildTx: () => any) {
+    let attempt = 0;
+
+    while (true) {
+        const tx = buildTx();
+        try {
+            return await tx.commit();
+        } catch (err) {
+            const locked = err && String((err as any).message).includes('database is locked');
+            if (locked && attempt < 10) {
+                attempt++;
+                await promiseWait(10 * attempt);
+                continue;
+            }
+            throw err;
+        }
+    }
+}

package/src/plugins/storage-denokv/denokv-query.ts

@@ -0,0 +1,135 @@
+import {
+    changeIndexableStringByOneQuantum,
+    getStartIndexStringFromLowerBound,
+    getStartIndexStringFromUpperBound
+} from '../../custom-index.ts';
+import type {
+    PreparedQuery,
+    QueryMatcher,
+    RxDocumentData,
+    RxStorageQueryResult
+} from '../../types/index.d.ts';
+import { ensureNotFalsy } from '../../plugins/utils/index.ts';
+import { getQueryMatcher, getSortComparator } from '../../rx-query-helper.ts';
+import { RxStorageInstanceDenoKV } from "./rx-storage-instance-denokv.ts";
+import { DENOKV_DOCUMENT_ROOT_PATH, getDenoKVIndexName } from "./denokv-helper.ts";
+
+export async function queryDenoKV<RxDocType>(
+    instance: RxStorageInstanceDenoKV<RxDocType>,
+    preparedQuery: PreparedQuery<RxDocType>
+): Promise<RxStorageQueryResult<RxDocType>> {
+    const queryPlan = preparedQuery.queryPlan;
+    const query = preparedQuery.query;
+    const skip = query.skip ? query.skip : 0;
+    const limit = query.limit ? query.limit : Infinity;
+    const skipPlusLimit = skip + limit;
+    const queryPlanFields: string[] = queryPlan.index;
+    const mustManuallyResort = !queryPlan.sortSatisfiedByIndex;
+
+
+    let queryMatcher: QueryMatcher<RxDocumentData<RxDocType>> | false = false;
+    if (!queryPlan.selectorSatisfiedByIndex) {
+        queryMatcher = getQueryMatcher(
+            instance.schema,
+            preparedQuery.query
+        );
+    }
+
+    const kv = await instance.kvPromise;
+
+    const indexForName = queryPlanFields.slice(0);
+    const indexName = getDenoKVIndexName(indexForName);
+    const indexMeta = ensureNotFalsy(instance.internals.indexes[indexName]);
+
+    let lowerBound: any[] = queryPlan.startKeys;
+    let lowerBoundString = getStartIndexStringFromLowerBound(
+        instance.schema,
+        indexForName,
+        lowerBound
+    );
+    if (!queryPlan.inclusiveStart) {
+        lowerBoundString = changeIndexableStringByOneQuantum(lowerBoundString, 1);
+    }
+
+    let upperBound: any[] = queryPlan.endKeys;
+    let upperBoundString = getStartIndexStringFromUpperBound(
+        instance.schema,
+        indexForName,
+        upperBound
+    );
+
+    if (queryPlan.inclusiveEnd) {
+        upperBoundString = changeIndexableStringByOneQuantum(upperBoundString, +1);
+    }
+
+    if (lowerBoundString > upperBoundString) {
+        return {
+            documents: []
+        };
+    }
+
+    let result: RxDocumentData<RxDocType>[] = [];
+
+
+    /**
+     * TODO for whatever reason the keySelectors like firstGreaterThan etc.
+     * do not work properly. So we have to hack here to find the correct
+     * document in case lowerBoundString===upperBoundString.
+     * This likely must be fixed in the foundationdb library.
+     * When it is fixed, we do not need this if-case and instead
+     * can rely on .getRangeBatch() in all cases.
+     */
+    if (lowerBoundString === upperBoundString) {
+        const singleDocResult = await kv.get([instance.keySpace, indexMeta.indexId, lowerBoundString], instance.kvOptions);
+        if (singleDocResult.value) {
+            const docId: string = singleDocResult.value;
+            const docDataResult = await kv.get([instance.keySpace, DENOKV_DOCUMENT_ROOT_PATH, docId], instance.kvOptions);
+            const docData = docDataResult.value;
+            if (docData && (!queryMatcher || queryMatcher(docData))) {
+                result.push(docData);
+            }
+        }
+        return {
+            documents: result
+        };
+    }
+
+    const range = kv.list({
+        start: [instance.keySpace, indexMeta.indexId, lowerBoundString],
+        end: [instance.keySpace, indexMeta.indexId, upperBoundString]
+    }, {
+        consistency: instance.settings.consistencyLevel,
+        limit: (!mustManuallyResort && queryPlan.selectorSatisfiedByIndex) ? skipPlusLimit : undefined,
+        batchSize: instance.settings.batchSize
+    });
+
+    for await (const indexDocEntry of range) {
+        const docId = indexDocEntry.value;
+        const docDataResult = await kv.get([instance.keySpace, DENOKV_DOCUMENT_ROOT_PATH, docId], instance.kvOptions);
+        const docData = docDataResult.value;
+        if (!docData) {
+            continue;
+        }
+        if (!queryMatcher || queryMatcher(docData)) {
+            result.push(docData);
+        }
+        if (
+            !mustManuallyResort &&
+            result.length === skipPlusLimit
+        ) {
+            break;
+        }
+    }
+
+    if (mustManuallyResort) {
+        const sortComparator = getSortComparator(instance.schema, preparedQuery.query);
+        result = result.sort(sortComparator);
+    }
+
+    // apply skip and limit boundaries.
+    result = result.slice(skip, skipPlusLimit);
+
+    return {
+        documents: result
+    };
+}

package/src/plugins/storage-denokv/denokv-types.ts

@@ -0,0 +1,19 @@
+import type { RxDocumentData } from "../../types/index.d.ts";
+
+export type DenoKVSettings = {
+    consistencyLevel: "strong" | "eventual";
+    openKvPath?: string;
+    batchSize?: number;
+};
+export type DenoKVStorageInternals<RxDocType> = {
+    indexes: {
+        [indexName: string]: DenoKVIndexMeta<RxDocType>;
+    };
+};
+
+export type DenoKVIndexMeta<RxDocType> = {
+    indexId: string;
+    indexName: string;
+    index: string[];
+    getIndexableString: (doc: RxDocumentData<RxDocType>) => string;
+};

package/src/plugins/storage-denokv/index.ts

@@ -0,0 +1,37 @@
+import type {
+    RxStorage,
+    RxStorageInstanceCreationParams
+} from '../../types/index.d.ts';
+import { ensureRxStorageInstanceParamsAreCorrect } from '../../rx-storage-helper.ts';
+import type { DenoKVSettings, DenoKVStorageInternals } from './denokv-types.ts';
+import { RX_STORAGE_NAME_DENOKV } from "./denokv-helper.ts";
+import { RxStorageInstanceDenoKV, createDenoKVStorageInstance } from "./rx-storage-instance-denokv.ts";
+import { RXDB_VERSION } from '../utils/utils-rxdb-version.ts';
+
+
+
+export class RxStorageDenoKV implements RxStorage<DenoKVStorageInternals<any>, DenoKVSettings> {
+    public name = RX_STORAGE_NAME_DENOKV;
+    public readonly rxdbVersion = RXDB_VERSION;
+
+    constructor(
+        public settings: DenoKVSettings
+    ) { }
+
+    public createStorageInstance<RxDocType>(
+        params: RxStorageInstanceCreationParams<RxDocType, DenoKVSettings>
+    ): Promise<RxStorageInstanceDenoKV<RxDocType>> {
+        ensureRxStorageInstanceParamsAreCorrect(params);
+        return createDenoKVStorageInstance(this, params, this.settings);
+    }
+}
+
+
+export function getRxStorageDenoKV(
+    settings: DenoKVSettings = {
+        consistencyLevel: 'strong'
+    }
+): RxStorageDenoKV {
+    const storage = new RxStorageDenoKV(settings);
+    return storage;
+}