@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/storage-dexie/dexie-helper.ts

@@ -0,0 +1,302 @@
+import type {
+    DexieStorageInternals,
+    RxDocumentData,
+    RxJsonSchema
+} from '../../types/index.d.ts';
+import { Dexie } from 'dexie';
+import type { DexieSettings } from '../../types/index.d.ts';
+import { flatClone, getFromMapOrCreate, getProperty, setProperty, toArray, uniqueArray } from '../utils/index.ts';
+import {
+    getPrimaryFieldOfPrimaryKey,
+    getSchemaByObjectPath
+} from '../../rx-schema-helper.ts';
+
+export const DEXIE_DOCS_TABLE_NAME = 'docs';
+export const DEXIE_CHANGES_TABLE_NAME = 'changes';
+export const DEXIE_ATTACHMENTS_TABLE_NAME = 'attachments';
+
+export const RX_STORAGE_NAME_DEXIE = 'dexie';
+
+const DEXIE_STATE_DB_BY_NAME: Map<string, DexieStorageInternals> = new Map();
+const REF_COUNT_PER_DEXIE_DB: Map<DexieStorageInternals, number> = new Map();
+export function getDexieDbWithTables(
+    databaseName: string,
+    collectionName: string,
+    settings: DexieSettings,
+    schema: RxJsonSchema<any>
+): DexieStorageInternals {
+    const dexieDbName = 'rxdb-dexie-' + databaseName + '--' + schema.version + '--' + collectionName;
+
+    const state = getFromMapOrCreate(
+        DEXIE_STATE_DB_BY_NAME,
+        dexieDbName,
+        () => {
+            const value = (async () => {
+                /**
+                 * IndexedDB was not designed for dynamically adding tables on the fly,
+                 * so we create one dexie database per RxDB storage instance.
+                 * @link https://github.com/dexie/Dexie.js/issues/684#issuecomment-373224696
+                 */
+                const useSettings = flatClone(settings);
+                useSettings.autoOpen = false;
+                const dexieDb = new Dexie(dexieDbName, useSettings);
+                if (settings.onCreate) {
+                    await settings.onCreate(dexieDb, dexieDbName);
+                }
+                const dexieStoresSettings = {
+                    [DEXIE_DOCS_TABLE_NAME]: getDexieStoreSchema(schema),
+                    [DEXIE_CHANGES_TABLE_NAME]: '++sequence, id',
+                    [DEXIE_ATTACHMENTS_TABLE_NAME]: 'id'
+                };
+
+                dexieDb.version(1).stores(dexieStoresSettings);
+                await dexieDb.open();
+
+                return {
+                    dexieDb,
+                    dexieTable: (dexieDb as any)[DEXIE_DOCS_TABLE_NAME],
+                    dexieAttachmentsTable: (dexieDb as any)[DEXIE_ATTACHMENTS_TABLE_NAME],
+                    booleanIndexes: getBooleanIndexes(schema)
+                };
+            })();
+            DEXIE_STATE_DB_BY_NAME.set(dexieDbName, state);
+            REF_COUNT_PER_DEXIE_DB.set(state, 0);
+            return value;
+        }
+    );
+    return state;
+}
+
+export async function closeDexieDb(statePromise: DexieStorageInternals) {
+    const state = await statePromise;
+    const prevCount = REF_COUNT_PER_DEXIE_DB.get(statePromise);
+    const newCount = (prevCount as any) - 1;
+    if (newCount === 0) {
+        state.dexieDb.close();
+        REF_COUNT_PER_DEXIE_DB.delete(statePromise);
+    } else {
+        REF_COUNT_PER_DEXIE_DB.set(statePromise, newCount);
+    }
+}
+
+
+
+/**
+ * It is not possible to set non-javascript-variable-syntax
+ * keys as IndexedDB indexes. So we have to substitute the pipe-char
+ * which comes from the key-compression plugin.
+ */
+export const DEXIE_PIPE_SUBSTITUTE = '__';
+export function dexieReplaceIfStartsWithPipe(str: string): string {
+    const split = str.split('.');
+    if (split.length > 1) {
+        return split.map(part => dexieReplaceIfStartsWithPipe(part)).join('.');
+    }
+
+    if (str.startsWith('|')) {
+        const withoutFirst = str.substring(1);
+        return DEXIE_PIPE_SUBSTITUTE + withoutFirst;
+    } else {
+        return str;
+    }
+}
+
+export function dexieReplaceIfStartsWithPipeRevert(str: string): string {
+    const split = str.split('.');
+    if (split.length > 1) {
+        return split.map(part => dexieReplaceIfStartsWithPipeRevert(part)).join('.');
+    }
+
+    if (str.startsWith(DEXIE_PIPE_SUBSTITUTE)) {
+        const withoutFirst = str.substring(DEXIE_PIPE_SUBSTITUTE.length);
+        return '|' + withoutFirst;
+    } else {
+        return str;
+    }
+}
+
+
+/**
+ * IndexedDB does not support boolean indexing.
+ * So we have to replace true/false with '1'/'0'
+ * @param d 
+ */
+export function fromStorageToDexie<RxDocType>(
+    booleanIndexes: string[],
+    inputDoc: RxDocumentData<RxDocType>
+): any {
+    if (!inputDoc) {
+        return inputDoc;
+    }
+    let d = flatClone(inputDoc);
+    d = fromStorageToDexieField(d);
+
+    booleanIndexes.forEach(idx => {
+        const val = getProperty(inputDoc, idx);
+        const newVal = val ? '1' : '0';
+        const useIndex = dexieReplaceIfStartsWithPipe(idx);
+        setProperty(d, useIndex, newVal);
+    });
+
+    return d;
+}
+export function fromDexieToStorage<RxDocType>(
+    booleanIndexes: string[],
+    d: any
+): RxDocumentData<RxDocType> {
+    if (!d) {
+        return d;
+    }
+
+    d = flatClone(d);
+    d = fromDexieToStorageField(d);
+
+    booleanIndexes.forEach(idx => {
+        const val = getProperty(d, idx);
+        const newVal = val === '1' ? true : false;
+        setProperty(d, idx, newVal);
+    });
+
+    return d;
+}
+
+/**
+ * @recursive
+ */
+export function fromStorageToDexieField(documentData: RxDocumentData<any>): any {
+    if (
+        !documentData ||
+        typeof documentData === 'string' ||
+        typeof documentData === 'number' ||
+        typeof documentData === 'boolean'
+    ) {
+        return documentData;
+    } else if (Array.isArray(documentData)) {
+        return documentData.map(row => fromStorageToDexieField(row));
+    } else if (typeof documentData === 'object') {
+        const ret: any = {};
+        Object.entries(documentData).forEach(([key, value]) => {
+            if (typeof value === 'object') {
+                value = fromStorageToDexieField(value);
+            }
+            ret[dexieReplaceIfStartsWithPipe(key)] = value;
+        });
+        return ret;
+    }
+}
+
+export function fromDexieToStorageField(documentData: any): RxDocumentData<any> {
+    if (!documentData || typeof documentData === 'string' || typeof documentData === 'number' || typeof documentData === 'boolean') {
+        return documentData;
+    } else if (Array.isArray(documentData)) {
+        return documentData.map(row => fromDexieToStorageField(row));
+    } else if (typeof documentData === 'object') {
+        const ret: any = {};
+        Object.entries(documentData).forEach(([key, value]) => {
+            if (typeof value === 'object' || Array.isArray(documentData)) {
+                value = fromDexieToStorageField(value);
+            }
+            ret[dexieReplaceIfStartsWithPipeRevert(key)] = value;
+        });
+        return ret;
+    }
+}
+
+
+/**
+ * Creates a string that can be used to create the dexie store.
+ * @link https://dexie.org/docs/API-Reference#quick-reference
+ */
+export function getDexieStoreSchema(
+    rxJsonSchema: RxJsonSchema<any>
+): string {
+    let parts: string[][] = [];
+
+    /**
+     * First part must be the primary key
+     * @link https://github.com/dexie/Dexie.js/issues/1307#issuecomment-846590912
+     */
+    const primaryKey = getPrimaryFieldOfPrimaryKey(rxJsonSchema.primaryKey);
+    parts.push([primaryKey]);
+    parts.push(['_deleted', primaryKey]);
+
+    // add other indexes
+    if (rxJsonSchema.indexes) {
+        rxJsonSchema.indexes.forEach(index => {
+            const arIndex = toArray(index);
+            parts.push(arIndex);
+        });
+    }
+
+    // we also need the _meta.lwt+primaryKey index for the getChangedDocumentsSince() method.
+    parts.push(['_meta.lwt', primaryKey]);
+
+    // and this one for the cleanup()
+    parts.push(['_meta.lwt']);
+
+    /**
+     * It is not possible to set non-javascript-variable-syntax
+     * keys as IndexedDB indexes. So we have to substitute the pipe-char
+     * which comes from the key-compression plugin.
+     */
+    parts = parts.map(part => {
+        return part.map(str => dexieReplaceIfStartsWithPipe(str));
+    });
+
+    let dexieSchemaRows = parts.map(part => {
+        if (part.length === 1) {
+            return part[0];
+        } else {
+            return '[' + part.join('+') + ']';
+        }
+    });
+    dexieSchemaRows = dexieSchemaRows.filter((elem: any, pos: any, arr: any) => arr.indexOf(elem) === pos); // unique;
+
+    const dexieSchema = dexieSchemaRows.join(', ');
+
+    return dexieSchema;
+}
+
+/**
+ * Returns all documents in the database.
+ * Non-deleted plus deleted ones.
+ */
+export async function getDocsInDb<RxDocType>(
+    internals: DexieStorageInternals,
+    docIds: string[]
+): Promise<RxDocumentData<RxDocType>[]> {
+    const state = await internals;
+    const docsInDb = await state.dexieTable.bulkGet(docIds);
+    return docsInDb.map((d: any) => fromDexieToStorage(state.booleanIndexes, d));
+}
+
+
+export function attachmentObjectId(documentId: string, attachmentId: string): string {
+    return documentId + '||' + attachmentId;
+}
+
+
+export function getBooleanIndexes(schema: RxJsonSchema<any>): string[] {
+    const checkedFields = new Set<string>();
+    const ret: string[] = [];
+    if (!schema.indexes) {
+        return ret;
+    }
+    schema.indexes.forEach(index => {
+        const fields = toArray(index);
+        fields.forEach(field => {
+            if (checkedFields.has(field)) {
+                return;
+            }
+            checkedFields.add(field);
+            const schemaObj = getSchemaByObjectPath(schema, field);
+            if (schemaObj.type === 'boolean') {
+                ret.push(field);
+            }
+        });
+    });
+    ret.push('_deleted');
+
+    return uniqueArray(ret);
+}
+

package/src/plugins/storage-dexie/dexie-query.ts

@@ -0,0 +1,252 @@
+import { INDEX_MAX, INDEX_MIN } from '../../query-planner.ts';
+import { getQueryMatcher, getSortComparator } from '../../rx-query-helper.ts';
+import type {
+    PreparedQuery,
+    QueryMatcher,
+    RxDocumentData,
+    RxQueryPlan,
+    RxStorageQueryResult
+} from '../../types/index.d.ts';
+import {
+    dexieReplaceIfStartsWithPipe,
+    DEXIE_DOCS_TABLE_NAME,
+    fromDexieToStorage
+} from './dexie-helper.ts';
+import type { RxStorageInstanceDexie } from './rx-storage-instance-dexie.ts';
+
+export function mapKeyForKeyRange(k: any) {
+    if (k === INDEX_MIN) {
+        return -Infinity;
+    } else {
+        return k;
+    }
+}
+
+function rangeFieldToBooleanSubstitute(
+    booleanIndexes: string[],
+    fieldName: string,
+    value: any
+) {
+    if (booleanIndexes.includes(fieldName)) {
+        const newValue = value === INDEX_MAX || value === true ? '1' : '0';
+        return newValue;
+    } else {
+        return value;
+    }
+}
+
+export function getKeyRangeByQueryPlan(
+    booleanIndexes: string[],
+    queryPlan: RxQueryPlan,
+    IDBKeyRange?: any
+) {
+    if (!IDBKeyRange) {
+        if (typeof window === 'undefined') {
+            throw new Error('IDBKeyRange missing');
+        } else {
+            IDBKeyRange = window.IDBKeyRange;
+        }
+    }
+
+    const startKeys = queryPlan.startKeys
+        .map((v, i) => {
+            const fieldName = queryPlan.index[i];
+            return rangeFieldToBooleanSubstitute(booleanIndexes, fieldName, v);
+        })
+        .map(mapKeyForKeyRange);
+    const endKeys = queryPlan.endKeys
+        .map((v, i) => {
+            const fieldName = queryPlan.index[i];
+            return rangeFieldToBooleanSubstitute(booleanIndexes, fieldName, v);
+        })
+        .map(mapKeyForKeyRange);
+
+    const keyRange = IDBKeyRange.bound(
+        startKeys,
+        endKeys,
+        !queryPlan.inclusiveStart,
+        !queryPlan.inclusiveEnd
+    );
+    return keyRange;
+}
+
+
+/**
+ * Runs mango queries over the Dexie.js database.
+ */
+export async function dexieQuery<RxDocType>(
+    instance: RxStorageInstanceDexie<RxDocType>,
+    preparedQuery: PreparedQuery<RxDocType>
+): Promise<RxStorageQueryResult<RxDocType>> {
+    const state = await instance.internals;
+    const query = preparedQuery.query;
+
+    const skip = query.skip ? query.skip : 0;
+    const limit = query.limit ? query.limit : Infinity;
+    const skipPlusLimit = skip + limit;
+    const queryPlan = preparedQuery.queryPlan;
+
+    let queryMatcher: QueryMatcher<RxDocumentData<RxDocType>> | false = false;
+    if (!queryPlan.selectorSatisfiedByIndex) {
+        queryMatcher = getQueryMatcher(
+            instance.schema,
+            preparedQuery.query
+        );
+    }
+    let keyRange;
+    try {
+        keyRange = getKeyRangeByQueryPlan(
+            state.booleanIndexes,
+            queryPlan,
+            (state.dexieDb as any)._options.IDBKeyRange
+        );
+    } catch (err: any) {
+        if (err.name === 'DataError') {
+            return { documents: [] };
+        }
+        throw err;
+    }
+
+    const queryPlanFields: string[] = queryPlan.index;
+
+    let rows: any[] = [];
+    await state.dexieDb.transaction(
+        'r',
+        state.dexieTable,
+        async (dexieTx: any) => {
+            /**
+             * Here we use the native IndexedDB transaction
+             * to get the cursor.
+             * Maybe we should not leave Dexie.js API and find
+             * a way to create the cursor with Dexie.js.
+             */
+            const tx = (dexieTx as any).idbtrans;
+
+            // const nativeIndexedDB = state.dexieDb.backendDB();
+            // const trans = nativeIndexedDB.transaction([DEXIE_DOCS_TABLE_NAME], 'readonly');
+
+            const store = tx.objectStore(DEXIE_DOCS_TABLE_NAME);
+            let index: any;
+            let indexName: string;
+            indexName = '[' +
+                queryPlanFields
+                    .map(field => dexieReplaceIfStartsWithPipe(field))
+                    .join('+')
+                + ']';
+            index = store.index(indexName);
+
+
+            const cursorReq = index.openCursor(keyRange);
+            await new Promise<void>(res => {
+                cursorReq.onsuccess = function (e: any) {
+                    const cursor = e.target.result;
+                    if (cursor) {
+                        // We have a record in cursor.value
+                        const docData = fromDexieToStorage<RxDocType>(state.booleanIndexes, cursor.value);
+                        if (!queryMatcher || queryMatcher(docData)) {
+                            rows.push(docData);
+                        }
+
+                        /**
+                         * If we do not have to manually sort
+                         * and have enough documents,
+                         * we can abort iterating over the cursor
+                         * because we already have every relevant document.
+                         */
+                        if (
+                            queryPlan.sortSatisfiedByIndex &&
+                            rows.length === skipPlusLimit
+                        ) {
+                            res();
+                        } else {
+                            cursor.continue();
+                        }
+                    } else {
+                        // Iteration complete
+                        res();
+                    }
+                };
+            });
+
+
+        }
+    );
+
+
+    if (!queryPlan.sortSatisfiedByIndex) {
+        const sortComparator = getSortComparator(instance.schema, preparedQuery.query);
+        rows = rows.sort(sortComparator);
+    }
+
+    // apply skip and limit boundaries.
+    rows = rows.slice(skip, skipPlusLimit);
+
+    /**
+     * Comment this in for debugging to check all fields in the database.
+     */
+    // const docsInDb = await state.dexieTable.filter(queryMatcher).toArray();
+    // let documents = docsInDb
+    //     .map(docData => stripDexieKey(docData))
+    //     .sort(sortComparator);
+    // if (preparedQuery.skip) {
+    //     documents = documents.slice(preparedQuery.skip);
+    // }
+    // if (preparedQuery.limit && documents.length > preparedQuery.limit) {
+    //     documents = documents.slice(0, preparedQuery.limit);
+    // }
+
+
+
+    return {
+        documents: rows
+    };
+}
+
+
+export async function dexieCount<RxDocType>(
+    instance: RxStorageInstanceDexie<RxDocType>,
+    preparedQuery: PreparedQuery<RxDocType>
+): Promise<number> {
+    const state = await instance.internals;
+    const queryPlan = preparedQuery.queryPlan;
+    const queryPlanFields: string[] = queryPlan.index;
+
+    let keyRange;
+    try {
+        keyRange = getKeyRangeByQueryPlan(
+            state.booleanIndexes,
+            queryPlan,
+            (state.dexieDb as any)._options.IDBKeyRange
+        );
+    } catch (err: any) {
+        if (err.name === 'DataError') {
+            return 0;
+        }
+        throw err;
+    }
+    let count: number = -1;
+    await state.dexieDb.transaction(
+        'r',
+        state.dexieTable,
+        async (dexieTx: any) => {
+            const tx = (dexieTx as any).idbtrans;
+            const store = tx.objectStore(DEXIE_DOCS_TABLE_NAME);
+            let index: any;
+            let indexName: string;
+            indexName = '[' +
+                queryPlanFields
+                    .map(field => dexieReplaceIfStartsWithPipe(field))
+                    .join('+')
+                + ']';
+            index = store.index(indexName);
+            const request = index.count(keyRange);
+            count = await new Promise<number>((res, rej) => {
+                request.onsuccess = function () {
+                    res(request.result);
+                };
+                request.onerror = (err: any) => rej(err);
+            });
+        }
+    );
+    return count;
+}

package/src/plugins/storage-dexie/index.ts

@@ -0,0 +1,4 @@
+export * from './rx-storage-dexie.ts';
+export * from './rx-storage-instance-dexie.ts';
+export * from './dexie-helper.ts';
+export * from './dexie-query.ts';

package/src/plugins/storage-dexie/rx-storage-dexie.ts

@@ -0,0 +1,62 @@
+import type {
+    RxStorage,
+    RxStorageInstanceCreationParams
+} from '../../types/index.d.ts';
+import {
+    RX_STORAGE_NAME_DEXIE
+} from './dexie-helper.ts';
+import type {
+    DexieSettings,
+    DexieStorageInternals
+} from '../../types/plugins/dexie.d.ts';
+import {
+    createDexieStorageInstance,
+    RxStorageInstanceDexie
+} from './rx-storage-instance-dexie.ts';
+import { ensureRxStorageInstanceParamsAreCorrect } from '../../rx-storage-helper.ts';
+import { RXDB_VERSION } from '../utils/utils-rxdb-version.ts';
+import { newRxError } from '../../rx-error.ts';
+
+
+
+export class RxStorageDexie implements RxStorage<DexieStorageInternals, DexieSettings> {
+    public name = RX_STORAGE_NAME_DEXIE;
+    public readonly rxdbVersion = RXDB_VERSION;
+    constructor(
+        public settings: DexieSettings
+    ) { }
+
+    public createStorageInstance<RxDocType>(
+        params: RxStorageInstanceCreationParams<RxDocType, DexieSettings>
+    ): Promise<RxStorageInstanceDexie<RxDocType>> {
+        ensureRxStorageInstanceParamsAreCorrect(params);
+
+        /**
+         * Dexie does not support non-required indexes and must throw if that is used.
+         * @link https://github.com/pubkey/rxdb/pull/6643#issuecomment-2505310082
+         */
+        if (params.schema.indexes) {
+            const indexFields = params.schema.indexes.flat();
+            indexFields
+                .filter(indexField => !indexField.includes('.'))
+                .forEach(indexField => {
+                    if (!params.schema.required || !params.schema.required.includes(indexField as any)) {
+                        throw newRxError('DXE1', {
+                            field: indexField,
+                            schema: params.schema,
+                        });
+                    }
+                });
+        }
+
+        return createDexieStorageInstance(this, params, this.settings);
+    }
+}
+
+
+export function getRxStorageDexie(
+    settings: DexieSettings = {}
+): RxStorageDexie {
+    const storage = new RxStorageDexie(settings);
+    return storage;
+}