@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/storage-localstorage/rx-storage-instance-localstorage.ts

@@ -0,0 +1,686 @@
+import { Observable, Subject, Subscription } from 'rxjs';
+import {
+    PROMISE_RESOLVE_TRUE,
+    PROMISE_RESOLVE_VOID,
+    arrayBufferToBase64,
+    ensureNotFalsy,
+    now,
+    toArray
+} from '../utils/index.ts';
+import type {
+    BulkWriteRow,
+    ById,
+    EventBulk,
+    PreparedQuery,
+    QueryMatcher,
+    RxDocumentData,
+    RxDocumentWriteData,
+    RxJsonSchema,
+    RxStorageBulkWriteResponse,
+    RxStorageChangeEvent,
+    RxStorageCountResult,
+    RxStorageDefaultCheckpoint,
+    RxStorageInstance,
+    RxStorageInstanceCreationParams,
+    RxStorageQueryResult,
+    StringKeys
+} from '../../types/index';
+import {
+    categorizeBulkWriteRows
+} from '../../rx-storage-helper.ts';
+import { getPrimaryFieldOfPrimaryKey } from '../../rx-schema-helper.ts';
+import { getQueryMatcher, getSortComparator } from '../../rx-query-helper.ts';
+import { newRxError } from '../../rx-error.ts';
+import type { RxStorageLocalstorage } from './index.ts';
+import {
+    getIndexableStringMonad,
+    getStartIndexStringFromLowerBound,
+    getStartIndexStringFromUpperBound
+} from '../../custom-index.ts';
+import { pushAtSortPosition } from 'array-push-at-sort-position';
+import { boundEQ, boundGE, boundGT, boundLE, boundLT } from '../storage-memory/binary-search-bounds.ts';
+
+export const RX_STORAGE_NAME_LOCALSTORAGE = 'localstorage';
+
+
+export type LocalstorageStorageInternals<RxDocType = any> = {
+    indexes: ById<IndexMeta<RxDocType>>;
+};
+
+export type LocalstorageInstanceCreationOptions = {};
+
+export type LocalstorageStorageSettings = {
+    localStorage?: typeof localStorage
+};
+
+// index-string to doc-id mapped
+export type LocalstorageIndex = string[][];
+
+export type ChangeStreamStoredData<RxDocType> = {
+    databaseInstanceToken: string;
+    eventBulk: EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, any>;
+}
+
+
+/**
+ * StorageEvents are not sent to the same
+ * browser tab where they were created.
+ * This makes it hard to write unit tests
+ * so we redistribute the events here instead.
+ */
+export const storageEventStream$: Subject<{
+    fromStorageEvent: boolean;
+    key: string;
+    newValue: string | null;
+    databaseInstanceToken?: string;
+}> = new Subject();
+const storageEventStreamObservable = storageEventStream$.asObservable();
+let storageEventStreamSubscribed = false;
+export function getStorageEventStream() {
+    if (!storageEventStreamSubscribed && typeof window !== 'undefined') {
+        storageEventStreamSubscribed = true;
+        window.addEventListener('storage', (ev: StorageEvent) => {
+            if (!ev.key) {
+                return;
+            }
+            storageEventStream$.next({
+                fromStorageEvent: true,
+                key: ev.key,
+                newValue: ev.newValue
+            });
+        });
+    }
+    return storageEventStreamObservable;
+}
+
+let instanceId = 0;
+export class RxStorageInstanceLocalstorage<RxDocType> implements RxStorageInstance<
+    RxDocType,
+    LocalstorageStorageInternals,
+    LocalstorageInstanceCreationOptions,
+    RxStorageDefaultCheckpoint
+> {
+    public readonly primaryPath: StringKeys<RxDocType>;
+
+    /**
+     * Under this key the whole state
+     * will be stored as stringified json
+     * inside of the localstorage.
+     */
+    public readonly docsKey: string;
+    public readonly attachmentsKey: string;
+    public readonly changestreamStorageKey: string;
+    public readonly indexesKey: string;
+    private changeStreamSub: Subscription;
+
+    private changes$: Subject<EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, RxStorageDefaultCheckpoint>> = new Subject();
+    public closed?: Promise<void>;
+    public readonly localStorage: typeof localStorage;
+    public removed: boolean = false;
+    public readonly instanceId = instanceId++;
+
+    constructor(
+        public readonly storage: RxStorageLocalstorage,
+        public readonly databaseName: string,
+        public readonly collectionName: string,
+        public readonly schema: Readonly<RxJsonSchema<RxDocumentData<RxDocType>>>,
+        public readonly internals: LocalstorageStorageInternals,
+        public readonly options: Readonly<LocalstorageInstanceCreationOptions>,
+        public readonly settings: LocalstorageStorageSettings,
+        public readonly multiInstance: boolean,
+        public readonly databaseInstanceToken: string
+    ) {
+        this.localStorage = settings.localStorage ? settings.localStorage : (typeof window !== 'undefined' ? window.localStorage : undefined as any);
+        this.primaryPath = getPrimaryFieldOfPrimaryKey(this.schema.primaryKey) as any;
+        this.docsKey = 'RxDB-ls-doc-' + this.databaseName + '--' + this.collectionName + '--' + this.schema.version;
+        this.changestreamStorageKey = 'RxDB-ls-changes-' + this.databaseName + '--' + this.collectionName + '--' + this.schema.version;
+        this.indexesKey = 'RxDB-ls-idx-' + this.databaseName + '--' + this.collectionName + '--' + this.schema.version;
+        this.attachmentsKey = 'RxDB-ls-attachment-' + this.databaseName + '--' + this.collectionName + '--' + this.schema.version;
+
+        this.changeStreamSub = getStorageEventStream().subscribe((ev: {
+            fromStorageEvent: boolean;
+            key: string;
+            newValue: string | null;
+            databaseInstanceToken?: string;
+        }) => {
+            if (
+                ev.key !== this.changestreamStorageKey ||
+                !ev.newValue ||
+                (
+                    ev.fromStorageEvent &&
+                    ev.databaseInstanceToken === this.databaseInstanceToken
+                )
+            ) {
+                return;
+            }
+
+            const latestChanges: ChangeStreamStoredData<RxDocType> = JSON.parse(ev.newValue);
+            if (
+                ev.fromStorageEvent &&
+                latestChanges.databaseInstanceToken === this.databaseInstanceToken
+            ) {
+                return;
+            }
+            this.changes$.next(latestChanges.eventBulk);
+        });
+    }
+
+    getDoc(docId: string | RxDocumentWriteData<RxDocType>[StringKeys<RxDocType>]): RxDocumentData<RxDocType> | undefined {
+        const docString = this.localStorage.getItem(this.docsKey + '-' + docId as string);
+        if (docString) {
+            return JSON.parse(docString);
+        }
+    }
+    setDoc(doc: RxDocumentData<RxDocType>) {
+        const docId = doc[this.primaryPath];
+        this.localStorage.setItem(this.docsKey + '-' + docId, JSON.stringify(doc));
+    }
+    getIndex(index: string[]): LocalstorageIndex {
+        const indexString = this.localStorage.getItem(this.indexesKey + getIndexName(index));
+        if (!indexString) {
+            return [];
+        } else {
+            return JSON.parse(indexString);
+        }
+    }
+    setIndex(index: string[], value: LocalstorageIndex) {
+        this.localStorage.setItem(this.indexesKey + getIndexName(index), JSON.stringify(value));
+    }
+
+
+    async bulkWrite(
+        documentWrites: BulkWriteRow<RxDocType>[],
+        context: string
+    ): Promise<RxStorageBulkWriteResponse<RxDocType>> {
+        const ret: RxStorageBulkWriteResponse<RxDocType> = {
+            error: []
+        };
+
+        /**
+         * Pre-convert all Blob attachment data to base64 BEFORE
+         * calling categorizeBulkWriteRows. localStorage has no transactions
+         * and is non-async, so all writes after the conflict check
+         * must be synchronous to avoid interleaving.
+         */
+        const attachmentBase64Map = new Map<Blob, string>();
+        await Promise.all(
+            documentWrites.map(async (row) => {
+                if (row.document._attachments) {
+                    await Promise.all(
+                        Object.values(row.document._attachments).map(async (att: any) => {
+                            if (att.data instanceof Blob) {
+                                const ab = await att.data.arrayBuffer();
+                                attachmentBase64Map.set(att.data, arrayBufferToBase64(ab));
+                            }
+                        })
+                    );
+                }
+            })
+        );
+
+        const docsInDb = new Map<RxDocumentData<RxDocType>[StringKeys<RxDocType>] | string, RxDocumentData<RxDocType>>();
+        documentWrites.forEach(row => {
+            const docId = row.document[this.primaryPath];
+            const doc = this.getDoc(docId);
+            if (doc) {
+                docsInDb.set(docId, doc);
+            }
+        });
+
+        const categorized = categorizeBulkWriteRows<RxDocType>(
+            this,
+            this.primaryPath,
+            docsInDb,
+            documentWrites,
+            context
+        );
+        ret.error = categorized.errors;
+
+
+        const indexValues = Object.values(this.internals.indexes).map(idx => {
+            return this.getIndex(idx.index);
+        });
+
+        [
+            categorized.bulkInsertDocs,
+            categorized.bulkUpdateDocs
+        ].forEach(rows => {
+            rows.forEach(row => {
+                // write new document data
+                this.setDoc(row.document);
+
+                // update the indexes
+                const docId = row.document[this.primaryPath] as string;
+                Object.values(this.internals.indexes).forEach((idx, i) => {
+                    const indexValue = indexValues[i];
+                    const newIndexString = idx.getIndexableString(row.document);
+                    const insertPosition = pushAtSortPosition<string[]>(
+                        indexValue,
+                        [
+                            newIndexString,
+                            docId,
+                        ],
+                        sortByIndexStringComparator,
+                        0
+                    );
+                    if (row.previous) {
+                        const previousIndexString = idx.getIndexableString(row.previous);
+                        if (previousIndexString === newIndexString) {
+                            /**
+                             * Performance shortcut.
+                             * If index was not changed -> The old doc must be before or after the new one.
+                             */
+                            const prev = indexValue[insertPosition - 1];
+                            if (prev && prev[1] === docId) {
+                                indexValue.splice(insertPosition - 1, 1);
+                            } else {
+                                const next = indexValue[insertPosition + 1];
+                                if (next[1] === docId) {
+                                    indexValue.splice(insertPosition + 1, 1);
+                                } else {
+                                    throw newRxError('SNH', {
+                                        document: row.document,
+                                        args: {
+                                            insertPosition,
+                                            indexValue,
+                                            row,
+                                            idx
+                                        }
+                                    });
+                                }
+                            }
+                        } else {
+                            /**
+                             * Index changed, we must search for the old one and remove it.
+                             */
+                            const indexBefore = boundEQ(
+                                indexValue,
+                                [
+                                    previousIndexString
+                                ] as any,
+                                compareDocsWithIndex
+                            );
+                            indexValue.splice(indexBefore, 1);
+                        }
+                    }
+                });
+
+            });
+        });
+
+        indexValues.forEach((indexValue, i) => {
+            const index = Object.values(this.internals.indexes);
+            this.setIndex(index[i].index, indexValue);
+        });
+
+        // attachments — use pre-converted base64 data (synchronous)
+        categorized.attachmentsAdd.forEach(attachment => {
+            const base64 = ensureNotFalsy(attachmentBase64Map.get(attachment.attachmentData.data));
+            const mimeType = attachment.attachmentData.type || 'application/octet-stream';
+            this.localStorage.setItem(
+                this.attachmentsKey +
+                '-' + attachment.documentId +
+                '||' + attachment.attachmentId,
+                'data:' + mimeType + ';base64,' + base64
+            );
+        });
+        categorized.attachmentsUpdate.forEach(attachment => {
+            const base64 = ensureNotFalsy(attachmentBase64Map.get(attachment.attachmentData.data));
+            const mimeType = attachment.attachmentData.type || 'application/octet-stream';
+            this.localStorage.setItem(
+                this.attachmentsKey +
+                '-' + attachment.documentId +
+                '||' + attachment.attachmentId,
+                'data:' + mimeType + ';base64,' + base64
+            );
+        });
+        categorized.attachmentsRemove.forEach(attachment => {
+            this.localStorage.removeItem(
+                this.attachmentsKey +
+                '-' + attachment.documentId +
+                '||' + attachment.attachmentId
+            );
+        });
+
+        if (categorized.eventBulk.events.length > 0) {
+            const lastState = ensureNotFalsy(categorized.newestRow).document;
+            categorized.eventBulk.checkpoint = {
+                id: lastState[this.primaryPath],
+                lwt: lastState._meta.lwt
+            };
+            const storageItemData: ChangeStreamStoredData<RxDocType> = {
+                databaseInstanceToken: this.databaseInstanceToken,
+                eventBulk: categorized.eventBulk
+            };
+            const itemString = JSON.stringify(storageItemData);
+            this.localStorage.setItem(
+                this.changestreamStorageKey,
+                itemString
+            );
+            storageEventStream$.next({
+                fromStorageEvent: false,
+                key: this.changestreamStorageKey,
+                newValue: itemString,
+                databaseInstanceToken: this.databaseInstanceToken
+            });
+        }
+        return Promise.resolve(ret);
+    }
+
+    async findDocumentsById(
+        docIds: string[],
+        withDeleted: boolean
+    ): Promise<RxDocumentData<RxDocType>[]> {
+        const ret: RxDocumentData<RxDocType>[] = [];
+        docIds.forEach(docId => {
+            const doc = this.getDoc(docId);
+            if (doc) {
+                if (withDeleted || !doc._deleted) {
+                    ret.push(doc);
+                }
+            }
+        });
+        return ret;
+    }
+
+    query(
+        preparedQuery: PreparedQuery<RxDocType>
+    ): Promise<RxStorageQueryResult<RxDocType>> {
+        const queryPlan = preparedQuery.queryPlan;
+        const query = preparedQuery.query;
+
+        const skip = query.skip ? query.skip : 0;
+        const limit = query.limit ? query.limit : Infinity;
+        const skipPlusLimit = skip + limit;
+
+        let queryMatcher: QueryMatcher<RxDocumentData<RxDocType>> | false = false;
+        if (!queryPlan.selectorSatisfiedByIndex) {
+            queryMatcher = getQueryMatcher(
+                this.schema,
+                preparedQuery.query
+            );
+        }
+
+        const queryPlanFields: string[] = queryPlan.index;
+        const mustManuallyResort = !queryPlan.sortSatisfiedByIndex;
+        const index: string[] | undefined = queryPlanFields;
+        const lowerBound: any[] = queryPlan.startKeys;
+        const lowerBoundString = getStartIndexStringFromLowerBound(
+            this.schema,
+            index,
+            lowerBound
+        );
+
+        let upperBound: any[] = queryPlan.endKeys;
+        upperBound = upperBound;
+        const upperBoundString = getStartIndexStringFromUpperBound(
+            this.schema,
+            index,
+            upperBound
+        );
+        const docsWithIndex = this.getIndex(index);
+        let indexOfLower = (queryPlan.inclusiveStart ? boundGE : boundGT)(
+            docsWithIndex,
+            [
+                lowerBoundString
+            ] as any,
+            compareDocsWithIndex
+        );
+
+        const indexOfUpper = (queryPlan.inclusiveEnd ? boundLE : boundLT)(
+            docsWithIndex,
+            [
+                upperBoundString
+            ] as any,
+            compareDocsWithIndex
+        );
+
+        let rows: RxDocumentData<RxDocType>[] = [];
+        let done = false;
+        while (!done) {
+            const currentRow = docsWithIndex[indexOfLower];
+            if (
+                !currentRow ||
+                indexOfLower > indexOfUpper
+            ) {
+                break;
+            }
+            const docId = currentRow[1];
+            const currentDoc = ensureNotFalsy(this.getDoc(docId));
+
+            if (!queryMatcher || queryMatcher(currentDoc)) {
+                rows.push(currentDoc);
+            }
+
+            if (
+                (rows.length >= skipPlusLimit && !mustManuallyResort)
+            ) {
+                done = true;
+            }
+
+            indexOfLower++;
+        }
+
+        if (mustManuallyResort) {
+            const sortComparator = getSortComparator(this.schema, preparedQuery.query);
+            rows = rows.sort(sortComparator);
+        }
+
+        // apply skip and limit boundaries.
+        rows = rows.slice(skip, skipPlusLimit);
+        return Promise.resolve({
+            documents: rows
+        });
+    }
+
+    async count(
+        preparedQuery: PreparedQuery<RxDocType>
+    ): Promise<RxStorageCountResult> {
+        const result = await this.query(preparedQuery);
+        return {
+            count: result.documents.length,
+            mode: 'fast'
+        };
+    }
+
+    changeStream(): Observable<EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, RxStorageDefaultCheckpoint>> {
+        return this.changes$.asObservable();
+    }
+    cleanup(minimumDeletedTime: number): Promise<boolean> {
+        const maxDeletionTime = now() - minimumDeletedTime;
+        const indexValue = this.getIndex(CLEANUP_INDEX);
+        const lowerBoundString = getStartIndexStringFromLowerBound(
+            this.schema,
+            CLEANUP_INDEX,
+            [
+                true,
+                0,
+                ''
+            ]
+        );
+        let indexOfLower = boundGT(
+            indexValue,
+            [
+                lowerBoundString
+            ] as any,
+            compareDocsWithIndex
+        );
+
+        const indexValues = Object.values(this.internals.indexes).map(idx => {
+            return this.getIndex(idx.index);
+        });
+
+        let done = false;
+        while (!done) {
+            const currentIndexRow = indexValue[indexOfLower];
+            if (!currentIndexRow) {
+                break;
+            }
+            const currentDocId = currentIndexRow[1];
+            const currentDoc = ensureNotFalsy(this.getDoc(currentDocId));
+            if (currentDoc._meta.lwt > maxDeletionTime) {
+                done = true;
+            } else {
+                this.localStorage.removeItem(this.docsKey + '-' + currentDocId);
+                Object.values(this.internals.indexes).forEach((idx, i) => {
+                    const indexValue = indexValues[i];
+                    const indexString = idx.getIndexableString(currentDoc);
+                    const indexBefore = boundEQ(
+                        indexValue,
+                        [
+                            indexString
+                        ] as any,
+                        compareDocsWithIndex
+                    );
+                    indexValue.splice(indexBefore, 1);
+                });
+                indexOfLower++;
+            }
+        }
+
+        indexValues.forEach((indexValue, i) => {
+            const index = Object.values(this.internals.indexes);
+            this.setIndex(index[i].index, indexValue);
+        });
+
+        return PROMISE_RESOLVE_TRUE;
+    }
+
+    async getAttachmentData(documentId: string, attachmentId: string): Promise<Blob> {
+        const stored = ensureNotFalsy(
+            this.localStorage.getItem(this.attachmentsKey + '-' + documentId + '||' + attachmentId)
+        );
+        // Stored value is a data URL like "data:<mime>;base64,<payload>".
+        // Fall back to application/octet-stream for legacy bare base64 payloads.
+        let dataUrl = stored;
+        if (!stored.startsWith('data:')) {
+            dataUrl = 'data:application/octet-stream;base64,' + stored;
+        }
+        const response = await fetch(dataUrl);
+        return response.blob();
+    }
+
+    remove(): Promise<void> {
+        ensureNotRemoved(this);
+        this.removed = true;
+
+        // delete changes
+        this.changeStreamSub.unsubscribe();
+        this.localStorage.removeItem(this.changestreamStorageKey);
+
+        // delete documents
+        const firstIndex = Object.values(this.internals.indexes)[0];
+        const indexedDocs = this.getIndex(firstIndex.index);
+        indexedDocs.forEach(row => {
+            const docId = row[1];
+            this.localStorage.removeItem(this.docsKey + '-' + docId);
+        });
+
+        // delete indexes
+        Object.values(this.internals.indexes).forEach(idx => {
+            this.localStorage.removeItem(this.indexesKey + idx.indexName);
+        });
+
+        return PROMISE_RESOLVE_VOID;
+    }
+
+    close(): Promise<void> {
+        this.changeStreamSub.unsubscribe();
+        this.removed = true;
+
+        if (this.closed) {
+            return this.closed;
+        }
+        this.closed = (async () => {
+            this.changes$.complete();
+            this.localStorage.removeItem(this.changestreamStorageKey);
+        })();
+        return this.closed;
+    }
+}
+
+export async function createLocalstorageStorageInstance<RxDocType>(
+    storage: RxStorageLocalstorage,
+    params: RxStorageInstanceCreationParams<RxDocType, LocalstorageInstanceCreationOptions>,
+    settings: LocalstorageStorageSettings
+): Promise<RxStorageInstanceLocalstorage<RxDocType>> {
+    const primaryPath = getPrimaryFieldOfPrimaryKey(params.schema.primaryKey);
+
+    const useIndexes = params.schema.indexes ? params.schema.indexes.slice(0) : [];
+    useIndexes.push([primaryPath]);
+    const useIndexesFinal = useIndexes.map(index => {
+        const indexAr = toArray(index);
+        return indexAr;
+    });
+    useIndexesFinal.push(CLEANUP_INDEX);
+    const indexes: ById<IndexMeta<RxDocType>> = {};
+    useIndexesFinal.forEach((indexAr, indexId) => {
+        const indexName = getIndexName(indexAr);
+        indexes[indexName] = {
+            indexId: '|' + indexId + '|',
+            indexName,
+            getIndexableString: getIndexableStringMonad(params.schema, indexAr),
+            index: indexAr
+        };
+    });
+
+    const internals: LocalstorageStorageInternals<RxDocType> = {
+        indexes
+    };
+
+    const instance = new RxStorageInstanceLocalstorage(
+        storage,
+        params.databaseName,
+        params.collectionName,
+        params.schema,
+        internals,
+        params.options,
+        settings,
+        params.multiInstance,
+        params.databaseInstanceToken
+    );
+    return instance;
+}
+
+
+export function getIndexName(index: string[]): string {
+    return index.join('|');
+}
+export const CLEANUP_INDEX: string[] = ['_deleted', '_meta.lwt'];
+export type IndexMeta<RxDocType> = {
+    indexId: string;
+    indexName: string;
+    index: string[];
+    getIndexableString: (doc: RxDocumentData<RxDocType>) => string;
+};
+
+function sortByIndexStringComparator(a: [string, string], b: [string, string]) {
+    if (a[0] < b[0]) {
+        return -1;
+    } else {
+        return 1;
+    }
+}
+
+function compareDocsWithIndex<RxDocType>(
+    a: [string, string],
+    b: [string, string]
+): 1 | 0 | -1 {
+    const indexStringA = a[0];
+    const indexStringB = b[0];
+    if (indexStringA < indexStringB) {
+        return -1;
+    } else if (indexStringA === indexStringB) {
+        return 0;
+    } else {
+        return 1;
+    }
+}
+
+function ensureNotRemoved(
+    instance: RxStorageInstanceLocalstorage<any>
+) {
+    if (instance.removed) {
+        throw new Error('removed');
+    }
+}