@connecttomahdi/rxdb 17.0.0-beta.17

package/scripts/transpile.mjs

@@ -0,0 +1,229 @@
+process.env['NODE_ENV'] = 'es5';
+
+console.log('# transpiling.. (this takes some time on first run)');
+import events from 'events';
+events.EventEmitter.defaultMaxListeners = 0;
+
+/**
+ * runs the babel-transpile
+ * remembers mtime of files and only transpiles the changed ones
+ */
+import nconf from 'nconf';
+import path from 'node:path';
+import fs from 'node:fs';
+import os from 'node:os';
+import walkSync from 'walk-sync';
+import shell from 'shelljs';
+import existsFile from 'exists-file';
+import { fileURLToPath } from 'node:url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const basePath = path.join(__dirname, '..');
+
+const confLocation = path.join(basePath, '.transpile_state.json');
+const cpuCount = os.cpus().length;
+
+const DEBUG = false;
+
+/**
+ * key->value | src -> compiled
+ */
+const TRANSPILE_FOLDERS = [
+    {
+        source: 'src',
+        goals: {
+            es5: 'dist/cjs',
+            es6: 'dist/esm'
+        }
+    },
+    {
+        source: 'test',
+        goals: {
+            es6: 'test_tmp'
+        }
+    }
+];
+
+nconf.argv()
+    .env()
+    .file({
+        file: confLocation
+    });
+
+
+
+async function transpileFile(
+    srcLocations,
+    outDir,
+    env
+) {
+    if (DEBUG) {
+        console.log('transpile: ' + srcLocations.join(', '));
+    }
+    // ensure folder exists
+    const folder = path.join(outDir);
+    await fs.promises.mkdir(
+        folder,
+        {
+            recursive: true,
+        }
+    ).catch(err => {
+        console.error('# transpile.mjs: could not create directory: ' + folder, err);
+    });
+
+    // const outFilePath =
+    // await del.promise([outDir]);
+    const cmd = 'cross-env NODE_ENV=' + env +
+        ' babel ' +
+        srcLocations.join(' ') +
+        ' --source-maps' +
+        ' --extensions ".ts,.tsx,.js"' +
+        ' --out-dir ' +
+        outDir;
+
+    if (DEBUG) {
+        console.dir(cmd);
+    }
+
+    const execRes = shell.exec(cmd, {
+        async: true
+    });
+    await new Promise(res => execRes.on('exit', res));
+
+    const exitCode = execRes.exitCode;
+    if (exitCode !== 0) {
+        console.error('transpiling failed with cmd: ' + cmd);
+        process.exit(1);
+    }
+
+    if (DEBUG) {
+        console.log('transpiled files: ' + srcLocations.join(', '));
+    }
+
+    return;
+}
+
+async function getFiles() {
+    const files = [];
+    await Promise.all(
+        TRANSPILE_FOLDERS
+            // make all file paths absolute
+            .map(transpileFolder => {
+                transpileFolder.source = path.join(basePath, transpileFolder.source);
+                Object.entries(transpileFolder.goals).forEach(([key, value]) => {
+                    transpileFolder.goals[key] = path.join(basePath, value);
+                });
+                return transpileFolder;
+            })
+            .map(transpileFolder => {
+                const srcFolder = transpileFolder.source;
+                return walkSync.entries(srcFolder)
+                    .filter(entry => !entry.isDirectory())
+                    .filter(entry => entry.relativePath.endsWith('.js') || entry.relativePath.endsWith('.ts') || entry.relativePath.endsWith('.tsx'))
+                    .filter(entry => !entry.relativePath.includes('/node_modules/'))
+                    .map(fileEntry => {
+                        // ensure goal-file-ending is .js
+                        const relativePathSplit = fileEntry.relativePath.split('.');
+                        relativePathSplit.pop();
+                        relativePathSplit.push('js');
+                        Object.entries(transpileFolder.goals).forEach(([env, toFolder]) => {
+                            const goalPath = path.join(toFolder, relativePathSplit.join('.'));
+                            const fullPath = path.join(fileEntry.basePath, fileEntry.relativePath);
+
+                            const lastTime = parseInt(nconf.get(fileEntry.fullPath), 10);
+                            const file = {
+                                env,
+                                fullPath,
+                                relativePath: fileEntry.relativePath,
+                                basePath: fileEntry.basePath,
+                                mtime: fileEntry.mtime,
+                                goalFolder: path.dirname(goalPath),
+                                goalPath: goalPath,
+                            };
+                            if (
+                                lastTime !== fileEntry.mtime ||
+                                !existsFile.sync(goalPath)
+                            ) {
+                                files.push(file);
+                            }
+                        });
+                    });
+            })
+    );
+
+    const filesByGoalFolder = {};
+    files.forEach(file => {
+        if (!filesByGoalFolder[file.goalFolder]) {
+            filesByGoalFolder[file.goalFolder] = [];
+        }
+        filesByGoalFolder[file.goalFolder].push(file);
+    });
+
+    if (DEBUG) {
+        console.dir(filesByGoalFolder);
+    }
+
+    return filesByGoalFolder;
+}
+
+async function run() {
+    const files = await getFiles();
+
+
+    let totalFiles = 0;
+    Object.values(files).forEach(f => {
+ totalFiles += f.length;
+});
+
+    if (totalFiles > 30) {
+        console.log('# Many files changed (' + totalFiles + '), running full build via npm scripts...');
+        const cmd = 'concurrently "npm run build:cjs" "npm run build:esm" "npm run build:test"';
+        const execRes = shell.exec(cmd, { async: true });
+        await new Promise(res => execRes.on('exit', res));
+        if (execRes.exitCode !== 0) {
+            console.error('transpiling failed with cmd: ' + cmd);
+            process.exit(1);
+        }
+        Object.values(files).forEach(filesWithSameGoalFolder => {
+            filesWithSameGoalFolder.forEach(file => {
+                nconf.set(file.fullPath, file.mtime);
+            });
+        });
+    } else {
+        await Promise.all(
+            Object.values(files).map(async (filesWithSameGoalFolder) => {
+
+                const byEnv = {};
+                filesWithSameGoalFolder.forEach(row => {
+                    const env = row.env;
+                    if (!byEnv[env]) {
+                        byEnv[env] = [];
+                    }
+                    const ar = byEnv[env];
+                    ar.push(row);
+                });
+
+                await Promise.all(
+                    Object.entries(byEnv)
+                        .map(async ([env, innerFiles]) => {
+                            await transpileFile(
+                                innerFiles.map(file => path.join(file.basePath, file.relativePath)),
+                                innerFiles[0].goalFolder,
+                                env
+                            );
+                            innerFiles.forEach(file => nconf.set(file.fullPath, file.mtime));
+                        })
+                );
+            })
+        );
+    }
+
+    nconf.save(function () {
+        if (DEBUG) {
+            console.log('conf saved');
+        }
+        console.log('# transpiling DONE (' + cpuCount + ' CPUs)');
+    });
+}
+run();
+

package/scripts/update-version-variable.mjs

@@ -0,0 +1,49 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const rootPath = path.join(
+    __dirname,
+    '../'
+);
+
+async function run() {
+
+    // update version in package.json
+    const packageJson = JSON.parse(
+        await fs.promises.readFile(
+            path.join(rootPath, 'package.json')
+        )
+    );
+    const newVersion = packageJson.version;
+    if (
+        !newVersion ||
+        typeof newVersion !== 'string'
+    ) {
+        throw new Error('got no version');
+    }
+
+    const templateContent = await fs.promises.readFile(
+        path.join(rootPath, 'src/plugins/utils/utils-rxdb-version.template.ts'),
+        'utf-8'
+    );
+    const placeholderFlag = '|PLACEHOLDER|';
+    if (!templateContent.includes(placeholderFlag)) {
+        throw new Error('placeholder not found');
+    }
+    const versionFileContent = templateContent.replace(placeholderFlag, newVersion);
+    const goalFilePath = path.join(rootPath, 'src/plugins/utils/utils-rxdb-version.ts');
+    await fs.promises.writeFile(
+        goalFilePath,
+        versionFileContent,
+        {
+            encoding: 'utf8',
+            // overwrite if exists
+            flag: 'w'
+        }
+    );
+}
+
+run();

package/src/change-event-buffer.ts

@@ -0,0 +1,171 @@
+/**
+ * a buffer-cache which holds the last X changeEvents of the collection
+ */
+import {
+    Subscription
+} from 'rxjs';
+import { filter } from 'rxjs/operators';
+import type {
+    RxChangeEventBulk,
+    RxCollection,
+    RxStorageChangeEvent
+} from './types/index.d.ts';
+import {
+    requestIdlePromiseNoQueue
+} from './plugins/utils/index.ts';
+
+
+/**
+ * This buffer rembemers previous change events
+ * so that queries can use them on .exec()
+ * to calculate the new result set via event-reduce instead
+ * of running the query against the storage.
+ */
+export class ChangeEventBuffer<RxDocType> {
+    /**
+     * These properties are private to ensure they cannot
+     * be read without first processing the lazy tasks.
+     */
+    private subs: Subscription[] = [];
+    private counter: number = 0;
+    private eventCounterMap: WeakMap<
+        RxStorageChangeEvent<RxDocType>,
+        number
+    > = new WeakMap();
+    /**
+     * array with changeEvents
+     * starts with oldest known event, ends with newest
+    */
+    private buffer: RxStorageChangeEvent<RxDocType>[] = [];
+
+    public limit: number = 100;
+
+
+
+    private tasks = new Set<Function>();
+
+    constructor(
+        public collection: RxCollection
+    ) {
+        this.subs.push(
+            this.collection.eventBulks$.pipe(
+                filter((bulk: RxChangeEventBulk<RxDocType>) => !bulk.isLocal)
+            ).subscribe((eventBulk: RxChangeEventBulk<RxDocType>) => {
+                this.tasks.add(() => this._handleChangeEvents(eventBulk.events));
+                if (this.tasks.size <= 1) {
+                    requestIdlePromiseNoQueue().then(() => {
+                        this.processTasks();
+                    });
+                }
+            })
+        );
+    }
+
+    private processTasks() {
+        if (this.tasks.size === 0) {
+            return;
+        }
+        const tasks = Array.from(this.tasks);
+        tasks.forEach(task => task());
+        this.tasks.clear();
+    }
+
+    private _handleChangeEvents(events: RxStorageChangeEvent<RxDocType>[]) {
+        const counterBefore = this.counter;
+        this.counter = this.counter + events.length;
+        if (events.length > this.limit) {
+            this.buffer = events.slice(events.length * -1);
+        } else {
+            this.buffer = this.buffer.concat(events);
+            this.buffer = this.buffer.slice(this.limit * -1);
+        }
+        const counterBase = counterBefore + 1;
+        const eventCounterMap = this.eventCounterMap;
+        for (let index = 0; index < events.length; index++) {
+            const event = events[index];
+            eventCounterMap.set(event, counterBase + index);
+        }
+    }
+
+    getCounter() {
+        this.processTasks();
+        return this.counter;
+    }
+    getBuffer() {
+        this.processTasks();
+        return this.buffer;
+    }
+
+    /**
+     * gets the array-index for the given pointer
+     * @return arrayIndex which can be used to iterate from there. If null, pointer is out of lower bound
+     */
+    getArrayIndexByPointer(pointer: number): number | null {
+        this.processTasks();
+        const oldestEvent = this.buffer[0];
+        const oldestCounter = this.eventCounterMap.get(
+            oldestEvent
+        ) as number;
+
+        if (pointer < oldestCounter)
+            return null; // out of bounds
+
+        const rest = pointer - oldestCounter;
+        return rest;
+    }
+
+    /**
+     * get all changeEvents which came in later than the pointer-event
+     * @return array with change-events. If null, pointer out of bounds
+     */
+    getFrom(pointer: number): RxStorageChangeEvent<RxDocType>[] | null {
+        this.processTasks();
+        const ret = [];
+        let currentIndex = this.getArrayIndexByPointer(pointer);
+        if (currentIndex === null) // out of bounds
+            return null;
+
+        while (true) {
+            const nextEvent = this.buffer[currentIndex];
+            currentIndex++;
+            if (!nextEvent) {
+                return ret;
+            } else {
+                ret.push(nextEvent);
+            }
+        }
+    }
+
+    runFrom(pointer: number, fn: Function) {
+        this.processTasks();
+        const ret = this.getFrom(pointer);
+        if (ret === null) {
+            throw new Error('out of bounds');
+        } else {
+            ret.forEach(cE => fn(cE));
+        }
+    }
+
+    /**
+     * no matter how many operations are done on one document,
+     * only the last operation has to be checked to calculate the new state
+     * this function reduces the events to the last ChangeEvent of each doc.
+     * This functionality is currently disabled. It is questionable if
+     * pre-merging the events would really be faster or actually slower.
+     */
+    reduceByLastOfDoc(changeEvents: RxStorageChangeEvent<RxDocType>[]): RxStorageChangeEvent<RxDocType>[] {
+        this.processTasks();
+        return changeEvents.slice(0);
+    }
+
+    close() {
+        this.tasks.clear();
+        this.subs.forEach(sub => sub.unsubscribe());
+    }
+}
+
+export function createChangeEventBuffer<RxdocType>(
+    collection: RxCollection<RxdocType, any>
+) {
+    return new ChangeEventBuffer<RxdocType>(collection);
+}