core-3nweb-client-lib 0.28.4 → 0.29.0

package/build/core/asmail/inbox/msg-indexing/sql-indexing.js

@@ -0,0 +1,320 @@
+"use strict";
+/*
+ Copyright (C) 2022 - 2023 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeSqliteBasedIndexedRecords = void 0;
+const timed_cache_1 = require("../../../../lib-common/timed-cache");
+const lib_sqlite_on_3nstorage_1 = require("../../../../lib-sqlite-on-3nstorage");
+const file_1 = require("../../../../lib-common/exceptions/file");
+const fs_sync_utils_1 = require("../../../../lib-client/fs-sync-utils");
+const operators_1 = require("rxjs/operators");
+const tab = new lib_sqlite_on_3nstorage_1.TableColumnsAndParams('inbox_index', {
+    msgId: ['msg_id', 'TEXT PRIMARY KEY'],
+    msgType: ['msg_type', 'TEXT'],
+    deliveryTS: ['delivery_ts', 'INTEGER'],
+    key: ['key', 'BLOB'],
+    keyStatus: ['key_status', 'TEXT'],
+    mainObjHeaderOfs: ['main_obj_header_ofs', 'INTEGER'],
+    removeAfter: ['remove_after', 'INTEGER DEFAULT 0']
+});
+const createIndexTab = `CREATE TABLE ${tab.name} (
+	${tab.columnsCreateSection}
+) STRICT`;
+const selectAllMsgInfos = `SELECT ${tab.colsSection('msgId', 'msgType', 'deliveryTS')}
+FROM ${tab.name}`;
+const selectMsgInfosFromTS = `SELECT ${tab.colsSection('msgId', 'msgType', 'deliveryTS')}
+FROM ${tab.name}
+WHERE ${tab.c['deliveryTS']}>$fromTS`;
+function listMsgInfos(db, fromTS) {
+    let result;
+    if (fromTS) {
+        result = db.exec(selectMsgInfosFromTS, { '$fromTS': fromTS });
+    }
+    else {
+        result = db.exec(selectAllMsgInfos);
+    }
+    return tab.fromQueryExecResult(result);
+}
+const deleteRec = `DELETE FROM ${tab.name}
+WHERE ${tab.colsEqualSection('msgId')}`;
+function deleteMsgFrom(db, msgId) {
+    db.exec(deleteRec, tab.toParams({ msgId }));
+}
+const selectMsgDeliveryTS = `SELECT ${tab.colsSection('deliveryTS')}
+FROM ${tab.name}
+WHERE ${tab.colsEqualSection('msgId')}`;
+function findMsgAndGetDeliveryTS(db, msgId) {
+    const result = db.exec(selectMsgDeliveryTS, tab.toParams({ msgId }));
+    const values = tab.fromQueryExecResult(result);
+    return ((values.length > 0) ? values[0].deliveryTS : undefined);
+}
+const selectMsgKey = `SELECT ${tab.colsSection('key', 'keyStatus', 'mainObjHeaderOfs')}
+FROM ${tab.name}
+WHERE ${tab.colsEqualSection('msgId')}`;
+function findMsgKey(db, msgId) {
+    const result = db.exec(selectMsgKey, tab.toParams({ msgId }));
+    if (result.length > 0) {
+        const { key: msgKey, keyStatus: msgKeyRole, mainObjHeaderOfs } = tab.fromQueryExecResult(result)[0];
+        return { msgKey, msgKeyRole, mainObjHeaderOfs };
+    }
+    else {
+        return;
+    }
+}
+const selectMsgPresence = `SELECT ${tab.colsSection('msgId')}
+FROM ${tab.name}
+WHERE ${tab.colsEqualSection('msgId')}`;
+function isMsgPresent(db, msgId) {
+    const result = db.exec(selectMsgPresence, tab.toParams({ msgId }));
+    return (result.length > 0);
+}
+const LIMIT_RECORDS_PER_FILE = 200;
+class RecordsInSQL {
+    constructor(files, latest, fileTSs) {
+        this.files = files;
+        this.latest = latest;
+        this.fileTSs = fileTSs;
+        this.older = (0, timed_cache_1.makeTimedCache)(10 * 60 * 1000);
+        Object.seal(this);
+    }
+    async add(msgInfo, decrInfo, removeAfter) {
+        const { msgId, msgType, deliveryTS } = msgInfo;
+        const { key, keyStatus, msgKeyPackLen: mainObjHeaderOfs } = decrInfo;
+        const { db } = await this.getDbFor(msgInfo.deliveryTS);
+        db.db.exec(tab.insertQuery, tab.toParams({
+            msgId, msgType, deliveryTS,
+            key, keyStatus,
+            mainObjHeaderOfs, removeAfter
+        }));
+        await db.saveToFile();
+    }
+    async saveLatestWithAttr() {
+        // XXX
+    }
+    async getDbFor(deliveryTS) {
+        if ((this.fileTSs.length === 0)
+            || (this.fileTSs[this.fileTSs.length - 1] < deliveryTS)) {
+            return { db: this.latest };
+        }
+        let fileTS = this.fileTSs[this.fileTSs.length - 1];
+        for (let i = (this.fileTSs.length - 2); i >= 0; i -= 1) {
+            if (this.fileTSs[i] >= deliveryTS) {
+                fileTS = this.fileTSs[i];
+            }
+            else {
+                break;
+            }
+        }
+        const db = await this.dbFromCacheOrInit(fileTS);
+        return { db, fileTS };
+    }
+    async dbFromCacheOrInit(fileTS) {
+        let db = this.older.get(fileTS);
+        if (db) {
+            return db;
+        }
+        const dbFile = await this.files.getDBFile(fileTS);
+        db = await lib_sqlite_on_3nstorage_1.SQLiteOn3NStorage.makeAndStart(dbFile);
+        this.older.set(fileTS, db);
+        return db;
+    }
+    async remove(msgId) {
+        for await (const { db, fileTS } of this.iterateDBs()) {
+            const deliveryTS = findMsgAndGetDeliveryTS(db.db, msgId);
+            if (deliveryTS) {
+                deleteMsgFrom(db.db, msgId);
+                await db.saveToFile();
+                return deliveryTS;
+            }
+        }
+        return undefined;
+    }
+    async *iterateDBs() {
+        yield { db: this.latest };
+        for (let i = (this.fileTSs.length - 1); i >= 0; i = -1) {
+            const fileTS = this.fileTSs[i];
+            if (!fileTS) {
+                continue;
+            }
+            const db = await this.dbFromCacheOrInit(fileTS);
+            if (db) {
+                yield { db, fileTS };
+            }
+        }
+    }
+    async listMsgs(fromTS) {
+        let lst = listMsgInfos(this.latest.db, fromTS);
+        for (let i = (this.fileTSs.length - 1); i >= 0; i -= 1) {
+            const fileTS = this.fileTSs[i];
+            if (fromTS && (fileTS <= fromTS)) {
+                break;
+            }
+            const older = await this.dbFromCacheOrInit(fileTS);
+            lst = listMsgInfos(older.db, fromTS).concat(lst);
+        }
+        lst.sort((a, b) => (a.deliveryTS - b.deliveryTS));
+        return lst;
+    }
+    async getIndexWith(deliveryTS) {
+        let fileTS = undefined;
+        for (let i = (this.fileTSs.length - 1); i >= 0; i -= 1) {
+            const fTS = this.fileTSs[i];
+            if (fTS < deliveryTS) {
+                break;
+            }
+            fileTS = fTS;
+        }
+        if (fileTS) {
+            return await this.dbFromCacheOrInit(fileTS);
+        }
+        else {
+            return this.latest;
+        }
+    }
+    async getKeyFor(msgId, deliveryTS) {
+        const db = await this.getIndexWith(deliveryTS);
+        return findMsgKey(db.db, msgId);
+    }
+    async msgExists({ msgId, deliveryTS }) {
+        const db = await this.getIndexWith(deliveryTS);
+        return isMsgPresent(db.db, msgId);
+    }
+}
+Object.freeze(RecordsInSQL.prototype);
+Object.freeze(RecordsInSQL);
+const DB_EXT = '.sqlite';
+const LATEST_DB = `latest${DB_EXT}`;
+function dbFileName(fileTS) {
+    return (fileTS ? LATEST_DB : `${fileTS}${DB_EXT}`);
+}
+class SqliteFiles {
+    constructor(dbsFS) {
+        this.dbsFS = dbsFS;
+        this.syncing = undefined;
+        Object.seal(this);
+    }
+    static async makeAndStart(syncedFS) {
+        (0, file_1.ensureCorrectFS)(syncedFS, 'synced', true);
+        const files = new SqliteFiles(syncedFS);
+        const records = await files.makeRecords();
+        files.startSyncing();
+        return { files, records };
+    }
+    async makeRecords() {
+        const latest = await this.readOrInitializeLatestDB();
+        const fileTSs = await this.fileTSsOfDBShards();
+        return new RecordsInSQL(this, latest, fileTSs);
+    }
+    async getDBFile(fileTS) {
+        return await this.dbsFS.writableFile(dbFileName(fileTS), { create: false });
+    }
+    async readOrInitializeLatestDB() {
+        if (await this.dbsFS.checkFilePresence(LATEST_DB)) {
+            const dbFile = await this.dbsFS.writableFile(LATEST_DB, { create: false });
+            return await lib_sqlite_on_3nstorage_1.SQLiteOn3NStorage.makeAndStart(dbFile);
+        }
+        else {
+            const dbFile = await this.dbsFS.writableFile(LATEST_DB, { create: true, exclusive: true });
+            const latest = await lib_sqlite_on_3nstorage_1.SQLiteOn3NStorage.makeAndStart(dbFile);
+            latest.db.run(createIndexTab);
+            await latest.saveToFile();
+            return latest;
+        }
+    }
+    async fileTSsOfDBShards() {
+        const lst = await this.dbsFS.listFolder('');
+        const fileTSs = [];
+        for (const { isFile, name } of lst) {
+            if (!isFile || !name.endsWith(DB_EXT)) {
+                continue;
+            }
+            const numStr = name.substring(0, DB_EXT.length);
+            const fileTS = parseInt(numStr);
+            if (isNaN(fileTS)) {
+                continue;
+            }
+            fileTSs.push(fileTS);
+        }
+        fileTSs.sort();
+        return fileTSs;
+    }
+    startSyncing() {
+        const db$ = (0, fs_sync_utils_1.observableFromTreeEvents)(this.dbsFS, '');
+        // XXX
+        // - start from data in fs, attempt to get fresh, etc. Both log and data
+        //   (in parallel ?).
+        // - start sync process
+        // - unblock processing, as init is done
+        // Write theses in functions that use RecordsInSQL and LogOfChanges
+        // structures.
+        // Somehow aforementioned processes ain't exclusive to either point.
+        // XXX
+        //  should start from reading folder and placing logs into yet unsynced db
+        this.syncing = db$
+            .pipe((0, operators_1.filter)(ev => ev.type.startsWith('remote-')))
+            // .subscribe({
+            // 	next: ev => console.log(`------ fs event:`, ev),
+            // 	complete: () => console.log(` +++ MsgIndex's sync process completed`),
+            // 	error: err => console.log(` *** error in MsgIndex's sync process`, err)
+            // });
+            .subscribe();
+    }
+    stopSyncing() {
+        if (this.syncing) {
+            this.syncing.unsubscribe();
+            this.syncing = undefined;
+        }
+    }
+}
+Object.freeze(SqliteFiles.prototype);
+Object.freeze(SqliteFiles);
+class SqliteBasedIndexedRecords {
+    constructor(files, records) {
+        this.files = files;
+        this.records = records;
+        Object.seal(this);
+    }
+    static async makeAndStart(syncedFS) {
+        const { files, records } = await SqliteFiles.makeAndStart(syncedFS);
+        return new SqliteBasedIndexedRecords(files, records);
+    }
+    async add(msgInfo, decrInfo, removeAfter) {
+        await this.records.add(msgInfo, decrInfo, removeAfter);
+    }
+    remove(msgId) {
+        return this.records.remove(msgId);
+    }
+    listMsgs(fromTS) {
+        return this.records.listMsgs(fromTS);
+    }
+    getKeyFor(msgId, deliveryTS) {
+        return this.records.getKeyFor(msgId, deliveryTS);
+    }
+    async msgExists({ msgId, deliveryTS }) {
+        return !!(await this.records.getKeyFor(msgId, deliveryTS));
+    }
+    stopSyncing() {
+        this.files.stopSyncing();
+    }
+}
+Object.freeze(SqliteBasedIndexedRecords.prototype);
+Object.freeze(SqliteBasedIndexedRecords);
+function makeSqliteBasedIndexedRecords(syncedFS) {
+    return SqliteBasedIndexedRecords.makeAndStart(syncedFS);
+}
+exports.makeSqliteBasedIndexedRecords = makeSqliteBasedIndexedRecords;
+Object.freeze(exports);