core-3nweb-client-lib 0.27.4 → 0.27.6

package/build/core/asmail/inbox/msg-indexing.js

@@ -1,6 +1,6 @@
 "use strict";
 /*
- Copyright (C) 2016 - 2020 3NSoft Inc.
+ Copyright (C) 2022 3NSoft Inc.
  
  This program is free software: you can redistribute it and/or modify it under
  the terms of the GNU General Public License as published by the Free Software
@@ -17,361 +17,490 @@
 */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MsgIndex = void 0;
-const synced_1 = require("../../../lib-common/processes/synced");
-const buffer_utils_1 = require("../../../lib-common/buffer-utils");
 const timed_cache_1 = require("../../../lib-common/timed-cache");
-const LIMIT_RECORDS_PER_FILE = 200;
-const LATEST_INDEX = 'latest.json';
-const INDEX_EXT = '.json';
-const INDEX_FNAME_REGEXP = /^\d+\.json$/;
-// XXX request to 'undefined.json' was spotted, and it probably comes from here.
-//		Either ensure that it doesn't happen here, or find it and fix it.
-function fileTSOrderComparator(a, b) {
-    return (a - b);
-}
-function insertInto(records, rec) {
-    records.byId.set(rec.msgId, rec);
-    if (records.ordered.length === 0) {
-        records.ordered.push(rec);
+const lib_sqlite_on_3nstorage_1 = require("../../../lib-sqlite-on-3nstorage");
+const file_1 = require("../../../lib-common/exceptions/file");
+const buffer_utils_1 = require("../../../lib-common/buffer-utils");
+const fs_sync_utils_1 = require("../../../lib-client/fs-sync-utils");
+const rxjs_1 = require("rxjs");
+const operators_1 = require("rxjs/operators");
+const indexTab = 'inbox_index';
+const column = {
+    msgId: 'msg_id',
+    msgType: 'msg_type',
+    deliveryTS: 'delivery_ts',
+    key: 'msg_key',
+    keyStatus: 'key_status',
+    mainObjHeaderOfs: 'main_obj_header_ofs',
+    removeAfter: 'remove_after'
+};
+Object.freeze(column);
+const createIndexTab = `CREATE TABLE ${indexTab} (
+	${column.msgId} TEXT PRIMARY KEY,
+	${column.msgType} TEXT,
+	${column.deliveryTS} INTEGER,
+	${column.key} BLOB,
+	${column.keyStatus} TEXT,
+	${column.mainObjHeaderOfs} INTEGER,
+	${column.removeAfter} INTEGER DEFAULT 0
+) STRICT`;
+const insertRec = `INSERT INTO ${indexTab} (
+	${column.msgId}, ${column.msgType}, ${column.deliveryTS},
+	${column.key}, ${column.keyStatus}, ${column.mainObjHeaderOfs},
+	${column.removeAfter}
+) VALUES (
+	$${column.msgId}, $${column.msgType}, $${column.deliveryTS},
+	$${column.key}, $${column.keyStatus}, $${column.mainObjHeaderOfs},
+	$${column.removeAfter}
+)`;
+const deleteRec = `DELETE FROM ${indexTab}
+WHERE ${column.msgId}=$${column.msgId}`;
+function listMsgInfos(db, fromTS) {
+    let result;
+    if (fromTS) {
+        result = db.exec(`SELECT ${column.msgId}, ${column.msgType}, ${column.deliveryTS} FROM ${indexTab} WHERE ${column.deliveryTS}>$fromTS`, { '$fromTS': fromTS });
     }
     else {
-        const ts = rec.deliveryTS;
-        for (let i = (records.ordered.length - 1); i >= 0; i -= 1) {
-            if (records.ordered[i].deliveryTS <= ts) {
-                records.ordered.splice(i + 1, 0, rec);
-                return;
-            }
-        }
-        records.ordered.splice(0, 0, rec);
+        result = db.exec(`SELECT ${column.msgId}, ${column.msgType}, ${column.deliveryTS} FROM ${indexTab}`);
     }
-}
-function removeFrom(records, msgId) {
-    records.byId.delete(msgId);
-    for (let i = 0; i < records.ordered.length; i += 1) {
-        if (records.ordered[i].msgId === msgId) {
-            records.ordered.splice(i, 1);
-            break;
-        }
+    if (result.length === 0) {
+        return [];
     }
+    const { columns, values: rows } = result[0];
+    const indecies = columnIndecies(columns, column.msgId, column.msgType, column.deliveryTS);
+    const msgs = [];
+    for (const row of rows) {
+        msgs.push({
+            msgId: row[indecies.get(column.msgId)],
+            msgType: row[indecies.get(column.msgType)],
+            deliveryTS: row[indecies.get(column.deliveryTS)]
+        });
+    }
+    return msgs;
 }
-function extractEarlyRecords(records, recToExtract) {
-    if (records.ordered.length < recToExtract) {
-        throw new Error('Given too few records.');
-    }
-    const byId = new Map();
-    const ordered = records.ordered.splice(0, recToExtract);
-    const fileTS = ordered[ordered.length - 1].deliveryTS;
-    for (const rec of ordered) {
-        byId.set(rec.msgId, rec);
-        records.byId.delete(rec.msgId);
-    }
-    return { byId, ordered, fileTS };
-}
-function reduceToMsgInfosInPlace(records) {
-    for (let i = 0; i < records.length; i += 1) {
-        const orig = records[i];
-        records[i] = {
-            msgType: orig.msgType,
-            msgId: orig.msgId,
-            deliveryTS: orig.deliveryTS
-        };
+function columnIndecies(columns, ...columnNames) {
+    const indecies = new Map();
+    for (const colName of columnNames) {
+        indecies.set(colName, columns.indexOf(colName));
     }
+    return indecies;
 }
-/**
- * This message index stores MsgRecord's for message present on the server, i.e.
- * inbox. Records contain message key info, time of delivery, and time of
- * desired removal. Note that when user wants to keep a particular message for
- * a long time, it should be copied elsewhere, and not kept in the asmail
- * server inbox. Therefore, this index has to deal only with more-or-less
- * recent messages.
- *
- * This implementation is not using database for records, but a log-like files.
- * Latest file is appended to certain length, and then is left to a mostly-read
- * existence, carrying messages' info till message removal.
- * When lookup is done only with message id, all files potentially need to be
- * read. When both message id and delivery timestamp are given, lookup will at
- * most touch two-three files.
- *
- */
-class MsgIndex {
-    constructor(files) {
+function deleteMsgFrom(db, msgId) {
+    db.exec(deleteRec, { [`$${column.msgId}`]: msgId });
+    return (db.getRowsModified() > 0);
+}
+const LIMIT_RECORDS_PER_FILE = 200;
+class RecordsInSQL {
+    constructor(files, latest, fileTSs) {
         this.files = files;
-        this.latest = undefined;
-        this.cached = (0, timed_cache_1.makeTimedCache)(10 * 60 * 1000);
-        this.fileTSs = undefined;
-        this.fileProc = new synced_1.SingleProc();
+        this.latest = latest;
+        this.fileTSs = fileTSs;
+        this.older = (0, timed_cache_1.makeTimedCache)(10 * 60 * 1000);
         Object.seal(this);
     }
-    static async makeWith(files) {
-        const index = new MsgIndex(files);
-        // 1) initialize latest records index
-        const latest = await index.getRecords(undefined);
-        if (latest) {
-            index.latest = latest;
+    async add(msgInfo, decrInfo, removeAfter) {
+        const { msgId, msgType, deliveryTS } = msgInfo;
+        const { key, keyStatus, msgKeyPackLen: mainObjHeaderOfs } = decrInfo;
+        const params = {
+            [`$${column.msgId}`]: msgId,
+            [`$${column.msgType}`]: msgType,
+            [`$${column.deliveryTS}`]: deliveryTS,
+            [`$${column.key}`]: key,
+            [`$${column.keyStatus}`]: keyStatus,
+            [`$${column.mainObjHeaderOfs}`]: mainObjHeaderOfs,
+            [`$${column.removeAfter}`]: removeAfter,
+        };
+        const { db, fileTS } = await this.getDbFor(msgInfo.deliveryTS);
+        db.db.exec(insertRec, params);
+        if (fileTS) {
+            await db.saveToFile();
+            return;
         }
         else {
-            index.latest = {
-                fileTS: undefined,
-                byId: new Map(),
-                ordered: []
+            return {
+                type: 'addition',
+                record: {
+                    msgId, msgType, deliveryTS,
+                    key: buffer_utils_1.base64.pack(key),
+                    keyStatus, mainObjHeaderOfs, removeAfter
+                }
             };
-            await index.saveRecords(index.latest);
         }
-        // 2) initialize list of file timestamps, that act as index file names
-        index.fileTSs = (await index.files.listFolder('.'))
-            .map(f => f.name)
-            .filter(fName => fName.match(INDEX_FNAME_REGEXP))
-            .map(fName => parseInt(fName.substring(0, fName.length - INDEX_EXT.length)))
-            .sort(fileTSOrderComparator);
-        return index;
     }
-    /**
-     * @param recs that need to be saved
-     * @return a promise, resolvable when given records are saved. Latest
-     * records can also be chunked and saved as needed.
-     */
-    async saveRecords(recs) {
-        if (typeof recs.fileTS !== 'number') {
-            if (this.latest.ordered.length > 1.25 * LIMIT_RECORDS_PER_FILE) {
-                const recs = extractEarlyRecords(this.latest, LIMIT_RECORDS_PER_FILE);
-                const recsFileTS = recs.ordered[recs.ordered.length - 1].deliveryTS;
-                const fName = recsFileTS + INDEX_EXT;
-                await this.files.writeJSONFile(fName, recs.ordered);
-                this.cached.set(recsFileTS, recs);
-                this.fileTSs.push(recsFileTS);
-            }
-            await this.files.writeJSONFile(LATEST_INDEX, this.latest.ordered);
-        }
-        else {
-            const fName = recs.fileTS + INDEX_EXT;
-            await this.files.writeJSONFile(fName, recs.ordered);
-            this.cached.set(recs.fileTS, recs);
-        }
+    async saveLatestWithAttr(logTail) {
+        // XXX
     }
-    /**
-     * @param fileTS is index file's timestamp. Undefined value means latest
-     * records.
-     * @return a promise, resolvable to found records, either from a memory
-     * cache, or from a file.
-     */
-    async getRecords(fileTS) {
-        if (typeof fileTS === 'number') {
-            const recs = this.cached.get(fileTS);
-            if (recs) {
-                return recs;
-            }
+    async getDbFor(deliveryTS) {
+        if ((this.fileTSs.length === 0)
+            || (this.fileTSs[this.fileTSs.length - 1] < deliveryTS)) {
+            return { db: this.latest };
         }
-        else {
-            if (this.latest) {
-                return this.latest;
+        let fileTS = this.fileTSs[this.fileTSs.length - 1];
+        for (let i = (this.fileTSs.length - 2); i >= 0; i -= 1) {
+            if (this.fileTSs[i] >= deliveryTS) {
+                fileTS = this.fileTSs[i];
+            }
+            else {
+                break;
             }
         }
-        const fName = (typeof fileTS === 'number') ?
-            `${fileTS}INDEX_EXT` : LATEST_INDEX;
-        const ordered = await this.files.readJSONFile(fName)
-            .catch(notFoundOrReThrow);
-        if (!ordered) {
-            return;
-        }
-        ordered.sort(sortMsgByDeliveryTime);
-        const byId = new Map();
-        for (const rec of ordered) {
-            byId.set(rec.msgId, rec);
-        }
-        const records = { byId, ordered, fileTS: fileTS };
-        if (typeof fileTS === 'number') {
-            this.cached.set(fileTS, records);
-        }
-        return records;
-    }
-    /**
-     * @param msgInfo is a minimal message info object
-     * @param decrInfo
-     * @return a promise, resolvable when given message info bits are recorded.
-     */
-    add(msgInfo, decrInfo) {
-        if (!decrInfo.key) {
-            throw new Error(`Given message decryption info doesn't have a key for message ${msgInfo.msgId}`);
+        const db = await this.dbFromCacheOrInit(fileTS);
+        return { db, fileTS };
+    }
+    async dbFromCacheOrInit(fileTS) {
+        let db = this.older.get(fileTS);
+        if (db) {
+            return db;
         }
-        const msg = {
-            msgType: msgInfo.msgType,
-            msgId: msgInfo.msgId,
-            deliveryTS: msgInfo.deliveryTS,
-            key: decrInfo.key,
-            keyStatus: decrInfo.keyStatus,
-            mainObjHeaderOfs: decrInfo.msgKeyPackLen
-        };
-        return this.fileProc.startOrChain(async () => {
-            // 1) msg should be inserted into latest part of index
-            if ((this.fileTSs.length === 0) ||
-                (msg.deliveryTS >= this.fileTSs[this.fileTSs.length - 1])) {
-                insertInto(this.latest, msg);
-                await this.saveRecords(this.latest);
-                return;
-            }
-            // 2) find non-latest file for insertion
-            let fileTS = this.fileTSs[this.fileTSs.length - 1];
-            for (let i = (this.fileTSs.length - 2); i <= 0; i -= 1) {
-                if (msg.deliveryTS >= this.fileTSs[i]) {
-                    break;
+        const dbFile = await this.files.getDBFile(fileTS);
+        db = await lib_sqlite_on_3nstorage_1.SQLiteOnSyncedFS.makeAndStart(dbFile);
+        this.older.set(fileTS, db);
+        return db;
+    }
+    async remove(msgId) {
+        for await (const { db, fileTS } of this.iterateDBs()) {
+            if (deleteMsgFrom(db.db, msgId)) {
+                if (fileTS) {
+                    await db.saveToFile();
+                    return;
                 }
                 else {
-                    fileTS = this.fileTSs[i];
+                    return {
+                        type: 'removal',
+                        msgId
+                    };
                 }
             }
-            const records = await this.getRecords(fileTS);
-            if (!records) {
-                throw new Error(`Expectation fail: there should be some message records.`);
-            }
-            insertInto(records, msg);
-            await this.saveRecords(records);
-        });
+        }
     }
-    async findRecordsWith(msg) {
-        // 1) msg should be in latest part of index
-        if ((this.fileTSs.length === 0) ||
-            (msg.deliveryTS >= this.fileTSs[this.fileTSs.length - 1])) {
-            if (!this.latest.byId.has(msg.msgId)) {
-                return;
+    async *iterateDBs() {
+        yield { db: this.latest };
+        for (let i = (this.fileTSs.length - 1); i >= 0; i = -1) {
+            const fileTS = this.fileTSs[i];
+            if (!fileTS) {
+                continue;
+            }
+            const db = await this.dbFromCacheOrInit(fileTS);
+            if (db) {
+                yield { db, fileTS };
             }
-            return this.latest;
         }
-        // 2) find non-latest index
-        let fileTS = this.fileTSs[this.fileTSs.length - 1];
-        for (let i = (this.fileTSs.length - 2); i <= 0; i -= 1) {
-            if (msg.deliveryTS >= this.fileTSs[i]) {
-                let records = await this.getRecords(fileTS);
-                if (!records || !records.byId.has(msg.msgId)) {
-                    if (msg.deliveryTS !== this.fileTSs[i]) {
-                        return;
-                    }
-                    fileTS = this.fileTSs[i];
-                    records = await this.getRecords(fileTS);
-                    if (!records || !records.byId.has(msg.msgId)) {
-                        return;
-                    }
-                }
-                return records;
+    }
+    async listMsgs(fromTS) {
+        let lst = listMsgInfos(this.latest.db, fromTS);
+        for (let i = (this.fileTSs.length - 1); i >= 0; i -= 1) {
+            const fileTS = this.fileTSs[i];
+            if (fromTS && (fileTS <= fromTS)) {
+                break;
             }
-            else {
-                fileTS = this.fileTSs[i];
+            const older = await this.dbFromCacheOrInit(fileTS);
+            lst = listMsgInfos(older.db, fromTS).concat(lst);
+        }
+        lst.sort((a, b) => (a.deliveryTS - b.deliveryTS));
+        return lst;
+    }
+    async getIndexWith(deliveryTS) {
+        let fileTS = undefined;
+        for (let i = (this.fileTSs.length - 1); i >= 0; i -= 1) {
+            const fTS = this.fileTSs[i];
+            if (fTS < deliveryTS) {
+                break;
             }
+            fileTS = fTS;
+        }
+        if (fileTS) {
+            return await this.dbFromCacheOrInit(fileTS);
         }
-        const records = await this.getRecords(this.fileTSs[0]);
-        if (!records || !records.byId.has(msg.msgId)) {
+        else {
+            return this.latest;
+        }
+    }
+    async getKeyFor(msgId, deliveryTS) {
+        const db = await this.getIndexWith(deliveryTS);
+        const result = db.db.exec(`SELECT
+			${column.key}, ${column.keyStatus}, ${column.mainObjHeaderOfs}
+			FROM ${indexTab}
+			WHERE ${column.msgId}=$${column.msgId}`, { [`$${column.msgId}`]: msgId });
+        if (result.length === 0) {
             return;
         }
-        return records;
-    }
-    /**
-     * @param msg is a message identifying info used to find and remove message
-     * @return a promise, resolvable when message is removed from this index.
-     */
-    remove(msg) {
-        return this.fileProc.startOrChain(async () => {
-            const records = await this.findRecordsWith(msg);
-            if (!records) {
-                return;
-            }
-            removeFrom(records, msg.msgId);
-            if ((records.fileTS !== null) && (records.ordered.length === 0)) {
-                await this.files.deleteFile(records.fileTS + INDEX_EXT);
-            }
-            else {
-                await this.saveRecords(records);
-            }
-        });
+        const { columns, values: [row] } = result[0];
+        const indecies = columnIndecies(columns, column.key, column.keyStatus, column.mainObjHeaderOfs);
+        return {
+            msgKey: row[indecies.get(column.key)],
+            msgKeyRole: row[indecies.get(column.keyStatus)],
+            mainObjHeaderOfs: row[indecies.get(column.mainObjHeaderOfs)]
+        };
+    }
+}
+Object.freeze(RecordsInSQL.prototype);
+Object.freeze(RecordsInSQL);
+class LogOfChanges {
+    constructor(files, logNums, latestLogVersion) {
+        this.files = files;
+        this.logNums = logNums;
+        this.latestLogVersion = latestLogVersion;
+        this.latestLogNum = this.logNums[this.logNums.length - 1];
+        Object.seal(this);
     }
-    /**
-     * @param msg is an id of a message to remove
-     * @return a promise, resolvable when message is removed from this index.
-     */
-    removeUsingIdOnly(msgId) {
-        return this.fileProc.startOrChain(async () => {
-            let records = this.latest;
-            let msgFound = records.byId.has(msgId);
-            const i = this.fileTSs.length - 1;
-            while (!msgFound && (i >= 0)) {
-                records = (await this.getRecords(this.fileTSs[i]));
-                msgFound = records.byId.has(msgId);
+    async push(change) {
+        const bytes = buffer_utils_1.utf8.pack(JSON.stringify(change));
+        const { uploadedVersion, writeOfs } = await this.files.appendLogFile(this.latestLogNum, bytes);
+        this.latestLogVersion = uploadedVersion;
+        return {
+            num: this.latestLogNum,
+            version: this.latestLogVersion,
+            writeOfs
+        };
+    }
+}
+Object.freeze(LogOfChanges.prototype);
+Object.freeze(LogOfChanges);
+const DBS_FOLDER = 'dbs';
+const CHANGES_FOLDER = 'changes';
+const LOG_EXT = '.log.json';
+const DB_EXT = '.sqlite';
+const LATEST_DB = `latest${DB_EXT}`;
+const COMMA_BYTE = buffer_utils_1.utf8.pack(',');
+const SQ_BRACKET_BYTE = buffer_utils_1.utf8.pack(']');
+class LogAndStructFiles {
+    constructor(logsFS, dbsFS) {
+        this.logsFS = logsFS;
+        this.dbsFS = dbsFS;
+        this.syncing = undefined;
+        (0, file_1.ensureCorrectFS)(this.logsFS, 'synced', true);
+        Object.seal(this);
+    }
+    static async makeAndStart(syncedFS) {
+        (0, file_1.ensureCorrectFS)(syncedFS, 'synced', true);
+        await (0, fs_sync_utils_1.getRemoteFolderChanges)(syncedFS);
+        const logsFS = await (0, fs_sync_utils_1.getOrMakeAndUploadFolderIn)(syncedFS, CHANGES_FOLDER);
+        const dbsFS = await (0, fs_sync_utils_1.getOrMakeAndUploadFolderIn)(syncedFS, DBS_FOLDER);
+        await (0, fs_sync_utils_1.uploadFolderChangesIfAny)(syncedFS);
+        const files = new LogAndStructFiles(logsFS, dbsFS);
+        const logs = await files.makeLogOfChanges();
+        const records = await files.makeRecords();
+        files.startSyncing();
+        return { files, logs, records };
+    }
+    async makeLogOfChanges() {
+        const logNums = await this.logsInFolder();
+        let logsTail;
+        if (logNums.length === 0) {
+            logsTail = await this.createNewLogFile(1);
+            logNums.push(logsTail.num);
+        }
+        else {
+            const lastLog = logNums[logNums.length - 1];
+            ({ tail: logsTail } = await this.statLogFile(lastLog));
+        }
+        return new LogOfChanges(this, logNums, logsTail.version);
+    }
+    logFileName(logNum) {
+        return `${logNum}${LOG_EXT}`;
+    }
+    async logsInFolder() {
+        const lst = await this.logsFS.listFolder(``);
+        const logNums = [];
+        for (const { isFile, name } of lst) {
+            if (!isFile || !name.endsWith(LOG_EXT)) {
+                continue;
             }
-            if (msgFound) {
-                removeFrom(records, msgId);
-                if (!!records.fileTS && (records.ordered.length === 0)) {
-                    await this.files.deleteFile(records.fileTS + INDEX_EXT);
-                }
-                else {
-                    await this.saveRecords(records);
-                }
+            const numStr = name.substring(0, LOG_EXT.length);
+            const logNum = parseInt(numStr);
+            if (isNaN(logNum)) {
+                continue;
             }
-        });
+            logNums.push(logNum);
+        }
+        logNums.sort();
+        return logNums;
     }
-    /**
-     * @param fromTS
-     * @return a promise, resolvable to an ordered array of MsgInfo's that have
-     * delivery timestamp same or later, than the given one.
-     */
-    async listMsgs(fromTS) {
-        return this.fileProc.startOrChain(async () => {
-            // find time starting point and get all records
-            let list = [];
-            for (let i = 0; i < this.fileTSs.length; i += 1) {
-                if (fromTS && (fromTS > this.fileTSs[i])) {
-                    continue;
-                }
-                const recs = (await this.getRecords(this.fileTSs[i]));
-                list = list.concat(recs.ordered);
-            }
-            list = list.concat(this.latest.ordered);
-            // truncate records
-            for (let i = 0; i < list.length; i += 1) {
-                if (fromTS && (fromTS >= list[i].deliveryTS)) {
-                    list = list.slice(i);
-                    break;
-                }
+    async createNewLogFile(logNum, jsonStr = '[]') {
+        const logFile = this.logFileName(logNum);
+        const version = await this.logsFS.v.writeTxtFile(logFile, jsonStr, { create: true, exclusive: true });
+        await this.logsFS.v.sync.upload(logFile);
+        await this.logsFS.v.sync.upload('');
+        return {
+            num: logNum,
+            version,
+            writeOfs: jsonStr.length - 1
+        };
+    }
+    async statLogFile(logNum) {
+        const logFile = this.logFileName(logNum);
+        const { state: syncState } = await this.logsFS.v.sync.status(logFile);
+        if (syncState === 'behind') {
+            await this.logsFS.v.sync.adoptRemote(logFile);
+        }
+        else if (syncState === 'unsynced') {
+            await this.logsFS.v.sync.upload(logFile);
+        }
+        else if (syncState === 'conflicting') {
+            // XXX
+            throw new Error(`conflict resolution needs implementation`);
+        }
+        const { size, version } = await this.logsFS.stat(logFile);
+        return {
+            syncState,
+            tail: {
+                num: logNum,
+                version: version,
+                writeOfs: size - 1
             }
-            reduceToMsgInfosInPlace(list);
-            return list;
-        });
+        };
+    }
+    async appendLogFile(logNum, bytes) {
+        const logFile = this.logFileName(logNum);
+        const { state } = await this.logsFS.v.sync.status(logFile);
+        if (state === 'behind') {
+            await this.logsFS.v.sync.adoptRemote(logFile);
+        }
+        else if (state === 'conflicting') {
+            // XXX
+            throw new Error(`conflict resolution needs implementation`);
+        }
+        const sink = await this.logsFS.getByteSink(logFile, { truncate: false });
+        const len = await sink.getSize();
+        let writeOfs;
+        if (len === 2) {
+            await sink.splice(len - 1, 1);
+            writeOfs = len - 1;
+        }
+        else {
+            await sink.splice(len - 1, 1, COMMA_BYTE);
+            writeOfs = len;
+        }
+        await sink.splice(writeOfs, 0, bytes);
+        writeOfs += bytes.length;
+        await sink.splice(writeOfs, 0, SQ_BRACKET_BYTE);
+        await sink.done();
+        const uploadedVersion = (await this.logsFS.v.sync.upload(logFile));
+        return { uploadedVersion, writeOfs };
+    }
+    dbFileName(fileTS) {
+        return `${fileTS}${DB_EXT}`;
+    }
+    async makeRecords() {
+        const latest = await this.readOrInitializeLatestDB();
+        const fileTSs = await this.fileTSsOfDBShards();
+        return new RecordsInSQL(this, latest, fileTSs);
+    }
+    async getDBFile(fileTS) {
+        return await this.dbsFS.writableFile(this.dbFileName(fileTS), { create: false });
     }
-    /**
-     * This returns a promise resolvable to message's file key holder and key
-     * role, when message is found, and resolvabel to an undefined, when message
-     * is not known.
-     * @param msg is a message identifying info used to find message
-     */
-    fKeyFor(msg) {
-        return this.fileProc.startOrChain(async () => {
-            const records = await this.findRecordsWith(msg);
-            if (!records) {
-                return;
+    async readOrInitializeLatestDB() {
+        if (await this.dbsFS.checkFilePresence(LATEST_DB)) {
+            const dbFile = await this.dbsFS.writableFile(LATEST_DB, { create: false });
+            return await lib_sqlite_on_3nstorage_1.SQLiteOnVersionedFS.makeAndStart(dbFile);
+        }
+        else {
+            const dbFile = await this.dbsFS.writableFile(LATEST_DB, { create: true, exclusive: true });
+            const latest = await lib_sqlite_on_3nstorage_1.SQLiteOnVersionedFS.makeAndStart(dbFile);
+            latest.db.run(createIndexTab);
+            await latest.saveToFile();
+            return latest;
+        }
+    }
+    async fileTSsOfDBShards() {
+        const lst = await this.dbsFS.listFolder('');
+        const fileTSs = [];
+        for (const { isFile, name } of lst) {
+            if (!isFile || !name.endsWith(DB_EXT)) {
+                continue;
             }
-            const rec = records.byId.get(msg.msgId);
-            if (!rec) {
-                return;
+            const numStr = name.substring(0, DB_EXT.length);
+            const fileTS = parseInt(numStr);
+            if (isNaN(fileTS)) {
+                continue;
             }
-            return {
-                msgKey: buffer_utils_1.base64.open(rec.key),
-                msgKeyRole: rec.keyStatus,
-                mainObjHeaderOfs: rec.mainObjHeaderOfs
-            };
-        });
+            fileTSs.push(fileTS);
+        }
+        fileTSs.sort();
+        return fileTSs;
+    }
+    startSyncing() {
+        const db$ = (0, fs_sync_utils_1.observableFromTreeEvents)(this.dbsFS, '');
+        const change$ = (0, fs_sync_utils_1.observableFromTreeEvents)(this.logsFS, '');
+        // XXX
+        // - start from data in fs, attempt to get fresh, etc. Both log and data
+        //   (in parallel ?).
+        // - start sync process
+        // - unblock processing, as init is done
+        // Write theses in functions that use RecordsInSQL and LogOfChanges
+        // structures.
+        // Somehow aforementioned processes ain't exclusive to either point.
+        // XXX
+        //  should start from reading folder and placing logs into yet unsynced db
+        this.syncing = (0, rxjs_1.merge)(change$, db$)
+            .pipe((0, operators_1.filter)(ev => ev.type.startsWith('remote-')))
+            // .subscribe({
+            // 	next: ev => console.log(`------ fs event:`, ev),
+            // 	complete: () => console.log(` +++ MsgIndex's sync process completed`),
+            // 	error: err => console.log(` *** error in MsgIndex's sync process`, err)
+            // });
+            .subscribe();
+    }
+    stopSyncing() {
+        if (this.syncing) {
+            this.syncing.unsubscribe();
+            this.syncing = undefined;
+        }
     }
 }
-exports.MsgIndex = MsgIndex;
-Object.freeze(MsgIndex.prototype);
-Object.freeze(MsgIndex);
+Object.freeze(LogAndStructFiles.prototype);
+Object.freeze(LogAndStructFiles);
 /**
- * This is a catch callback, which returns undefined on file(folder) not found
- * exception, and re-throws all other exceptions/errors.
+ * This message index stores info for messages present on the server, in the
+ * inbox. Records contain message key info, time of delivery, and time of
+ * desired removal.
+ *
+ * Message info with keys is stored in SQLite dbs sharded/partitioned by
+ * delivery timestamp. The latest shard, shard without upper time limit
+ * is stored in local storage, while all other shards with limits are stored in
+ * synced storage. Information in synced storage is a sum of all limited shards
+ * and action logs. Action logs
+ *
  */
-function notFoundOrReThrow(exc) {
-    if (!exc.notFound) {
-        throw exc;
+class MsgIndex {
+    constructor(files, records, changes) {
+        this.files = files;
+        this.records = records;
+        this.changes = changes;
+        Object.seal(this);
+    }
+    static async make(syncedFS) {
+        const { files, logs, records } = await LogAndStructFiles.makeAndStart(syncedFS);
+        const index = new MsgIndex(files, records, logs);
+        return index;
+    }
+    stopSyncing() {
+        this.files.stopSyncing();
+    }
+    async add(msgInfo, decrInfo, removeAfter = 0) {
+        const logChange = await this.records.add(msgInfo, decrInfo, removeAfter);
+        if (!logChange) {
+            return;
+        }
+        const logTail = await this.changes.push(logChange);
+        await this.records.saveLatestWithAttr(logTail);
+    }
+    async remove(msgId) {
+        const logChange = await this.records.remove(msgId);
+        if (!logChange) {
+            return;
+        }
+        const logTail = await this.changes.push(logChange);
+        await this.records.saveLatestWithAttr(logTail);
+    }
+    listMsgs(fromTS) {
+        return this.records.listMsgs(fromTS);
+    }
+    getKeyFor(msgId, deliveryTS) {
+        return this.records.getKeyFor(msgId, deliveryTS);
     }
-    return;
-}
-function sortMsgByDeliveryTime(a, b) {
-    return (a.deliveryTS - b.deliveryTS);
 }
+exports.MsgIndex = MsgIndex;
+Object.freeze(MsgIndex.prototype);
+Object.freeze(MsgIndex);
 Object.freeze(exports);