core-3nweb-client-lib 0.49.5 → 0.49.7

package/build/core/asmail/inbox/msg-indexing/dataset/v1/sql-db.js

@@ -0,0 +1,477 @@
+"use strict";
+/*
+ Copyright (C) 2022 - 2023, 2025 - 2026 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeSqliteDBs = makeSqliteDBs;
+const timed_cache_1 = require("../../../../../../lib-common/timed-cache");
+const lib_sqlite_on_3nstorage_1 = require("../../../../../../lib-sqlite-on-3nstorage");
+const logs_1 = require("./logs");
+const for_sqlite_1 = require("../../../../../../lib-sqlite-on-3nstorage/for-sqlite");
+const sql_db_v_none_1 = require("../v0/sql-db-v-none");
+const queryToCreateMsgsIndexDbV1 = [
+    `--sql
+	CREATE TABLE inbox_index (
+		msgId TEXT PRIMARY KEY,
+		msgType TEXT,
+		deliveryTS INTEGER,
+		key BLOB,
+		keyStatus TEXT,
+		mainObjHeaderOfs INTEGER,
+		removeAfter INTEGER DEFAULT 0
+	) STRICT
+	`,
+    `--sql
+    CREATE INDEX IF NOT EXISTS delivery_ts ON inbox_index (
+      deliveryTS ASC
+    )
+  `,
+].join(';\n');
+const msgsIndexFields = {
+    msgId: 'as-is',
+    msgType: 'as-is',
+    deliveryTS: 'as-is',
+    key: 'as-is',
+    keyStatus: 'as-is',
+    mainObjHeaderOfs: 'as-is'
+};
+function msgWhereParamsFor(msgId) {
+    const whereMsgParams = (0, for_sqlite_1.queryParamsFrom)({
+        msgId
+    }, msgsIndexFields);
+    const whereMsg = (0, for_sqlite_1.andEqualExprFor)(whereMsgParams);
+    return { whereMsgParams, whereMsg };
+}
+function listMsgInfos(db, fromTS) {
+    let result;
+    if (fromTS) {
+        result = db.exec(`--sql
+			SELECT msgId, msgType, deliveryTS FROM inbox_index
+			WHERE deliveryTS > $fromTS`, { '$fromTS': fromTS });
+    }
+    else {
+        result = db.exec(`--sql
+			SELECT msgId, msgType, deliveryTS FROM inbox_index`);
+    }
+    return ((result.length > 0) ? (0, for_sqlite_1.fromQueryResult)(result[0], msgsIndexFields) : []);
+}
+function deleteMsgFrom(db, msgId) {
+    const { whereMsg, whereMsgParams } = msgWhereParamsFor(msgId);
+    db.exec(`--sql
+		DELETE FROM inbox_index WHERE ${whereMsg}`, whereMsgParams);
+    return (db.getRowsModified() > 0);
+}
+function deleteAllMsgsAfterTS(db, ts) {
+    db.exec(`--sql
+		DELETE FROM inbox_index WHERE deliveryTS > $ts
+		`, { '$ts': ts });
+}
+function deleteAllMsgsUpToAndAtTS(db, ts) {
+    db.exec(`--sql
+		DELETE FROM inbox_index WHERE deliveryTS <= $ts
+		`, { '$ts': ts });
+}
+function findMsgAndGetDeliveryTS(db, msgId) {
+    const { whereMsg, whereMsgParams } = msgWhereParamsFor(msgId);
+    const result = db.exec(`--sql
+		SELECT deliveryTS FROM inbox_index WHERE ${whereMsg}`, whereMsgParams)[0];
+    if (result) {
+        const values = (0, for_sqlite_1.fromQueryResult)(result, msgsIndexFields);
+        return ((values.length > 0) ? values[0].deliveryTS : undefined);
+    }
+}
+function countUpToTS(db, tillTS) {
+    const result = db.exec(`--sql
+		SELECT count() FROM inbox_index WHERE deliveryTS <= $tillTS`, { '$tillTS': tillTS });
+    return result[0].values[0][0];
+}
+function findMsgKey(db, msgId) {
+    const { whereMsg, whereMsgParams } = msgWhereParamsFor(msgId);
+    const result = db.exec(`--sql
+		SELECT key, keyStatus, mainObjHeaderOfs FROM inbox_index WHERE ${whereMsg}`, whereMsgParams);
+    if (result.length > 0) {
+        const { key: msgKey, keyStatus: msgKeyRole, mainObjHeaderOfs } = (0, for_sqlite_1.fromQueryResult)(result[0], msgsIndexFields)[0];
+        return { msgKey, msgKeyRole, mainObjHeaderOfs };
+    }
+    else {
+        return;
+    }
+}
+function isMsgPresent(db, msgId) {
+    const { whereMsg, whereMsgParams } = msgWhereParamsFor(msgId);
+    const result = db.exec(`--sql
+		SELECT msgId FROM inbox_index WHERE ${whereMsg}`, whereMsgParams);
+    return (result.length > 0);
+}
+async function readOrInitializeLatestDB(dbsFS) {
+    if (await dbsFS.checkFilePresence(LATEST_DB)) {
+        const dbFile = await dbsFS.writableFile(LATEST_DB, { create: false });
+        return await readDbAndUpgradeVersionIfNeeded(dbFile);
+    }
+    else {
+        const dbFile = await dbsFS.writableFile(LATEST_DB, { create: true, exclusive: true });
+        const latest = await lib_sqlite_on_3nstorage_1.SQLiteOn3NStorage.makeSynced(dbFile);
+        latest.db.run(queryToCreateMsgsIndexDbV1);
+        await setMsgIndexInfoIntoAttrOf(latest.dbFile, {});
+        await latest.saveToFile();
+        await latest.dbFileSync.upload();
+        await dbsFS.v.sync.upload('');
+        return latest;
+    }
+}
+function insertMsgEntryInto(db, msgEntry) {
+    const { orderedColumns, orderedValues, insertParams } = (0, for_sqlite_1.forTableInsert)(msgEntry, msgsIndexFields);
+    db.run(`--sql
+		INSERT INTO inbox_index (${orderedColumns}) VALUES (${orderedValues})`, insertParams);
+}
+async function readDbAndUpgradeVersionIfNeeded(dbFile) {
+    const dbInfo = await dbFile.getXAttr(MSG_INDEX_DB_ATTR);
+    if (!dbInfo) {
+        const preV1 = await lib_sqlite_on_3nstorage_1.SQLiteOn3NStorage.makeSynced(dbFile);
+        const preV1Records = (0, sql_db_v_none_1.getAllRecordsFromVersionNone)(preV1.db);
+        const msgs = preV1Records.map(({ msg_id, msg_type, delivery_ts, key_status, key, main_obj_header_ofs, remove_after }) => ({
+            msgId: msg_id, msgType: msg_type, deliveryTS: delivery_ts,
+            keyStatus: key_status, key, mainObjHeaderOfs: main_obj_header_ofs
+        }));
+        const dbV1 = await lib_sqlite_on_3nstorage_1.SQLiteOn3NStorage.makeSynced(dbFile, false);
+        dbV1.db.run(queryToCreateMsgsIndexDbV1);
+        for (const msgEntry of msgs) {
+            insertMsgEntryInto(dbV1.db, msgEntry);
+        }
+        await setMsgIndexInfoIntoAttrOf(dbV1.dbFile, {});
+        await dbV1.saveToFile();
+        return dbV1;
+    }
+    else if (dbInfo.dbVersion === 1) {
+        return await lib_sqlite_on_3nstorage_1.SQLiteOn3NStorage.makeSynced(dbFile);
+    }
+    else {
+        throw new Error(`Platform needs an upgrade: MsgIndex db file has unknown version ${dbInfo.dbVersion}`);
+    }
+}
+async function fileTSsOfDBShards(dbsFS) {
+    const lst = await dbsFS.listFolder('');
+    const fileTSs = [];
+    for (const { isFile, name } of lst) {
+        if (!isFile || !name.endsWith(DB_EXT)) {
+            continue;
+        }
+        const numStr = name.substring(0, DB_EXT.length);
+        const fileTS = parseInt(numStr);
+        if (isNaN(fileTS)) {
+            continue;
+        }
+        fileTSs.push(fileTS);
+    }
+    fileTSs.sort();
+    return fileTSs;
+}
+const DB_EXT = '.sqlite';
+const LATEST_DB = `latest${DB_EXT}`;
+const MSG_INDEX_DB_ATTR = `msg-index-db`;
+async function setMsgIndexInfoIntoAttrOf(dbFile, info) {
+    const completeInfo = {
+        dbVersion: 1,
+        ...info
+    };
+    await dbFile.updateXAttrs({
+        set: { [MSG_INDEX_DB_ATTR]: completeInfo }
+    });
+}
+function dbFileName(shardTS) {
+    return (shardTS ? LATEST_DB : `${shardTS}${DB_EXT}`);
+}
+function shardTSFromFileName(fName) {
+    if (fName.endsWith(DB_EXT)) {
+        const shardTS = parseInt(fName.slice(0, fName.length - DB_EXT.length));
+        return ((isNaN(shardTS) || (fName !== dbFileName(shardTS))) ? undefined : shardTS);
+    }
+    else {
+        return;
+    }
+}
+async function makeSqliteDBs(dbsFS) {
+    const shardsCache = (0, timed_cache_1.makeTimedCache)(10 * 60 * 1000);
+    const latest = await readOrInitializeLatestDB(dbsFS);
+    const fileTSs = await fileTSsOfDBShards(dbsFS);
+    async function getDBFile(fileTS) {
+        return await dbsFS.writableFile(dbFileName(fileTS), { create: false });
+    }
+    function numberOfRecordsInLatestBefore(ts) {
+        return countUpToTS(latest.db, ts);
+    }
+    /**
+     * This return true when msg should be added to log, and false otherwise.
+     * @param msgInfo
+     * @param decrInfo
+     * @param removeAfter
+     */
+    async function addToLatest(msgInfo, decrInfo) {
+        if (!isForLatestDb(msgInfo.deliveryTS)) {
+            return false;
+        }
+        const recordAdded = addMsgRecordToLatest(msgInfo, decrInfo);
+        if (recordAdded) {
+            await latest.saveToFile();
+        }
+        return recordAdded;
+    }
+    function addMsgRecordToLatest(msgInfo, decrInfo) {
+        const { msgId, msgType, deliveryTS } = msgInfo;
+        const { db } = latest;
+        if (isMsgPresent(db, msgId)) {
+            return false;
+        }
+        const { key, keyStatus, msgKeyPackLen: mainObjHeaderOfs } = decrInfo;
+        insertMsgEntryInto(db, {
+            msgId, msgType, deliveryTS, key: key, keyStatus, mainObjHeaderOfs
+        });
+        return true;
+    }
+    function isForLatestDb(deliveryTS) {
+        return ((fileTSs.length === 0) || (fileTSs[fileTSs.length - 1] < deliveryTS));
+    }
+    async function dbShardFromCacheOrFS(fileTS) {
+        let db = shardsCache.get(fileTS);
+        if (db) {
+            return db;
+        }
+        const dbFile = await getDBFile(fileTS);
+        db = await lib_sqlite_on_3nstorage_1.SQLiteOn3NStorage.makeSynced(dbFile);
+        shardsCache.set(fileTS, db);
+        return db;
+    }
+    async function remove(msgId) {
+        for await (const { db, fileTS } of iterateDBsFromLatestToOldest()) {
+            const deliveryTS = findMsgAndGetDeliveryTS(db.db, msgId);
+            if (deliveryTS) {
+                deleteMsgFrom(db.db, msgId);
+                await db.saveToFile();
+                return deliveryTS;
+            }
+        }
+        return undefined;
+    }
+    async function* iterateDBsFromLatestToOldest() {
+        yield { db: latest };
+        for (let i = (fileTSs.length - 1); i >= 0; i = -1) {
+            const fileTS = fileTSs[i];
+            if (!fileTS) {
+                continue;
+            }
+            const db = await dbShardFromCacheOrFS(fileTS);
+            if (db) {
+                yield { db, fileTS };
+            }
+        }
+    }
+    async function listMsgs(fromTS) {
+        let lst = listMsgInfos(latest.db, fromTS);
+        for (let i = (fileTSs.length - 1); i >= 0; i -= 1) {
+            const fileTS = fileTSs[i];
+            if (fromTS && (fileTS <= fromTS)) {
+                break;
+            }
+            const older = await dbShardFromCacheOrFS(fileTS);
+            lst = listMsgInfos(older.db, fromTS).concat(lst);
+        }
+        lst.sort((a, b) => (a.deliveryTS - b.deliveryTS));
+        return lst;
+    }
+    async function getIndexWith(deliveryTS) {
+        let fileTS = undefined;
+        for (let i = (fileTSs.length - 1); i >= 0; i -= 1) {
+            const fTS = fileTSs[i];
+            if (fTS < deliveryTS) {
+                break;
+            }
+            fileTS = fTS;
+        }
+        if (fileTS) {
+            return await dbShardFromCacheOrFS(fileTS);
+        }
+        else {
+            return latest;
+        }
+    }
+    async function getKeyFor(msgId, deliveryTS) {
+        const db = await getIndexWith(deliveryTS);
+        return findMsgKey(db.db, msgId);
+    }
+    async function msgExists({ msgId, deliveryTS }) {
+        const db = await getIndexWith(deliveryTS);
+        return isMsgPresent(db.db, msgId);
+    }
+    // XXX at a startup we should check if current local latest is too old
+    // XXX one should watch for reconnect event and run similar checks of possibly missed events
+    function watchAndApplyOpsFromOtherDevices(changeProc, resetSyncInterval) {
+        return dbsFS.watchTree('.', 1, {
+            next: fsEvent => changeProc.startOrChain(async () => {
+                if (fsEvent.type === 'remote-change') {
+                    if (fsEvent.path === '.') {
+                        doWhenDBsFolderChangedOnRemote();
+                    }
+                    else if (fsEvent.path === LATEST_DB) {
+                        // XXX latest db shard file has been cut
+                        //  - remove stuff from latest and from log
+                        const { state, remote } = await latest.dbFileSync.status();
+                        if (state === 'behind') {
+                            await latest.dbFileSync.adoptRemote();
+                        }
+                        else if (state === 'conflicting') {
+                            const remoteAttr = await latest.dbFile.v.getXAttr(MSG_INDEX_DB_ATTR, { remoteVersion: remote === null || remote === void 0 ? void 0 : remote.latest });
+                        }
+                    }
+                    else {
+                        // XXX old db shard file has changed.
+                        //   -> file should adopt remote state, and may be trigger download
+                    }
+                }
+                else if ((fsEvent.type === 'entry-addition') && (fsEvent.src === 'sync') && fsEvent.entry.isFile) {
+                    resetSyncInterval();
+                    await doWhenShardWasAddedFromRemote(fsEvent.entry.name);
+                }
+                else if ((fsEvent.type === 'entry-removal') && (fsEvent.src === 'sync') && fsEvent.name) {
+                    doWhenShardWasRemovedFromRemote(fsEvent.name);
+                }
+            })
+        });
+    }
+    async function doWhenDBsFolderChangedOnRemote() {
+        const { state } = await dbsFS.v.sync.status('.');
+        if (state === 'behind') {
+            await dbsFS.v.sync.adoptRemote('.');
+        }
+        else if (state === 'conflicting') {
+            await dbsFS.v.sync.absorbRemoteFolderChanges('.');
+            await dbsFS.v.sync.upload('.');
+        }
+    }
+    async function doWhenShardWasAddedFromRemote(shardFName) {
+        const shardTS = shardTSFromFileName(shardFName);
+        if (!shardTS) {
+            return;
+        }
+        for (let i = fileTSs.length; i >= 0; i - 1) {
+            const ts = fileTSs[i];
+            if (ts < shardTS) {
+                fileTSs.slice(i + 1, shardTS);
+                break;
+            }
+            else if (ts === shardTS) {
+                return;
+            }
+        }
+        const stats = await dbsFS.stat(shardFName);
+        await dbsFS.v.sync.startDownload(shardFName, stats.version);
+    }
+    async function doWhenShardWasRemovedFromRemote(shardFName) {
+        const shardTS = shardTSFromFileName(shardFName);
+        if (!shardTS) {
+            return;
+        }
+        const tsInd = fileTSs.indexOf(shardTS);
+        if (tsInd < 0) {
+            return;
+        }
+        fileTSs.splice(tsInd, 1);
+        shardsCache.delete(shardTS);
+    }
+    async function latestDbHasNewRemoteVersion() {
+        const { state } = await latest.dbFile.v.sync.status();
+        return ((state === 'behind') || (state === 'conflicting'));
+    }
+    async function absorbOpsFromOtherDevices(changeProc, ops) {
+        await changeProc.startOrChain(async () => {
+            let dbsToSave = new Set();
+            for (const op of ops) {
+                if (op.p) {
+                    if (isForLatestDb(op.p.deliveryTS)) {
+                        if (op.p.eventType === 'msg-opened') {
+                            const { msgInfo, decrInfo } = (0, logs_1.msgOpenedEventToRecordParams)(op.p);
+                            const addedMsg = addMsgRecordToLatest(msgInfo, decrInfo);
+                            if (addedMsg) {
+                                dbsToSave.add(latest);
+                            }
+                        }
+                        else if (op.p.eventType === 'msg-removed') {
+                            const removedMsg = deleteMsgFrom(latest.db, op.p.msgId);
+                            if (removedMsg) {
+                                dbsToSave.add(latest);
+                            }
+                        }
+                    }
+                    else if (op.p.eventType === 'msg-removed') {
+                        const shard = await getIndexWith(op.p.deliveryTS);
+                        const removedMsg = deleteMsgFrom(shard.db, op.p.msgId);
+                        if (removedMsg) {
+                            dbsToSave.add(shard);
+                        }
+                    }
+                }
+                else if (op.syncPoint) {
+                    // XXX
+                    //  - 
+                }
+            }
+            for (const db of dbsToSave) {
+                await db.saveToFile();
+            }
+        });
+    }
+    async function makeNewShardWithRecordsUpToTS(shardTS) {
+        // make and upload shard
+        const shardFile = await dbsFS.writableFile(dbFileName(shardTS), { create: true, exclusive: true });
+        const shard = await latest.makeCopyIn(shardFile);
+        deleteAllMsgsAfterTS(shard.db, shardTS);
+        await shard.saveToFile();
+        await setMsgIndexInfoIntoAttrOf(shard.dbFile, {
+            shardTS,
+            prevShardTS: ((fileTSs.length > 0) ? fileTSs[fileTSs.length - 1] : undefined)
+        });
+        await shard.dbFileSync.upload();
+        await dbsFS.v.sync.upload('');
+        // cut and upload latest
+        deleteAllMsgsUpToAndAtTS(latest.db, shardTS);
+        fileTSs.push(shardTS);
+        await setMsgIndexInfoIntoAttrOf(latest.dbFile, { prevShardTS: shardTS });
+        const syncedIndexDbVersion = await latest.dbFileSync.upload();
+        return syncedIndexDbVersion;
+    }
+    async function syncPastShards() {
+        // XXX only removals are done on older shards, so conflicts should be easy to process
+        //  - check sync status of shard file without instantiating dbs.
+        //  - push unsynced
+        //  - adopt and download a thing that is behind
+        //  - 
+    }
+    return {
+        absorbOpsFromOtherDevices,
+        addToLatest,
+        getDBFile,
+        getKeyFor,
+        isForLatestDb,
+        latestDbHasNewRemoteVersion,
+        listMsgs,
+        msgExists,
+        makeNewShardWithRecordsUpToTS,
+        syncPastShards,
+        numberOfRecordsInLatestBefore,
+        remove,
+        watchAndApplyOpsFromOtherDevices
+    };
+}
+Object.freeze(exports);

package/build/core/asmail/inbox/msg-indexing/index.d.ts

@@ -1,31 +1,17 @@
-import { MsgKeyInfo, MsgKeyRole } from '../../../keyring';
+import type { MsgKeyInfo } from '../../../keyring';
+import type { LogError } from '../../../../lib-client/logging/log-to-file';
 type WritableFS = web3n.files.WritableFS;
 type MsgInfo = web3n.asmail.MsgInfo;
 /**
- * This message index stores info for messages present on the server, in the
- * inbox. Records contain message key info, time of delivery, and time of
- * desired removal.
- *
- * Message info with keys is stored in SQLite dbs sharded/partitioned by
- * delivery timestamp. The latest shard, shard without upper time limit
- * is stored in local storage, while all other shards with limits are stored in
- * synced storage. Information in synced storage is a sum of all limited shards
- * and action logs. Action logs
- *
+ * This message index stores info for messages present on the server, in the inbox.
+ * Records contain message key info, time of delivery, and time of desired removal.
  */
-export declare class MsgIndex {
-    private readonly logs;
-    private readonly indexed;
-    private constructor();
-    static make(syncedFS: WritableFS): Promise<MsgIndex>;
-    stopSyncing(): void;
-    add(msgInfo: MsgInfo, decrInfo: MsgKeyInfo, removeAfter?: number): Promise<void>;
-    remove(msgId: string): Promise<void>;
-    listMsgs(fromTS: number | undefined): Promise<MsgInfo[]>;
-    getKeyFor(msgId: string, deliveryTS: number): Promise<{
-        msgKey: Uint8Array;
-        msgKeyRole: MsgKeyRole;
-        mainObjHeaderOfs: number;
-    } | undefined>;
-}
+export type MsgIndex = Awaited<ReturnType<typeof makeMsgIndex>>;
+export declare function makeMsgIndex(syncedFS: WritableFS, localFS: WritableFS, logError: LogError): Promise<{
+    add: (msgInfo: MsgInfo, decrInfo: MsgKeyInfo) => Promise<void>;
+    remove: (msgId: string) => Promise<void>;
+    listMsgs: (fromTS: number | undefined) => Promise<web3n.asmail.MsgInfo[]>;
+    getKeyFor: (msgId: string, deliveryTS: number) => Promise<import("./dataset/v1/sql-db").MsgKey | undefined>;
+    stopSyncing: () => void;
+}>;
 export {};

package/build/core/asmail/inbox/msg-indexing/index.js

@@ -1,6 +1,6 @@
 "use strict";
 /*
- Copyright (C) 2022 - 2023 3NSoft Inc.
+ Copyright (C) 2022 - 2023, 2026 3NSoft Inc.
  
  This program is free software: you can redistribute it and/or modify it under
  the terms of the GNU General Public License as published by the Free Software
@@ -16,81 +16,60 @@
  this program. If not, see <http://www.gnu.org/licenses/>.
 */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MsgIndex = void 0;
-const logs_n_entries_1 = require("./logs-n-entries");
+exports.makeMsgIndex = makeMsgIndex;
+const sql_db_1 = require("./dataset/v1/sql-db");
+const logs_1 = require("./dataset/v1/logs");
+const fs_sync_utils_1 = require("../../../../lib-client/fs-utils/fs-sync-utils");
+const synced_1 = require("../../../../lib-common/processes/synced");
+const per_device_json_array_logs_1 = require("../../../../lib-common/dataset-sync/per-device-json-array-logs");
 const LOGS_DIR = 'logs';
 const INDEX_DIR = 'index';
-async function getOrMakeDirStructure(syncedFS) {
-    const logsFS = await syncedFS.writableSubRoot(LOGS_DIR);
-    const indexFS = await syncedFS.writableSubRoot(INDEX_DIR);
-    return { logsFS, indexFS };
-}
-async function syncDirStructureIfNeeded(syncedFS) {
-    // XXX need uploading of initial folders, and of syncedFS itself
-    // const logsDirInfo = await logsFS.v!.sync!.status(LOGS_DIR);
-    // logsDirInfo.state
-    // Or, is this enough?
-    // await getRemoteFolderChanges(syncedFS);
-    // await uploadFolderChangesIfAny(syncedFS);
-}
-/**
- * This message index stores info for messages present on the server, in the
- * inbox. Records contain message key info, time of delivery, and time of
- * desired removal.
- *
- * Message info with keys is stored in SQLite dbs sharded/partitioned by
- * delivery timestamp. The latest shard, shard without upper time limit
- * is stored in local storage, while all other shards with limits are stored in
- * synced storage. Information in synced storage is a sum of all limited shards
- * and action logs. Action logs
- *
- */
-class MsgIndex {
-    constructor(logs, indexed) {
-        this.logs = logs;
-        this.indexed = indexed;
-        Object.seal(this);
-    }
-    static async make(syncedFS) {
-        const { logsFS, indexFS } = await getOrMakeDirStructure(syncedFS);
-        await syncDirStructureIfNeeded(syncedFS);
-        const logs = await logs_n_entries_1.MsgLogs.makeAndStartSyncing(logsFS);
-        let indexed;
-        if (global.WebAssembly) {
-            indexed = await require('./sql-indexing').makeSqliteBasedIndexedRecords(indexFS);
-        }
-        else {
-            indexed = (0, logs_n_entries_1.makeJsonBasedIndexedRecords)(logs);
-        }
-        const index = new MsgIndex(logs, indexed);
-        return index;
-    }
-    stopSyncing() {
-        this.logs.stopSyncing();
-        this.indexed.stopSyncing();
-    }
-    async add(msgInfo, decrInfo, removeAfter = 0) {
-        const msgAlreadyExists = await this.indexed.msgExists(msgInfo);
-        if (msgAlreadyExists) {
-            return;
+const NUM_OF_ENTRIES_TO_CHOP_DB = 200;
+async function makeMsgIndex(syncedFS, localFS, logError) {
+    // file systems within dataset
+    const logsFS = await (0, fs_sync_utils_1.getOrMakeSyncedDirOnInit)(syncedFS, LOGS_DIR);
+    const localLogsFS = await localFS.writableSubRoot(LOGS_DIR);
+    const indexFS = await (0, fs_sync_utils_1.getOrMakeSyncedDirOnInit)(syncedFS, INDEX_DIR);
+    const logs = await (0, logs_1.makeRecentMsgIndexChangesLogs)(logsFS, localLogsFS, logError);
+    const indexed = await (0, sql_db_1.makeSqliteDBs)(indexFS);
+    async function add(msgInfo, decrInfo) {
+        const shouldAddToLog = await indexed.addToLatest(msgInfo, decrInfo);
+        if (shouldAddToLog) {
+            await logs.recordMsgAddition(msgInfo, decrInfo);
         }
-        await this.logs.add(msgInfo, decrInfo, removeAfter);
-        await this.indexed.add(msgInfo, decrInfo, removeAfter);
     }
-    async remove(msgId) {
-        const deliveryTS = await this.indexed.remove(msgId);
+    async function remove(msgId) {
+        const deliveryTS = await indexed.remove(msgId);
         if (deliveryTS) {
-            await this.logs.remove(msgId, deliveryTS);
+            await logs.recordMsgRemoval(msgId, deliveryTS);
         }
     }
-    listMsgs(fromTS) {
-        return this.indexed.listMsgs(fromTS);
+    async function periodicCheckAndUploadOfData() {
+        // XXX disable uploading for a moment by commenting out
+        // const shardTS = Date.now() - 10*60*1000;
+        // const countBefore = indexed.numberOfRecordsInLatestBefore(shardTS);
+        // if (countBefore >= NUM_OF_ENTRIES_TO_CHOP_DB) {
+        // 	const syncedIndexDbVersion = await indexed.makeNewShardWithRecordsUpToTS(shardTS);
+        // 	await logs.cutLogOnDatasetSyncAndUpload(syncedIndexDbVersion, { shardTS });
+        // }
     }
-    getKeyFor(msgId, deliveryTS) {
-        return this.indexed.getKeyFor(msgId, deliveryTS);
+    function millisBeforeNextRun() {
+        return 30 * 60 * 1000 + Math.floor(10 * 60 * 1000 * Math.random());
     }
+    const { stopSyncing, changeProc } = (0, per_device_json_array_logs_1.startDatasetSync)(periodicCheckAndUploadOfData, millisBeforeNextRun, {
+        watchAndApplyOpsFromOtherDevices: logs.watchAndApplyOpsFromOtherDevices
+    }, {
+        absorbOpsFromOtherDevices: indexed.absorbOpsFromOtherDevices,
+        watchAndApplyOpsFromOtherDevices: indexed.watchAndApplyOpsFromOtherDevices
+    });
+    // XXX
+    //  - check if indexed latest is too old, chopping to the latest shard
+    return {
+        add: (0, synced_1.makeSyncedFunc)(changeProc, undefined, add),
+        remove: (0, synced_1.makeSyncedFunc)(changeProc, undefined, remove),
+        listMsgs: (0, synced_1.makeSyncedFunc)(changeProc, undefined, indexed.listMsgs),
+        getKeyFor: (0, synced_1.makeSyncedFunc)(changeProc, undefined, indexed.getKeyFor),
+        stopSyncing
+    };
 }
-exports.MsgIndex = MsgIndex;
-Object.freeze(MsgIndex.prototype);
-Object.freeze(MsgIndex);
 Object.freeze(exports);

package/build/core/asmail/inbox/msg-on-disk.js

@@ -75,8 +75,7 @@ class MsgOnDisk {
             throw new Error(`Obj ${objId} not present in message ${this.msgId}`);
         }
         const path = this.objFilePath(objId);
-        const isOnDisk = !!(await fs.stat(path)
-            .catch((exc) => {
+        const isOnDisk = !!(await fs.stat(path).catch((exc) => {
             if (exc.notFound) {
                 return;
             }