core-3nweb-client-lib 0.28.3 → 0.29.0

package/build/core/asmail/inbox/{msg-indexing.d.ts → msg-indexing/index.d.ts}

@@ -1,4 +1,4 @@
-import { MsgKeyInfo, MsgKeyRole } from '../keyring';
+import { MsgKeyInfo, MsgKeyRole } from '../../keyring';
 declare type WritableFS = web3n.files.WritableFS;
 declare type MsgInfo = web3n.asmail.MsgInfo;
 /**
@@ -14,9 +14,8 @@ declare type MsgInfo = web3n.asmail.MsgInfo;
  *
  */
 export declare class MsgIndex {
-    private readonly files;
-    private readonly records;
-    private readonly changes;
+    private readonly logs;
+    private readonly indexed;
     private constructor();
     static make(syncedFS: WritableFS): Promise<MsgIndex>;
     stopSyncing(): void;

package/build/core/asmail/inbox/msg-indexing/index.js

@@ -0,0 +1,96 @@
+"use strict";
+/*
+ Copyright (C) 2022 - 2023 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MsgIndex = void 0;
+const logs_n_entries_1 = require("./logs-n-entries");
+const LOGS_DIR = 'logs';
+const INDEX_DIR = 'index';
+async function getOrMakeDirStructure(syncedFS) {
+    const logsFS = await syncedFS.writableSubRoot(LOGS_DIR);
+    const indexFS = await syncedFS.writableSubRoot(INDEX_DIR);
+    return { logsFS, indexFS };
+}
+async function syncDirStructureIfNeeded(syncedFS) {
+    // XXX need uploading of initial folders, and of syncedFS itself
+    // const logsDirInfo = await logsFS.v!.sync!.status(LOGS_DIR);
+    // logsDirInfo.state
+    // Or, is this enough?
+    // await getRemoteFolderChanges(syncedFS);
+    // await uploadFolderChangesIfAny(syncedFS);
+}
+/**
+ * This message index stores info for messages present on the server, in the
+ * inbox. Records contain message key info, time of delivery, and time of
+ * desired removal.
+ *
+ * Message info with keys is stored in SQLite dbs sharded/partitioned by
+ * delivery timestamp. The latest shard, shard without upper time limit
+ * is stored in local storage, while all other shards with limits are stored in
+ * synced storage. Information in synced storage is a sum of all limited shards
+ * and action logs. Action logs
+ *
+ */
+class MsgIndex {
+    constructor(logs, indexed) {
+        this.logs = logs;
+        this.indexed = indexed;
+        Object.seal(this);
+    }
+    static async make(syncedFS) {
+        const { logsFS, indexFS } = await getOrMakeDirStructure(syncedFS);
+        await syncDirStructureIfNeeded(syncedFS);
+        const logs = await logs_n_entries_1.MsgLogs.makeAndStartSyncing(logsFS);
+        let indexed;
+        if (global.WebAssembly) {
+            indexed = await require('./sql-indexing').makeSqliteBasedIndexedRecords(indexFS);
+        }
+        else {
+            indexed = (0, logs_n_entries_1.makeJsonBasedIndexedRecords)(logs);
+        }
+        const index = new MsgIndex(logs, indexed);
+        return index;
+    }
+    stopSyncing() {
+        this.logs.stopSyncing();
+        this.indexed.stopSyncing();
+    }
+    async add(msgInfo, decrInfo, removeAfter = 0) {
+        const msgAlreadyExists = await this.indexed.msgExists(msgInfo);
+        if (msgAlreadyExists) {
+            return;
+        }
+        await this.logs.add(msgInfo, decrInfo, removeAfter);
+        await this.indexed.add(msgInfo, decrInfo, removeAfter);
+    }
+    async remove(msgId) {
+        const deliveryTS = await this.indexed.remove(msgId);
+        if (deliveryTS) {
+            await this.logs.remove(msgId, deliveryTS);
+        }
+    }
+    listMsgs(fromTS) {
+        return this.indexed.listMsgs(fromTS);
+    }
+    getKeyFor(msgId, deliveryTS) {
+        return this.indexed.getKeyFor(msgId, deliveryTS);
+    }
+}
+exports.MsgIndex = MsgIndex;
+Object.freeze(MsgIndex.prototype);
+Object.freeze(MsgIndex);
+Object.freeze(exports);

package/build/core/asmail/inbox/msg-indexing/logs-n-entries.d.ts

@@ -0,0 +1,52 @@
+import { MsgKeyInfo, MsgKeyRole } from '../../keyring';
+declare type WritableFS = web3n.files.WritableFS;
+declare type MsgInfo = web3n.asmail.MsgInfo;
+interface MsgOpenedLog {
+    msgState: 'opened';
+    msgId: string;
+    msgType: string;
+    deliveryTS: number;
+    keyB64: string;
+    keyStatus: MsgKeyRole;
+    mainObjHeaderOfs: number;
+    removeAfter?: number;
+}
+interface MsgRemovedLog {
+    msgState: 'removed';
+    msgId: string;
+    deliveryTS: number;
+}
+declare type MsgLog = MsgOpenedLog | MsgRemovedLog;
+export interface MsgKey {
+    msgKey: Uint8Array;
+    msgKeyRole: MsgKeyRole;
+    mainObjHeaderOfs: number;
+}
+export interface IndexedRecords {
+    add(msgInfo: MsgInfo, decrInfo: MsgKeyInfo, removeAfter: number): Promise<void>;
+    remove(msgId: string): Promise<number | undefined>;
+    listMsgs(fromTS: number | undefined): Promise<MsgInfo[]>;
+    getKeyFor(msgId: string, deliveryTS: number): Promise<MsgKey | undefined>;
+    msgExists(msgInfo: MsgInfo): Promise<boolean>;
+    stopSyncing(): void;
+}
+export declare function makeJsonBasedIndexedRecords(logs: MsgLogs): IndexedRecords;
+export declare class MsgLogs {
+    private readonly fs;
+    private latest;
+    private fileTSs;
+    private readonly changeProc;
+    private constructor();
+    static makeAndStartSyncing(logsFS: WritableFS): Promise<MsgLogs>;
+    stopSyncing(): void;
+    add(msgInfo: MsgInfo, decrInfo: MsgKeyInfo, removeAfter: number | undefined): Promise<void>;
+    private updateLogsFile;
+    getLogsFromFile(fileTS: number | undefined): Promise<MsgLog[] | undefined>;
+    private fileTSforMsgTS;
+    remove(msgId: string, deliveryTS: number): Promise<void>;
+    private scheduleRemovalOfMsgOpenedLog;
+    getMsgOpenedLog(msgId: string, deliveryTS: number): Promise<MsgOpenedLog | undefined>;
+    getLogFileTSs(): number[];
+    getLatestMsgLogs(): MsgLog[];
+}
+export {};

package/build/core/asmail/inbox/msg-indexing/logs-n-entries.js

@@ -0,0 +1,333 @@
+"use strict";
+/*
+ Copyright (C) 2016 - 2020, 2023 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MsgLogs = exports.makeJsonBasedIndexedRecords = void 0;
+const synced_1 = require("../../../../lib-common/processes/synced");
+const buffer_utils_1 = require("../../../../lib-common/buffer-utils");
+const timed_cache_1 = require("../../../../lib-common/timed-cache");
+const error_1 = require("../../../../lib-common/exceptions/error");
+const LIMIT_RECORDS_PER_FILE = 200;
+const LATEST_LOG = 'latest.json';
+const LOG_EXT = '.json';
+const INDEX_FNAME_REGEXP = /^\d+\.json$/;
+function logsFileName(fileTS) {
+    return (fileTS ? LATEST_LOG : `${fileTS}${LOG_EXT}`);
+}
+function fileTSOrderComparator(a, b) {
+    return (a - b);
+}
+function insertInto(records, rec) {
+    if (records.length === 0) {
+        records.push(rec);
+        return;
+    }
+    const ts = rec.deliveryTS;
+    for (let i = (records.length - 1); i >= 0; i -= 1) {
+        if (records[i].deliveryTS <= ts) {
+            records.splice(i + 1, 0, rec);
+            return;
+        }
+    }
+    records.splice(0, 0, rec);
+}
+function removeMsgOpenedLogFrom(logs, msgId) {
+    const indToRm = logs.findIndex(log => ((log.msgId === msgId) && (log.msgState === 'opened')));
+    if (indToRm > -1) {
+        logs.splice(indToRm, 1);
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+/**
+ * This is a catch callback, which returns undefined on file(folder) not found
+ * exception, and re-throws all other exceptions/errors.
+ */
+function notFoundOrReThrow(exc) {
+    if (!exc.notFound) {
+        throw exc;
+    }
+    return;
+}
+function makeJsonBasedIndexedRecords(logs) {
+    return new JsonBasedIndexedRecords(logs);
+}
+exports.makeJsonBasedIndexedRecords = makeJsonBasedIndexedRecords;
+function toMsgOpenedLog(msgInfo, decrInfo, removeAfter) {
+    if (!decrInfo.key) {
+        throw new Error(`Given message decryption info doesn't have a key for message ${msgInfo.msgId}`);
+    }
+    return {
+        msgState: 'opened',
+        msgType: msgInfo.msgType,
+        msgId: msgInfo.msgId,
+        deliveryTS: msgInfo.deliveryTS,
+        keyB64: buffer_utils_1.base64.pack(decrInfo.key),
+        keyStatus: decrInfo.keyStatus,
+        mainObjHeaderOfs: decrInfo.msgKeyPackLen,
+        removeAfter
+    };
+}
+function keyInfoFrom(log) {
+    return {
+        msgKey: buffer_utils_1.base64.open(log.keyB64),
+        msgKeyRole: log.keyStatus,
+        mainObjHeaderOfs: log.mainObjHeaderOfs
+    };
+}
+function addOnlyNonRemovedMsgs(dst, ignore, src) {
+    for (const log of src) {
+        if (log.msgState === 'removed') {
+            ignore.add(log.msgId);
+        }
+    }
+    for (const log of src) {
+        if ((log.msgState === 'opened') && !ignore.has(log.msgId)) {
+            dst.push(log);
+        }
+    }
+}
+function cutEarlierMsgs(msgs, fromTS) {
+    let timedEnd = msgs.length;
+    for (let i = (msgs.length - 1); i >= 0; i -= 1) {
+        const msg = msgs[i];
+        if (msg.deliveryTS < fromTS) {
+            timedEnd = i;
+        }
+        else {
+            break;
+        }
+    }
+    if (timedEnd < msgs.length) {
+        msgs.splice(timedEnd, (msgs.length - timedEnd));
+    }
+}
+class JsonBasedIndexedRecords {
+    constructor(logs) {
+        this.logs = logs;
+        this.cachedLogs = (0, timed_cache_1.makeTimedCache)(10 * 60 * 1000);
+        Object.seal(this);
+    }
+    async add(msgInfo, decrInfo, removeAfter) {
+        const msg = toMsgOpenedLog(msgInfo, decrInfo, removeAfter);
+        this.cachedLogs.set(msg.msgId, msg);
+    }
+    async remove(msgId) {
+        let foundLog = this.cachedLogs.get(msgId);
+        if (foundLog) {
+            this.cachedLogs.delete(msgId);
+            return foundLog.deliveryTS;
+        }
+        for (const fileTS of [undefined, ...this.logs.getLogFileTSs()]) {
+            let logs;
+            if ((fileTS === undefined)) {
+                logs = this.logs.getLatestMsgLogs();
+            }
+            else {
+                const logsFromFile = await this.logs.getLogsFromFile(fileTS);
+                if (!logsFromFile) {
+                    continue;
+                }
+                logs = logsFromFile;
+            }
+            for (const log of logs) {
+                if (log.msgId === msgId) {
+                    return log.deliveryTS;
+                }
+            }
+        }
+        return undefined;
+    }
+    async listMsgs(fromTS) {
+        const ignore = new Set();
+        const msgs = [];
+        addOnlyNonRemovedMsgs(msgs, ignore, this.logs.getLatestMsgLogs());
+        const fileTSs = this.logs.getLogFileTSs();
+        if (fromTS === undefined) {
+            for (const fileTS of fileTSs) {
+                const logs = await this.logs.getLogsFromFile(fileTS);
+                if (!logs) {
+                    continue;
+                }
+                addOnlyNonRemovedMsgs(msgs, ignore, logs);
+            }
+        }
+        else {
+            for (const fileTS of fileTSs) {
+                if (fromTS > fileTS) {
+                    break;
+                }
+                const logs = await this.logs.getLogsFromFile(fileTS);
+                if (!logs) {
+                    continue;
+                }
+                addOnlyNonRemovedMsgs(msgs, ignore, logs);
+            }
+        }
+        // note sorting later messages to array's head
+        msgs.sort((m1, m2) => (m2.deliveryTS - m1.deliveryTS));
+        if (fromTS !== undefined) {
+            cutEarlierMsgs(msgs, fromTS);
+        }
+        for (const log of msgs) {
+            this.cachedLogs.set(log.msgId, log);
+        }
+        return msgs.map(log => ({
+            msgId: log.msgId,
+            msgType: log.msgType,
+            deliveryTS: log.deliveryTS
+        }));
+    }
+    async getKeyFor(msgId, deliveryTS) {
+        let log = this.cachedLogs.get(msgId);
+        if (log) {
+            return keyInfoFrom(log);
+        }
+        log = await this.logs.getMsgOpenedLog(msgId, deliveryTS);
+        if (!log) {
+            return undefined;
+        }
+        this.cachedLogs.set(log.msgId, log);
+        return keyInfoFrom(log);
+    }
+    async msgExists(msgInfo) {
+        const foundLog = this.cachedLogs.get(msgInfo.msgId);
+        return (!!foundLog && (foundLog.deliveryTS === msgInfo.deliveryTS));
+    }
+    stopSyncing() { }
+}
+Object.freeze(JsonBasedIndexedRecords.prototype);
+Object.freeze(JsonBasedIndexedRecords);
+class MsgLogs {
+    constructor(fs, latest, fileTSs) {
+        this.fs = fs;
+        this.latest = latest;
+        this.fileTSs = fileTSs;
+        this.changeProc = new synced_1.SingleProc();
+        Object.seal(this);
+    }
+    static async makeAndStartSyncing(logsFS) {
+        const fName = logsFileName(undefined);
+        let latest = await logsFS.readJSONFile(fName)
+            .catch(notFoundOrReThrow);
+        if (!latest) {
+            latest = [];
+            await logsFS.writeJSONFile(fName, latest, { create: true, exclusive: true });
+        }
+        const fileTSs = (await logsFS.listFolder('.'))
+            .map(f => f.name)
+            .filter(fName => fName.match(INDEX_FNAME_REGEXP))
+            .map(fName => parseInt(fName.substring(0, fName.length - LOG_EXT.length)))
+            .filter(fileTS => !isNaN(fileTS))
+            .sort(fileTSOrderComparator);
+        const logs = new MsgLogs(logsFS, latest, fileTSs);
+        return logs;
+    }
+    stopSyncing() {
+        // XXX fill this with content
+    }
+    async add(msgInfo, decrInfo, removeAfter) {
+        const msg = toMsgOpenedLog(msgInfo, decrInfo, removeAfter);
+        return this.changeProc.startOrChain(async () => {
+            const fileTS = this.fileTSforMsgTS(msg.deliveryTS);
+            const logs = ((fileTS === undefined) ?
+                this.latest : await this.getLogsFromFile(fileTS));
+            if (!logs) {
+                throw (0, error_1.errWithCause)(`${logsFileName(fileTS)} not found`, `Expectation fail: there should be some message records.`);
+            }
+            insertInto(logs, msg);
+            await this.updateLogsFile(fileTS, logs);
+        });
+    }
+    async updateLogsFile(fileTS, logs) {
+        await this.fs.writeJSONFile(logsFileName(fileTS), logs, { create: false });
+    }
+    async getLogsFromFile(fileTS) {
+        const fName = logsFileName(undefined);
+        const logs = await this.fs.readJSONFile(fName)
+            .catch(notFoundOrReThrow);
+        return logs;
+    }
+    fileTSforMsgTS(deliveryTS) {
+        if ((this.fileTSs.length === 0)
+            || (deliveryTS >= this.fileTSs[this.fileTSs.length - 1])) {
+            return undefined;
+        }
+        let fileTS = this.fileTSs[this.fileTSs.length - 1];
+        for (let i = (this.fileTSs.length - 2); i <= 0; i -= 1) {
+            if (deliveryTS >= this.fileTSs[i]) {
+                break;
+            }
+            else {
+                fileTS = this.fileTSs[i];
+            }
+        }
+        return fileTS;
+    }
+    async remove(msgId, deliveryTS) {
+        const msgRmLog = {
+            msgState: 'removed',
+            msgId,
+            deliveryTS
+        };
+        return this.changeProc.startOrChain(async () => {
+            this.latest.push(msgRmLog);
+            const fileTS = this.fileTSforMsgTS(deliveryTS);
+            if (fileTS === undefined) {
+                removeMsgOpenedLogFrom(this.latest, msgId);
+            }
+            await this.updateLogsFile(undefined, this.latest);
+            if (fileTS !== undefined) {
+                this.scheduleRemovalOfMsgOpenedLog(fileTS, msgId);
+            }
+        });
+    }
+    scheduleRemovalOfMsgOpenedLog(fileTS, msgId) {
+        this.changeProc.startOrChain(async () => {
+            const logs = await this.getLogsFromFile(fileTS);
+            if (!logs) {
+                return;
+            }
+            const changed = removeMsgOpenedLogFrom(logs, msgId);
+            if (!changed) {
+                return;
+            }
+            await this.updateLogsFile(fileTS, logs);
+        });
+    }
+    async getMsgOpenedLog(msgId, deliveryTS) {
+        const fileTS = this.fileTSforMsgTS(deliveryTS);
+        const logs = ((fileTS === undefined) ?
+            this.latest : await this.getLogsFromFile(fileTS));
+        if (!logs) {
+            return undefined;
+        }
+        return logs.find(log => ((log.msgId === msgId) && (log.msgState === 'opened')));
+    }
+    getLogFileTSs() {
+        return this.fileTSs.concat([]);
+    }
+    getLatestMsgLogs() {
+        return this.latest.concat([]);
+    }
+}
+exports.MsgLogs = MsgLogs;
+Object.freeze(MsgLogs.prototype);
+Object.freeze(MsgLogs);
+Object.freeze(exports);

package/build/core/asmail/inbox/msg-indexing/sql-indexing.d.ts

@@ -0,0 +1,4 @@
+import { IndexedRecords } from './logs-n-entries';
+declare type WritableFS = web3n.files.WritableFS;
+export declare function makeSqliteBasedIndexedRecords(syncedFS: WritableFS): Promise<IndexedRecords>;
+export {};