core-3nweb-client-lib 0.27.4 → 0.27.6

package/build/api-defs/files.d.ts

@@ -44,6 +44,7 @@ declare namespace web3n.files {
 		versionMismatch?: true;
 		isEndless?: true;
 		storageClosed?: true;
+		remoteNotSet?: true;
 	}
 
 	interface exceptionCode {
@@ -192,7 +193,17 @@ declare namespace web3n.files {
 		 * @param len maximum number of bytes to read from file. If undefine is
 		 * given, all bytes are read from current postion to the end of file.
 		 */
-		read(len: number|undefined): Promise<Uint8Array|undefined>;
+		readNext(len: number|undefined): Promise<Uint8Array|undefined>;
+
+		/**
+		 * This moves current position to a given value and read given number of
+		 * bytes. It is equivalent to calling seek() and readNext() with
+		 * respective arguments.
+		 * @param pos is a position at which to start read.
+		 * @param len maximum number of bytes to read from file. If undefine is
+		 * given, all bytes are read from current postion to the end of file.
+		 */
+		readAt(pos: number, len: number|undefined): Promise<Uint8Array|undefined>;
 
 		/**
 		 * This returns a promise, resolvable to the size of this file.
@@ -264,7 +275,8 @@ declare namespace web3n.files {
 		 * sink with this error. When err is given, no errors will be thrown back
 		 * to this call.
 		 */
-		done(err?: any): Promise<void>;
+		done(err?: any, xattrChanges?: XAttrsChanges): Promise<void>;
+
 	}
 
 	type Linkable = File | FS;
@@ -382,20 +394,28 @@ declare namespace web3n.files {
 		/**
 		 * This returns a promise, resolvable when file is written
 		 * @param bytes is a complete file content to write
+		 * @param xattrChanges is optional changes to xattrs, to pack them in the
+		 * same go
 		 */
-		writeBytes(bytes: Uint8Array): Promise<void>;
+		writeBytes(
+			bytes: Uint8Array, xattrChanges?: XAttrsChanges
+		): Promise<void>;
 
 		/**
 		 * This returns a promise, resolvable when file is written
 		 * @param txt to write to file, using utf8 encoding
+		 * @param xattrChanges is optional changes to xattrs, to pack them in the
+		 * same go
 		 */
-		writeTxt(txt: string): Promise<void>;
+		writeTxt(txt: string, xattrChanges?: XAttrsChanges): Promise<void>;
 
 		/**
 		 * This returns a promise, resolvable when file is written
 		 * @param json
+		 * @param xattrChanges is optional changes to xattrs, to pack them in the
+		 * same go
 		 */
-		writeJSON(json: any): Promise<void>;
+		writeJSON(json: any, xattrChanges?: XAttrsChanges): Promise<void>;
 
 		/**
 		 * This returns a promise, resolvable to byte sink with seek
@@ -495,22 +515,30 @@ declare namespace web3n.files {
 		 * This returns a promise, resolvable to new file's version when file is
 		 * written
 		 * @param bytes is a complete file content to write
+		 * @param xattrChanges is optional changes to xattrs, to pack them into
+		 * the same file version
 		 */
-		writeBytes(bytes: Uint8Array): Promise<number>;
+		writeBytes(
+			bytes: Uint8Array, xattrChanges?: XAttrsChanges
+		): Promise<number>;
 
 		/**
 		 * This returns a promise, resolvable to new file's version when file is
 		 * written
 		 * @param txt to write to file, using utf8 encoding
+		 * @param xattrChanges is optional changes to xattrs, to pack them into
+		 * the same file version
 		 */
-		writeTxt(txt: string): Promise<number>;
+		writeTxt(txt: string, xattrChanges?: XAttrsChanges): Promise<number>;
 
 		/**
 		 * This returns a promise, resolvable to new file's version when file is
 		 * written
 		 * @param json
+		 * @param xattrChanges is optional changes to xattrs, to pack them into
+		 * the same file version
 		 */
-		writeJSON(json: any): Promise<number>;
+		writeJSON(json: any, xattrChanges?: XAttrsChanges): Promise<number>;
 
 		/**
 		 * This returns a promise, resolvable to byte sink with seek, and a file
@@ -567,7 +595,7 @@ declare namespace web3n.files {
 
 	interface WritableFileSyncAPI extends ReadonlyFileSyncAPI {
 
-		upload(opts?: OptionsToUploadLocal): Promise<void>;
+		upload(opts?: OptionsToUploadLocal): Promise<number|undefined>;
 
 	}
 
@@ -1191,7 +1219,9 @@ declare namespace web3n.files {
 
 	interface WritableFSSyncAPI extends ReadonlyFSSyncAPI {
 
-		upload(path: string, opts?: OptionsToUploadLocal): Promise<void>;
+		upload(
+			path: string, opts?: OptionsToUploadLocal
+		): Promise<number|undefined>;
 
 		adoptRemoteFolderItem(
 			path: string, itemName: string, opts?: OptionsToAdoptRemoteItem

package/build/core/asmail/delivery/per-recipient-wip.js

@@ -370,7 +370,7 @@ class WIP {
         if (!isEndless) {
             obj.upload.expectedSegsSize = size;
         }
-        let segsChunk = await obj.src.segSrc.read(this.sender.maxChunkSize - header.length);
+        let segsChunk = await obj.src.segSrc.readNext(this.sender.maxChunkSize - header.length);
         // check if we'll be done in this request
         const segsSize = obj.upload.expectedSegsSize;
         let isObjDone = false;
@@ -406,7 +406,7 @@ class WIP {
     async followingObjSendingRequest(obj) {
         // read from source
         const segsSize = obj.upload.expectedSegsSize;
-        const chunk = await obj.src.segSrc.read(this.sender.maxChunkSize);
+        const chunk = await obj.src.segSrc.readNext(this.sender.maxChunkSize);
         // return early, when there are no bytes
         if (!chunk) {
             await this.sender.sendObj(obj.objId, EMPTY_BYTE_ARR, undefined, { ofs: obj.upload.segsOffset, last: true });

package/build/core/asmail/inbox/inbox-events.js

@@ -30,11 +30,16 @@ class InboxEvents {
     constructor(msgReceiver, getMsg, logError) {
         const serverEvents = new server_events_1.ServerEvents(() => msgReceiver.openEventSource(), SERVER_EVENTS_RESTART_WAIT_SECS);
         this.newMsg$ = serverEvents.observe(retrieval_1.msgRecievedCompletely.EVENT_NAME)
-            .pipe((0, operators_1.mergeMap)(ev => getMsg(ev.msgId)
-            .catch(async (err) => {
-            // TODO should more error handling logic be added here?
-            await logError(err, `Cannot get message ${ev.msgId}`);
-        })), (0, operators_1.filter)(msg => !!msg), (0, operators_1.share)());
+            .pipe((0, operators_1.mergeMap)(async (ev) => {
+            try {
+                const msg = await getMsg(ev.msgId);
+                return msg;
+            }
+            catch (err) {
+                // TODO should more error handling logic be added here?
+                await logError(err, `Cannot get message ${ev.msgId}`);
+            }
+        }), (0, operators_1.filter)(msg => !!msg), (0, operators_1.share)());
         Object.seal(this);
     }
     subscribe(event, observer) {

package/build/core/asmail/inbox/index.d.ts

@@ -38,7 +38,7 @@ export interface ResourcesForReceiving {
          * object.
          * @param checkMidKeyCerts is a certifying function for MailerId certs.
          */
-        msgDecryptor: (msgMeta: delivApi.msgMeta.CryptoInfo, getMainObjHeader: () => Promise<Uint8Array>, getOpenedMsg: (mainObjFileKey: string, msgKeyPackLen: number) => Promise<OpenedMsg>, checkMidKeyCerts: (certs: confApi.p.initPubKey.Certs) => Promise<{
+        msgDecryptor: (msgMeta: delivApi.msgMeta.CryptoInfo, getMainObjHeader: () => Promise<Uint8Array>, getOpenedMsg: (mainObjFileKey: Uint8Array, msgKeyPackLen: number) => Promise<OpenedMsg>, checkMidKeyCerts: (certs: confApi.p.initPubKey.Certs) => Promise<{
             pkey: JsonKey;
             address: string;
         }>) => Promise<{
@@ -83,7 +83,8 @@ export declare class InboxOnServer {
     private readonly procs;
     private readonly recentlyOpenedMsgs;
     private constructor();
-    static makeAndStart(cachePath: string, fs: WritableFS, r: ResourcesForReceiving): Promise<InboxOnServer>;
+    static makeAndStart(cachePath: string, syncedFS: WritableFS, r: ResourcesForReceiving): Promise<InboxOnServer>;
+    close(): Promise<void>;
     wrap(): InboxService;
     private removeMsg;
     private removeMsgFromServerAndCache;

package/build/core/asmail/inbox/index.js

@@ -24,13 +24,14 @@ const opener_1 = require("../msg/opener");
 const msg_indexing_1 = require("./msg-indexing");
 const fs_1 = require("./attachments/fs");
 const canonical_address_1 = require("../../../lib-common/canonical-address");
-const buffer_utils_1 = require("../../../lib-common/buffer-utils");
 const key_verification_1 = require("../key-verification");
 const inbox_events_1 = require("./inbox-events");
 const file_1 = require("../../../lib-common/exceptions/file");
 const msg_downloader_1 = require("./msg-downloader");
 const cached_msgs_1 = require("./cached-msgs");
 const timed_cache_1 = require("../../../lib-common/timed-cache");
+const fs_sync_utils_1 = require("../../../lib-client/fs-sync-utils");
+const MSG_INDEX_FOLDER = 'msg-index';
 /**
  * Instance of this class represents inbox-on-mail-server.
  * It uses api to manage messages on a ASMail server, caching and recording
@@ -53,19 +54,24 @@ class InboxOnServer {
         this.inboxEvents = new inbox_events_1.InboxEvents(this.msgReceiver, this.getMsg.bind(this), this.logError);
         Object.seal(this);
     }
-    static async makeAndStart(cachePath, fs, r) {
+    static async makeAndStart(cachePath, syncedFS, r) {
         try {
-            (0, file_1.ensureCorrectFS)(fs, 'synced', true);
+            (0, file_1.ensureCorrectFS)(syncedFS, 'synced', true);
             const msgReceiver = new recipient_1.MailRecipient(r.address, r.getSigner, () => r.asmailResolver(r.address), r.makeNet());
             const downloader = new msg_downloader_1.MsgDownloader(msgReceiver);
             const cache = await cached_msgs_1.CachedMessages.makeFor(cachePath, downloader, r.logError);
-            const index = await msg_indexing_1.MsgIndex.makeWith(fs);
+            const indexSyncedFS = await (0, fs_sync_utils_1.getOrMakeAndUploadFolderIn)(syncedFS, MSG_INDEX_FOLDER);
+            const index = await msg_indexing_1.MsgIndex.make(indexSyncedFS);
+            await (0, fs_sync_utils_1.uploadFolderChangesIfAny)(syncedFS);
             return new InboxOnServer(r.correspondents, msgReceiver, r.getStorages, r.cryptor, downloader, cache, index, r.logError);
         }
         catch (err) {
             throw (0, error_1.errWithCause)(err, 'Failed to initialize Inbox');
         }
     }
+    async close() {
+        this.index.stopSyncing();
+    }
     wrap() {
         const service = {
             getMsg: this.getMsg.bind(this),
@@ -85,7 +91,7 @@ class InboxOnServer {
         // start removal process
         return this.procs.start(procId, (async () => {
             await Promise.all([
-                this.index.removeUsingIdOnly(msgId),
+                this.index.remove(msgId),
                 this.removeMsgFromServerAndCache(msgId)
             ]);
         }));
@@ -121,9 +127,7 @@ class InboxOnServer {
                 if (!mainObjSrc) {
                     mainObjSrc = await msgOnDisk.getMsgObj(mainObjId);
                 }
-                const fKey = buffer_utils_1.base64.open(mainObjFileKey);
-                const openedMsg = await (0, opener_1.openMsg)(msgId, mainObjId, mainObjSrc, msgKeyPackLen, fKey, this.cryptor);
-                fKey.fill(0);
+                const openedMsg = await (0, opener_1.openMsg)(msgId, mainObjId, mainObjSrc, msgKeyPackLen, mainObjFileKey, this.cryptor);
                 return openedMsg;
             };
             const checkMidKeyCerts = (certs) => {
@@ -149,7 +153,7 @@ class InboxOnServer {
             }
             else {
                 // check, if msg has already been indexed
-                const knownDecr = await this.index.fKeyFor({ msgId, deliveryTS: msgOnDisk.deliveryTS });
+                const knownDecr = await this.index.getKeyFor(msgId, msgOnDisk.deliveryTS);
                 if (!knownDecr) {
                     await msgOnDisk.updateMsgKeyStatus('not-found');
                     return false;
@@ -263,7 +267,7 @@ class InboxOnServer {
             const meta = await msgOnDisk.getMsgMeta();
             const mainObjId = meta.extMeta.objIds[0];
             const mainObj = await msgOnDisk.getMsgObj(mainObjId);
-            const msgKey = await this.index.fKeyFor({ msgId, deliveryTS: meta.deliveryCompletion });
+            const msgKey = await this.index.getKeyFor(msgId, meta.deliveryCompletion);
             if (!msgKey) {
                 throw (0, recipient_1.makeMsgNotFoundException)(msgId);
             }

package/build/core/asmail/inbox/msg-indexing.d.ts

@@ -1,76 +1,29 @@
 import { MsgKeyInfo, MsgKeyRole } from '../keyring';
 declare type WritableFS = web3n.files.WritableFS;
-interface MsgInfoWithoutType {
-    msgId: string;
-    deliveryTS: number;
-}
+declare type MsgInfo = web3n.asmail.MsgInfo;
 /**
- * This message index stores MsgRecord's for message present on the server, i.e.
+ * This message index stores info for messages present on the server, in the
  * inbox. Records contain message key info, time of delivery, and time of
- * desired removal. Note that when user wants to keep a particular message for
- * a long time, it should be copied elsewhere, and not kept in the asmail
- * server inbox. Therefore, this index has to deal only with more-or-less
- * recent messages.
+ * desired removal.
  *
- * This implementation is not using database for records, but a log-like files.
- * Latest file is appended to certain length, and then is left to a mostly-read
- * existence, carrying messages' info till message removal.
- * When lookup is done only with message id, all files potentially need to be
- * read. When both message id and delivery timestamp are given, lookup will at
- * most touch two-three files.
+ * Message info with keys is stored in SQLite dbs sharded/partitioned by
+ * delivery timestamp. The latest shard, shard without upper time limit
+ * is stored in local storage, while all other shards with limits are stored in
+ * synced storage. Information in synced storage is a sum of all limited shards
+ * and action logs. Action logs
  *
  */
 export declare class MsgIndex {
-    private files;
-    private latest;
-    private readonly cached;
-    private fileTSs;
-    private readonly fileProc;
+    private readonly files;
+    private readonly records;
+    private readonly changes;
     private constructor();
-    static makeWith(files: WritableFS): Promise<MsgIndex>;
-    /**
-     * @param recs that need to be saved
-     * @return a promise, resolvable when given records are saved. Latest
-     * records can also be chunked and saved as needed.
-     */
-    private saveRecords;
-    /**
-     * @param fileTS is index file's timestamp. Undefined value means latest
-     * records.
-     * @return a promise, resolvable to found records, either from a memory
-     * cache, or from a file.
-     */
-    private getRecords;
-    /**
-     * @param msgInfo is a minimal message info object
-     * @param decrInfo
-     * @return a promise, resolvable when given message info bits are recorded.
-     */
-    add(msgInfo: web3n.asmail.MsgInfo, decrInfo: MsgKeyInfo): Promise<void>;
-    private findRecordsWith;
-    /**
-     * @param msg is a message identifying info used to find and remove message
-     * @return a promise, resolvable when message is removed from this index.
-     */
-    remove(msg: MsgInfoWithoutType): Promise<void>;
-    /**
-     * @param msg is an id of a message to remove
-     * @return a promise, resolvable when message is removed from this index.
-     */
-    removeUsingIdOnly(msgId: string): Promise<void>;
-    /**
-     * @param fromTS
-     * @return a promise, resolvable to an ordered array of MsgInfo's that have
-     * delivery timestamp same or later, than the given one.
-     */
-    listMsgs(fromTS: number | undefined): Promise<web3n.asmail.MsgInfo[]>;
-    /**
-     * This returns a promise resolvable to message's file key holder and key
-     * role, when message is found, and resolvabel to an undefined, when message
-     * is not known.
-     * @param msg is a message identifying info used to find message
-     */
-    fKeyFor(msg: MsgInfoWithoutType): Promise<{
+    static make(syncedFS: WritableFS): Promise<MsgIndex>;
+    stopSyncing(): void;
+    add(msgInfo: MsgInfo, decrInfo: MsgKeyInfo, removeAfter?: number): Promise<void>;
+    remove(msgId: string): Promise<void>;
+    listMsgs(fromTS: number | undefined): Promise<MsgInfo[]>;
+    getKeyFor(msgId: string, deliveryTS: number): Promise<{
         msgKey: Uint8Array;
         msgKeyRole: MsgKeyRole;
         mainObjHeaderOfs: number;