mosquito-transport-js 0.3.0 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mosquito-transport-js",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Javascript web sdk for mosquito-transport (https://github.com/deflexable/mosquito-transport)",
   "main": "src/index.js",
   "type": "module",
@@ -32,17 +32,17 @@
     "bson": "^6.8.0",
     "buffer": "^6.0.3",
     "crypto-js": "^4.2.0",
-    "guard-object": "^1.1.2",
+    "guard-object": "^1.1.3",
+    "limit-task": "1.0.0",
     "json-buffer": "^3.0.1",
     "lodash.clonedeep": "^4.5.0",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",
     "lodash.unset": "^4.5.2",
-    "set-large-timeout": "^1.0.1",
     "simplify-error": "^1.0.1",
     "socket.io-client": "^4.6.2",
     "subscription-listener": "^1.1.2",
-    "tweetnacl": "^1.0.3"
+    "tweetnacl-functional": "^1.0.4"
   },
   "devDependencies": {
     "@babel/core": "^7.24.6",

package/src/helpers/e2e_worker.js

@@ -0,0 +1,109 @@
+import LimitTask from "limit-task";
+import naclPkg from 'tweetnacl-functional';
+
+function e2e_baseCode() {
+    const serializeE2E = (data, serverPublicKey) => {
+        const pair = box.keyPair(),
+            nonce = randomBytes(box.nonceLength);
+
+        return {
+            data: box(data, nonce, serverPublicKey, pair.secretKey),
+            pair,
+            nonce
+        };
+    };
+
+    const deserializeE2E = (data, nonce, serverPublicKey, clientPrivateKey) => {
+        const result = box.open(
+            data,
+            nonce,
+            serverPublicKey,
+            clientPrivateKey
+        );
+
+        if (!result) throw 'Decrypting e2e message failed';
+        return result;
+    };
+
+    self.onmessage = function (event) {
+        const { data } = event;
+
+        try {
+            let response;
+            if (data.type === 'encrypt') {
+                response = serializeE2E(...data.params);
+            } else if (data.type === 'decrypt') {
+                response = deserializeE2E(...data.params);
+            }
+            self.postMessage([response, data.session]);
+        } catch (error) {
+            self.postMessage([undefined, undefined, { error }]);
+        }
+    };
+}
+
+const workerCode = `
+   ${naclPkg.NACL.toString()}
+   const naclPkg = {};
+
+   NACL(naclPkg);
+
+   const { box, randomBytes } = naclPkg;
+
+   ${e2e_baseCode.toString()}
+   e2e_baseCode();
+`;
+
+const spawnWorker = () => {
+    const resolution = {};
+    let ite = 0;
+    const blob = new Blob([workerCode], { type: 'application/javascript' });
+    const worker = new Worker(URL.createObjectURL(blob));
+
+    // Receive the result from the worker
+    worker.onmessage = function (event) {
+        const [response, session, error] = event.data;
+        if (error) {
+            resolution[session][1](error.error);
+        } else {
+            resolution[session][0](response);
+        }
+        delete resolution[session];
+    };
+
+    const queue = LimitTask(3);
+
+    return (type, params) => queue(() =>
+        new Promise((resolve, reject) => {
+            const session = `${++ite}`;
+
+            resolution[session] = [resolve, reject];
+            worker.postMessage({ type, params, session });
+        })
+    );
+};
+
+let currentTask = -1;
+const e2e_engines = [];
+
+const MAX_WORKERS = 7;
+
+const addTask = (type, params) => {
+    ++currentTask;
+    if (currentTask >= e2e_engines.length) {
+        if (e2e_engines.length < MAX_WORKERS) {
+            // spawn new engine
+            e2e_engines.push(spawnWorker());
+        } else currentTask = 0;
+    }
+    return e2e_engines[currentTask](type, params);
+}
+
+export default {
+    encrypt: (bufferData, serverPublicKey) => {
+        return addTask('encrypt', [bufferData, serverPublicKey]);
+    },
+    decrypt: (data, nonce, serverPublicKey, clientPrivateKey) => {
+        return addTask('decrypt', [data, nonce, serverPublicKey, clientPrivateKey]);
+    }
+};

package/src/helpers/peripherals.js

@@ -2,8 +2,9 @@ import { Buffer } from "buffer";
 import { ServerReachableListener } from "./listeners";
 import aes_pkg from 'crypto-js/aes.js';
 import Utf8Encoder from 'crypto-js/enc-utf8.js';
-import naclPkg from 'tweetnacl';
+import naclPkg from 'tweetnacl-functional';
 import getLodash from "lodash.get";
+import e2e_worker from "./e2e_worker";
 
 const { encrypt, decrypt } = aes_pkg;
 const { box, randomBytes } = naclPkg;
@@ -36,6 +37,10 @@ export const niceTry = (promise) => new Promise(async resolve => {
     } catch (e) { resolve(); }
 });
 
+export const normalizeRoute = (route = '') => route.split('').map((v, i, a) =>
+    ((!i && v === '/') || (i === a.length - 1 && v === '/') || (i && a[i - 1] === '/' && v === '/')) ? '' : v
+).join('');
+
 export const shuffleArray = (n) => {
     const array = [...n];
     let currentIndex = array.length, randomIndex;
@@ -98,7 +103,21 @@ export const decryptString = (txt, password, iv) => {
     return decrypt(txt, `${password || ''}${iv || ''}`).toString(Utf8Encoder);
 };
 
-export const serializeE2E = (data, auth_token, serverPublicKey) => {
+export const serializeE2E = async (data, auth_token, serverPublicKey) => {
+    const inputData = new Uint8Array(Buffer.from(JSON.stringify([data, auth_token]), 'utf8'));
+
+    if (inputData.byteLength > 10240) {
+        // dispatch to background thread
+        const { data, pair, nonce } = e2e_worker.encrypt(inputData, serverPublicKey);
+        const pubBase64 = Buffer.from(pair.publicKey).toString('base64'),
+            nonceBase64 = Buffer.from(nonce).toString('base64');
+
+        return [
+            `${pubBase64}.${nonceBase64}.${Buffer.from(data).toString('base64')}`,
+            [pair.secretKey, pair.publicKey]
+        ];
+    }
+
     const pair = box.keyPair(),
         nonce = randomBytes(box.nonceLength),
         pubBase64 = Buffer.from(pair.publicKey).toString('base64'),
@@ -107,12 +126,9 @@ export const serializeE2E = (data, auth_token, serverPublicKey) => {
     return [
         `${pubBase64}.${nonceBase64}.${Buffer.from(
             box(
-                Buffer.from(JSON.stringify([
-                    data,
-                    auth_token
-                ]), 'utf8'),
+                inputData,
                 nonce,
-                Buffer.from(serverPublicKey, 'base64'),
+                serverPublicKey,
                 pair.secretKey
             )
         ).toString('base64')}`,
@@ -120,14 +136,16 @@ export const serializeE2E = (data, auth_token, serverPublicKey) => {
     ];
 };
 
-export const deserializeE2E = (data, serverPublicKey, clientPrivateKey) => {
-    const [binaryNonce, binaryData] = data.split('.'),
-        baseArray = box.open(
-            Buffer.from(binaryData, 'base64'),
-            Buffer.from(binaryNonce, 'base64'),
-            Buffer.from(serverPublicKey, 'base64'),
-            clientPrivateKey
-        );
+export const deserializeE2E = async (data = '', serverPublicKey, clientPrivateKey) => {
+    const [binaryNonce, binaryData] = data.split('.').map(v => new Uint8Array(Buffer.from(v, 'base64')));
+    let baseArray;
+
+    if (binaryData.byteLength > 10240) {
+        // dispatch to background thread
+        baseArray = e2e_worker.decrypt(binaryData, binaryNonce, serverPublicKey, clientPrivateKey);
+    } else {
+        baseArray = box.open(binaryData, binaryNonce, serverPublicKey, clientPrivateKey);
+    }
 
     if (!baseArray) throw 'Decrypting e2e message failed';
     return JSON.parse(Buffer.from(baseArray).toString('utf8'))[0];

package/src/helpers/utils.js

@@ -116,9 +116,9 @@ export const getReachableServer = (projectUrl) => new Promise(resolve => {
     }, true);
 });
 
-export const buildFetchInterface = ({ body, accessKey, authToken, method, uglify, serverE2E_PublicKey }) => {
+export const buildFetchInterface = async ({ body, accessKey, authToken, method, uglify, serverE2E_PublicKey }) => {
     if (!uglify) body = JSON.stringify({ ...body });
-    const [plate, keyPair] = uglify ? serializeE2E(body, authToken, serverE2E_PublicKey) : [undefined, []];
+    const [plate, keyPair] = uglify ? await serializeE2E(body, authToken, serverE2E_PublicKey) : [undefined, []];
 
     return [{
         body: uglify ? plate : body,

package/src/index.d.ts

@@ -115,7 +115,7 @@ interface MTSocket {
         emitWithAck: (...args: any) => Promise<any>;
     });
     emit: (...args: any) => void;
-    emitWithAck: () => Promise<any>;
+    emitWithAck: (...args: any) => Promise<any>;
     on: (route: string, callback?: () => any) => void;
     once: (route: string, callback?: () => any) => void;
     destroy: () => void;
@@ -380,7 +380,7 @@ interface MTAuth {
     getAuthToken: () => Promise<string>;
     getRefreshToken: () => Promise<string>;
     getRefreshTokenData: () => Promise<RefreshTokenData>;
-    parseToken: () => string;
+    parseToken: (token: string) => AuthData | RefreshTokenData;
     listenAuth: (callback: (auth: TokenEventData) => void) => () => void;
     getAuth: () => Promise<TokenEventData>;
     signOut: () => Promise<void>;

package/src/index.js

@@ -1,3 +1,4 @@
+import './polyfill';
 import { deserializeE2E, listenReachableServer, serializeE2E } from "./helpers/peripherals";
 import { releaseCacheStore, awaitStore } from "./helpers/utils";
 import { CacheStore, Scoped } from "./helpers/variables";
@@ -5,7 +6,7 @@ import { MTAuth } from "./products/auth";
 import { MTCollection, batchWrite, trySendPendingWrite } from "./products/database";
 import { MTStorage } from "./products/storage";
 import { ServerReachableListener, TokenRefreshListener } from "./helpers/listeners";
-import { initTokenRefresher, listenToken, listenTokenReady, triggerAuthToken } from "./products/auth/accessor";
+import { initTokenRefresher, listenToken, listenTokenReady, parseToken, triggerAuthToken } from "./products/auth/accessor";
 import { TIMESTAMP, DOCUMENT_EXTRACTION, FIND_GEO_JSON, GEO_JSON } from "./products/database/types";
 import { mfetch } from "./products/http_callable";
 import { io } from "socket.io-client";
@@ -15,6 +16,7 @@ import { parse, stringify } from 'json-buffer';
 import { Validator } from 'guard-object';
 import sendMessage from "./helpers/broadcaster";
 import cloneDeep from "lodash.clonedeep";
+import { Buffer } from "buffer";
 
 const {
     _listenCollection,
@@ -24,11 +26,22 @@ const {
     _listenUserVerification
 } = EngineApi;
 
+// https://socket.io/docs/v3/emit-cheatsheet/#reserved-events
+const reservedEventName = [
+    'connect',
+    'connect_error',
+    'disconnect',
+    'disconnecting',
+    'newListener',
+    'removeListener'
+];
+
 export class MosquitoTransport {
     constructor(config) {
         validateMTConfig(config, this);
         this.config = {
             ...config,
+            serverE2E_PublicKey: config.serverE2E_PublicKey && new Uint8Array(Buffer.from(config.serverE2E_PublicKey, 'base64')),
             castBSON: config.castBSON === undefined || config.castBSON,
             maxRetries: config.maxRetries || 3,
             uglify: config.enableE2E_Encryption
@@ -166,20 +179,25 @@ export class MosquitoTransport {
             tokenListener,
             clientPrivateKey;
 
-        const listenerCallback = (callback) => function () {
+        const listenerCallback = (route, callback) => async function () {
+            if (reservedEventName.includes(route)) {
+                callback?.(...[...arguments]);
+                return;
+            }
+
             const [args, emitable] = [...arguments];
             let res;
 
             if (uglify) {
-                res = parse(deserializeE2E(args, serverE2E_PublicKey, clientPrivateKey));
+                res = parse(await deserializeE2E(args, serverE2E_PublicKey, clientPrivateKey));
             } else res = args;
 
-            callback?.(...res || [], ...typeof emitable === 'function' ? [function () {
+            callback?.(...res || [], ...typeof emitable === 'function' ? [async function () {
                 const args = [...arguments];
                 let res;
 
                 if (uglify) {
-                    res = serializeE2E(stringify(args), undefined, serverE2E_PublicKey)[0];
+                    res = (await serializeE2E(stringify(args), undefined, serverE2E_PublicKey))[0];
                 } else res = args;
 
                 emitable(res);
@@ -213,26 +231,26 @@ export class MosquitoTransport {
                 const hasEmitable = typeof lastEmit === 'function';
                 const mit = hasEmitable ? emittion.slice(0, -1) : emittion;
 
-                const [reqBuilder, [privateKey]] = uglify ? serializeE2E(stringify(mit), undefined, serverE2E_PublicKey) : [undefined, []];
+                const [reqBuilder, [privateKey]] = uglify ? await serializeE2E(stringify(mit), undefined, serverE2E_PublicKey) : [undefined, []];
 
                 if (hasEmitable && promise)
                     throw 'emitWithAck cannot have function in it argument';
 
                 const result = await thisSocket[promise ? 'emitWithAck' : 'emit'](route,
                     uglify ? reqBuilder : mit,
-                    ...hasEmitable ? [function () {
+                    ...hasEmitable ? [async function () {
                         const [args] = [...arguments];
                         let res;
 
                         if (uglify) {
-                            res = parse(deserializeE2E(args, serverE2E_PublicKey, privateKey));
+                            res = parse(await deserializeE2E(args, serverE2E_PublicKey, privateKey));
                         } else res = args;
 
                         lastEmit(...res || []);
                     }] : []
                 );
 
-                resolve((promise && result) ? uglify ? parse(deserializeE2E(result, serverE2E_PublicKey, privateKey))[0] : result[0] : undefined);
+                resolve((promise && result) ? uglify ? parse(await deserializeE2E(result, serverE2E_PublicKey, privateKey))[0] : result[0] : undefined);
             } catch (e) {
                 reject(e);
             }
@@ -241,7 +259,7 @@ export class MosquitoTransport {
         const init = async () => {
             if (hasCancelled) return;
             const mtoken = disableAuth ? undefined : Scoped.AuthJWTToken[projectUrl];
-            const [reqBuilder, [privateKey]] = uglify ? serializeE2E({ accessKey, a_extras: authHandshake }, mtoken, serverE2E_PublicKey) : [null, []];
+            const [reqBuilder, [privateKey]] = uglify ? await serializeE2E({ accessKey, a_extras: authHandshake }, mtoken, serverE2E_PublicKey) : [null, []];
 
             socket = io(`${wsPrefix}://${projectUrl.split('://')[1]}`, {
                 transports: ['websocket', 'polling', 'flashsocket'],
@@ -307,7 +325,7 @@ export class MosquitoTransport {
                 if (restrictedRoute.includes(route))
                     throw `${route} is a restricted socket path, avoid using any of ${restrictedRoute}`;
                 const ref = ++socketListenerIte,
-                    listener = listenerCallback(callback);
+                    listener = listenerCallback(route, callback);
 
                 socketListenerList.push([ref, 'on', route, listener]);
                 if (socket) socket.on(route, listener);
@@ -321,7 +339,7 @@ export class MosquitoTransport {
                 if (restrictedRoute.includes(route))
                     throw `${route} is a restricted socket path, avoid using any of ${restrictedRoute}`;
                 const ref = ++socketListenerIte,
-                    listener = listenerCallback(callback);
+                    listener = listenerCallback(route, callback);
 
                 socketListenerList.push([ref, 'once', route, listener]);
                 if (socket) socket.once(route, listener);

package/src/polyfill.js

@@ -0,0 +1,3 @@
+import { Buffer } from "buffer";
+
+if (!globalThis.Buffer) globalThis.Buffer = Buffer;

package/src/products/auth/accessor.js

@@ -81,7 +81,7 @@ const refreshToken = (builder, processRef, remainRetries = 7, initialRetries = 7
     try {
         const { token, refreshToken: r_token } = CacheStore.AuthStore[projectUrl];
 
-        const [reqBuilder, [privateKey]] = buildFetchInterface({
+        const [reqBuilder, [privateKey]] = await buildFetchInterface({
             body: { token, r_token },
             accessKey,
             uglify,
@@ -96,7 +96,7 @@ const refreshToken = (builder, processRef, remainRetries = 7, initialRetries = 7
         }
         if (r.simpleError) throw r;
 
-        const f = uglify ? deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r;
+        const f = uglify ? await deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r;
 
         if (CacheStore.AuthStore[projectUrl]) {
             CacheStore.AuthStore[projectUrl].token = f.result.token;

package/src/products/auth/index.js

@@ -61,7 +61,7 @@ export class MTAuth {
             }
             if (processID !== lastInitRef) return;
             const mtoken = Scoped.AuthJWTToken[projectUrl],
-                [reqBuilder, [privateKey]] = uglify ? serializeE2E({ mtoken }, undefined, serverE2E_PublicKey) : [null, []];
+                [reqBuilder, [privateKey]] = uglify ? await serializeE2E({ mtoken }, undefined, serverE2E_PublicKey) : [null, []];
 
             socket = io(`${wsPrefix}://${baseUrl}`, {
                 transports: ['websocket', 'polling', 'flashsocket'],
@@ -74,11 +74,11 @@ export class MTAuth {
 
             socket.emit(_listenUserVerification(uglify));
 
-            socket.on("onVerificationChanged", ([err, verified]) => {
+            socket.on("onVerificationChanged", async ([err, verified]) => {
                 if (err) {
                     onError?.(simplifyCaughtError(err).simpleError);
                 } else {
-                    callback?.(uglify ? deserializeE2E(verified, serverE2E_PublicKey, privateKey) : verified);
+                    callback?.(uglify ? await deserializeE2E(verified, serverE2E_PublicKey, privateKey) : verified);
                 }
             });
 
@@ -171,7 +171,7 @@ const doCustomSignin = (builder, email, password) => new Promise(async (resolve,
 
     try {
         await awaitStore();
-        const [reqBuilder, [privateKey]] = buildFetchInterface({
+        const [reqBuilder, [privateKey]] = await buildFetchInterface({
             body: { data: `${encodeBinary(email)}.${encodeBinary(password)}` },
             accessKey,
             serverE2E_PublicKey,
@@ -181,7 +181,7 @@ const doCustomSignin = (builder, email, password) => new Promise(async (resolve,
         const f = await (await fetch(_customSignin(projectUrl, uglify), reqBuilder)).json();
         if (f.simpleError) throw f;
 
-        const r = uglify ? deserializeE2E(f.e2e, serverE2E_PublicKey, privateKey) : f;
+        const r = uglify ? await deserializeE2E(f.e2e, serverE2E_PublicKey, privateKey) : f;
 
         resolve({
             user: parseToken(r.result.token),
@@ -199,7 +199,7 @@ const doCustomSignup = (builder, email, password, name, metadata) => new Promise
 
     try {
         await awaitStore();
-        const [reqBuilder, [privateKey]] = buildFetchInterface({
+        const [reqBuilder, [privateKey]] = await buildFetchInterface({
             body: {
                 data: `${encodeBinary(email)}.${encodeBinary(password)}.${(encodeBinary((name || '').trim()))}`,
                 metadata,
@@ -212,7 +212,7 @@ const doCustomSignup = (builder, email, password, name, metadata) => new Promise
         const f = await (await fetch(_customSignup(projectUrl, uglify), reqBuilder)).json();
         if (f.simpleError) throw f;
 
-        const r = uglify ? deserializeE2E(f.e2e, serverE2E_PublicKey, privateKey) : f;
+        const r = uglify ? await deserializeE2E(f.e2e, serverE2E_PublicKey, privateKey) : f;
 
         resolve({
             user: parseToken(r.result.token),
@@ -249,7 +249,7 @@ export const doSignOut = async (builder) => {
         try {
             await awaitReachableServer(projectUrl);
 
-            const [reqBuilder] = buildFetchInterface({
+            const [reqBuilder] = await buildFetchInterface({
                 body: { token, r_token },
                 accessKey,
                 uglify,
@@ -269,7 +269,7 @@ const doGoogleSignin = (builder, token) => new Promise(async (resolve, reject) =
 
     try {
         await awaitStore();
-        const [reqBuilder, [privateKey]] = buildFetchInterface({
+        const [reqBuilder, [privateKey]] = await buildFetchInterface({
             body: { token },
             accessKey,
             uglify,
@@ -279,7 +279,7 @@ const doGoogleSignin = (builder, token) => new Promise(async (resolve, reject) =
         const r = await (await fetch(_googleSignin(projectUrl, uglify), reqBuilder)).json();
         if (r.simpleError) throw r;
 
-        const f = uglify ? deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r;
+        const f = uglify ? await deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r;
 
         resolve({
             user: parseToken(f.result.token),

package/src/products/database/accessor.js

@@ -1,7 +1,7 @@
 import { niceHash, shuffleArray, sortArrayByObjectKey } from "../../helpers/peripherals";
 import { awaitStore, updateCacheStore } from "../../helpers/utils";
 import { CacheStore, Scoped } from "../../helpers/variables";
-import { CompareBson, confirmFilterDoc, defaultBSON, downcastBSON, validateCollectionName, validateFilter } from "./validator";
+import { assignExtractionFind, CompareBson, confirmFilterDoc, defaultBSON, downcastBSON, validateCollectionName, validateFilter } from "./validator";
 import getLodash from 'lodash.get';
 import setLodash from 'lodash.set';
 import unsetLodash from 'lodash.unset';
@@ -48,7 +48,7 @@ export const insertRecord = async (builder, config, accessId, value) => {
 
     await awaitStore();
     const { projectUrl, dbUrl, dbName, path, command } = builder;
-    const entityId = generateRecordID({}, config);
+    const entityId = await generateRecordID({}, config);
     const colData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'data', entityId], { config, command, listing: [] });
     const trackedData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'record', accessId]);
 
@@ -106,7 +106,7 @@ export const getRecord = async (builder, config, accessId) => {
     await awaitStore();
     const { projectUrl, dbUrl, dbName, path, command } = builder;
     const { find, findOne, sort, direction, limit, random } = command;
-    const entityId = generateRecordID({}, config);
+    const entityId = await generateRecordID({}, config);
     const colData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'data', entityId]);
     const colRecord = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'record', accessId]);
 
@@ -193,16 +193,18 @@ const recurseNonAtomicWrite = (obj, i, type) => {
             if (k === '_id') throw `avoid providing "_id" for ${type}() operation as _id only reference a single document`;
             if (k === '_foreign_doc') throw '"_foreign_doc" is readonly';
         }
-        if (k.includes('$') || k.includes('.'))
-            throw `invalid property "${k}", ${type}() operation fields must not contain .$`;
-        if (Validator.OBJECT(v)) recurseNonAtomicWrite(obj, i + 1, type);
+        if (k.includes('$') || k.includes('.')) {
+            if (!(k === '$timestamp' && v === 'now'))
+                throw `invalid property "${k}", ${type}() operation fields must not contain .$`;
+        }
+        if (Validator.OBJECT(v)) recurseNonAtomicWrite(v, i + 1, type);
     });
 };
 
 const recurseAtomicWrite = (obj, i, type) => {
     if (!Validator.OBJECT(obj)) throw `expected a document but got ${obj}`;
-    Object.entries(obj).forEach(([k]) => {
-        if (!(k in AtomicWriter)) throw `Unknown update operator: ${k}`;
+    Object.entries(obj).forEach(([k, v]) => {
+        if (!i && !(k in AtomicWriter)) throw `Unknown update operator: ${k}`;
         if (i === 1) {
             if ((k === '_id' || k.startsWith('_id.')))
                 throw `avoid providing "_id" for ${type}() operation as _id only reference a single document`;
@@ -211,7 +213,7 @@ const recurseAtomicWrite = (obj, i, type) => {
                 throw '"_foreign_doc" is readonly';
         }
         if (k.includes('.$')) throw `unsupported operation at "${k}"`;
-        if (!i) recurseAtomicWrite(obj, i + 1, type);
+        if (!i || Validator.OBJECT(v)) recurseAtomicWrite(v, i + 1, type);
     });
 };
 
@@ -299,7 +301,7 @@ export const syncCache = (builder, result) => {
             result.value.map(({ scope, value, find, path }) =>
                 ({ type: scope, value, find, path })
             )
-            : { ...result, path: builder.path }
+            : [{ ...result, path: builder.path }]
     ).forEach(({ value: writeObj, find, type, path }) => {
         WriteValidator[type]({ find, value: writeObj });
         validateCollectionName(path);
@@ -600,22 +602,6 @@ const getFindFields = (find) => {
     return result.filter((v, i, a) => a.findIndex(b => b === v) === i);
 };
 
-const assignExtractionFind = (data, find) => {
-    if (!find) return find;
-
-    if (niceGuard({ $dynamicValue: GuardSignal.NON_EMPTY_STRING }, find)) {
-        return getLodash(data, find.$dynamicValue) || null;
-    } else if (Validator.OBJECT(find)) {
-        return Object.fromEntries(
-            Object.entries(find).map(([k, v]) =>
-                Validator.JSON(v) ? [k, assignExtractionFind(data, v)] : [k, v]
-            )
-        );
-    } else if (Array.isArray(find)) {
-        return find.map(v => assignExtractionFind(data, v));
-    } else return find;
-};
-
 const deserializeWriteValue = (value) => {
     if (!value) return value;
 

package/src/products/database/index.js

@@ -133,9 +133,9 @@ const listenDocument = (callback, onError, builder, config) => {
     const { projectUrl, wsPrefix, serverE2E_PublicKey, baseUrl, dbUrl, dbName, accessKey, path, disableCache, command, uglify, castBSON } = builder;
     const { find, findOne, sort, direction, limit } = command;
     const { disableAuth } = config || {};
-    const accessId = generateRecordID(builder, config);
     const shouldCache = !disableCache;
     const processId = `${++Scoped.AnyProcessIte}`;
+    let accessId;
 
     validateListenFindConfig(config);
     validateFilter(findOne || find);
@@ -159,9 +159,13 @@ const listenDocument = (callback, onError, builder, config) => {
     };
 
     if (shouldCache) {
-        cacheListener = listenQueryEntry(snapshot => {
-            if (!Scoped.IS_CONNECTED[projectUrl]) dispatchSnapshot(snapshot);
-        }, { accessId, builder, config, processId });
+        accessId = generateRecordID(builder, config).then(hash => {
+            if (hasCancelled) return hash;
+            cacheListener = listenQueryEntry(snapshot => {
+                if (!Scoped.IS_CONNECTED[projectUrl]) dispatchSnapshot(snapshot);
+            }, { accessId: hash, builder, config, processId });
+            return hash;
+        });
 
         awaitStore().then(() => {
             if (hasCancelled) return;
@@ -193,7 +197,7 @@ const listenDocument = (callback, onError, builder, config) => {
             dbUrl
         };
 
-        const [encPlate, [privateKey]] = uglify ? serializeE2E({ accessKey, _body: authObj }, mtoken, serverE2E_PublicKey) : ['', []];
+        const [encPlate, [privateKey]] = uglify ? await serializeE2E({ accessKey, _body: authObj }, mtoken, serverE2E_PublicKey) : ['', []];
 
         socket = io(`${wsPrefix}://${baseUrl}`, {
             transports: ['websocket', 'polling', 'flashsocket'],
@@ -210,13 +214,15 @@ const listenDocument = (callback, onError, builder, config) => {
         socket.on('mSnapshot', async ([err, snapshot]) => {
             hasRespond = true;
             if (err) {
-                onError?.(simplifyCaughtError(err).simpleError);
+                if (typeof onError === 'function') {
+                    onError(simplifyCaughtError(err).simpleError);
+                } else console.error('unhandled listen for:', { path, find }, ' error:', err);
             } else {
-                if (uglify) snapshot = deserializeE2E(snapshot, serverE2E_PublicKey, privateKey);
+                if (uglify) snapshot = await deserializeE2E(snapshot, serverE2E_PublicKey, privateKey);
                 snapshot = deserializeBSON(snapshot)._;
                 dispatchSnapshot(snapshot);
 
-                if (shouldCache) insertRecord(builder, config, accessId, snapshot);
+                if (shouldCache) insertRecord(builder, config, await accessId, snapshot);
             }
         });
 
@@ -284,7 +290,7 @@ const initOnDisconnectionTask = (builder, value, type) => {
         socket = io(`${wsPrefix}://${baseUrl}`, {
             transports: ['websocket', 'polling', 'flashsocket'],
             auth: uglify ? {
-                e2e: serializeE2E(authObj, mtoken, serverE2E_PublicKey)[0],
+                e2e: (await serializeE2E({ accessKey, _body: authObj }, mtoken, serverE2E_PublicKey))[0],
                 _m_internal: true
             } : {
                 ...mtoken ? { mtoken } : {},
@@ -334,7 +340,7 @@ const countCollection = async (builder, config) => {
     const { projectUrl, serverE2E_PublicKey, dbUrl, dbName, accessKey, maxRetries = 7, uglify, path, disableCache, command = {} } = builder;
     const { find } = command;
     const { disableAuth } = config || {};
-    const accessId = generateRecordID({ ...builder, countDoc: true }, config);
+    const accessId = await generateRecordID({ ...builder, countDoc: true }, config);
 
     await awaitStore();
     if (config !== undefined)
@@ -359,7 +365,7 @@ const countCollection = async (builder, config) => {
             if (!disableAuth && await getReachableServer(projectUrl))
                 await awaitRefreshToken(projectUrl);
 
-            const [reqBuilder, [privateKey]] = buildFetchInterface({
+            const [reqBuilder, [privateKey]] = await buildFetchInterface({
                 body: {
                     commands: { path, find: serializeToBase64(find) },
                     dbName,
@@ -374,7 +380,7 @@ const countCollection = async (builder, config) => {
             const r = await (await fetch(_documentCount(projectUrl, uglify), reqBuilder)).json();
             if (r.simpleError) throw r;
 
-            const f = uglify ? deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r;
+            const f = uglify ? await deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r;
 
             if (!disableCache)
                 setLodash(CacheStore.DatabaseCountResult, [projectUrl, dbUrl, dbName, accessId], f.result);
@@ -424,7 +430,7 @@ const findObject = async (builder, config) => {
     const { find, findOne, sort, direction, limit, random } = command;
     const { retrieval = RETRIEVAL.DEFAULT, episode = 0, disableAuth, disableMinimizer } = config || {};
     const enableMinimizer = !disableMinimizer;
-    const accessId = generateRecordID(builder, config);
+    const accessId = await generateRecordID(builder, config);
     const processAccessId = `${accessId}${projectUrl}${dbUrl}${dbName}${retrieval}`;
     const getRecordData = () => getRecord(builder, config, accessId);
     const shouldCache = (retrieval !== RETRIEVAL.DEFAULT || !disableCache) &&
@@ -491,7 +497,7 @@ const findObject = async (builder, config) => {
             if (!disableAuth && await getReachableServer(projectUrl))
                 await awaitRefreshToken(projectUrl);
 
-            const [reqBuilder, [privateKey]] = buildFetchInterface({
+            const [reqBuilder, [privateKey]] = await buildFetchInterface({
                 body: {
                     commands: {
                         config: pureConfig && serializeToBase64(pureConfig),
@@ -514,7 +520,7 @@ const findObject = async (builder, config) => {
             const r = await (await fetch((findOne ? _readDocument : _queryCollection)(projectUrl, uglify), reqBuilder)).json();
             if (r.simpleError) throw r;
 
-            const result = deserializeBSON((uglify ? deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r).result)._;
+            const result = deserializeBSON((uglify ? await deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r).result)._;
 
             if (shouldCache) insertRecord(builder, config, accessId, result);
             finalize({ liveResult: result || null });
@@ -628,7 +634,7 @@ const commitData = async (builder, value, type, config) => {
             if (!disableAuth && await getReachableServer(projectUrl))
                 await awaitRefreshToken(projectUrl);
 
-            const [reqBuilder, [privateKey]] = buildFetchInterface({
+            const [reqBuilder, [privateKey]] = await buildFetchInterface({
                 body: {
                     commands: {
                         value: value && serializeToBase64({ _: value }),
@@ -650,7 +656,7 @@ const commitData = async (builder, value, type, config) => {
             const r = await (await fetch((isBatchWrite ? _writeMapDocument : _writeDocument)(projectUrl, uglify), reqBuilder)).json();
             if (r.simpleError) throw r;
 
-            const f = uglify ? deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r;
+            const f = uglify ? await deserializeE2E(r.e2e, serverE2E_PublicKey, privateKey) : r;
 
             finalize({ ...f }, undefined, { removeCache: true });
         } catch (e) {

package/src/products/database/validator.js

@@ -6,10 +6,15 @@ import { Binary, BSONRegExp, BSONSymbol, Code, DBRef, Decimal128, Double, Int32,
 import { bboxPolygon, booleanIntersects, booleanWithin, circle, distance, polygon } from "@turf/turf";
 
 const DirectionList = [1, -1, 'asc', 'desc', 'ascending', 'descending'];
-const FilterFootPrint = t => validateFilter(t);
+const FilterFootPrint = t => {
+    validateFilter(t);
+    return true;
+};
 const ReturnAndExcludeFootprint = t => t === undefined ||
     !(Array.isArray(t) ? t : [t]).filter(v => !Validator.TRIMMED_NON_EMPTY_STRING(v)).length;
 
+const ConfigFind = t => t && FilterFootPrint(assignExtractionFind({}, t));
+
 const FindConfig = {
     extraction: t => t === undefined ||
         (Array.isArray(t) ? t : [t]).filter(m =>
@@ -18,8 +23,8 @@ const FindConfig = {
                 sort: (t, p) => t === undefined || (Validator.TRIMMED_NON_EMPTY_STRING(t) && p.find),
                 direction: (t, p) => t === undefined || (p.sort && p.find && DirectionList.includes(t)),
                 limit: (t, p) => t === undefined || (Validator.POSITIVE_INTEGER(t) && p.find),
-                find: (t, p) => (t === undefined && p.findOne) || (!p.findOne && FilterFootPrint(t)),
-                findOne: (t, p) => (t === undefined && p.find) || (!p.find && FilterFootPrint(t)),
+                find: (t, p) => (t === undefined && p.findOne) || (!p.findOne && ConfigFind(t)),
+                findOne: (t, p) => (t === undefined && p.find) || (!p.find && ConfigFind(t)),
                 returnOnly: ReturnAndExcludeFootprint,
                 excludeFields: ReturnAndExcludeFootprint
             }).validate(m)
@@ -41,7 +46,8 @@ export const validateListenFindConfig = (config) => config === undefined ||
         extraction: FindConfig.extraction,
         returnOnly: FindConfig.returnOnly,
         excludeFields: FindConfig.excludeFields,
-        disableAuth: FindConfig.disableAuth
+        disableAuth: FindConfig.disableAuth,
+        episode: t => [undefined, 0, 1].includes(t)
     }).validate(config);
 
 export const validateFindObject = command =>
@@ -55,6 +61,22 @@ export const validateFindObject = command =>
         random: (t, p) => t === undefined || (!p.sort && t === true),
     }).validate({ ...command });
 
+export const assignExtractionFind = (data, find) => {
+    if (!find) return find;
+
+    if (niceGuard({ $dynamicValue: GuardSignal.NON_EMPTY_STRING }, find)) {
+        return getLodash(data, find.$dynamicValue) || null;
+    } else if (Validator.OBJECT(find)) {
+        return Object.fromEntries(
+            Object.entries(find).map(([k, v]) =>
+                Validator.JSON(v) ? [k, assignExtractionFind(data, v)] : [k, v]
+            )
+        );
+    } else if (Array.isArray(find)) {
+        return find.map(v => assignExtractionFind(data, v));
+    } else return find;
+};
+
 export const validateCollectionName = collectionName => {
     // Check if the collection name is empty
     if (!collectionName || typeof collectionName !== 'string')

package/src/products/http_callable/index.js

@@ -1,5 +1,5 @@
 import { Buffer } from "buffer";
-import { deserializeE2E, listenReachableServer, niceHash, serializeE2E } from "../../helpers/peripherals";
+import { deserializeE2E, listenReachableServer, niceHash, normalizeRoute, serializeE2E } from "../../helpers/peripherals";
 import { awaitStore, getReachableServer, updateCacheStore } from "../../helpers/utils";
 import { RETRIEVAL } from "../../helpers/values";
 import { CacheStore, Scoped } from "../../helpers/variables";
@@ -7,6 +7,7 @@ import { awaitRefreshToken } from "../auth/accessor";
 import { simplifyCaughtError } from "simplify-error";
 import { guardObject, Validator } from "guard-object";
 import cloneDeep from "lodash.clonedeep";
+import { stringify } from "json-buffer";
 
 const buildFetchData = (data) => {
     const { ok, type, status, statusText, redirected, url, headers, size, base64 } = data;
@@ -35,7 +36,7 @@ export const mfetch = async (input = '', init, config) => {
     const { projectUrl, serverE2E_PublicKey, method, maxRetries = 7, disableCache, accessKey, uglify } = config;
     const { headers, body } = init || {};
 
-    if (config !== undefined)
+    if (method !== undefined)
         guardObject({
             enableMinimizer: t => t === undefined || Validator.BOOLEAN(t),
             rawApproach: t => t === undefined || Validator.BOOLEAN(t),
@@ -48,6 +49,8 @@ export const mfetch = async (input = '', init, config) => {
     const disableAuth = method?.disableAuth || isBaseUrl;
     const shouldCache = (retrieval !== RETRIEVAL.DEFAULT || !disableCache) &&
         retrieval !== RETRIEVAL.NO_CACHE_NO_AWAIT;
+    const uglified = !!(!isBaseUrl && uglify);
+
     const rawHeader = Object.fromEntries(
         [...new Headers(headers).entries()]
     );
@@ -58,16 +61,25 @@ export const mfetch = async (input = '', init, config) => {
     if ('uglified' in rawHeader)
         throw '"uglified" in header is a reserved prop';
 
-    if (input.startsWith(projectUrl) && !rawApproach)
+    if (isBaseUrl && !rawApproach)
         throw `please set { rawApproach: true } if you're trying to access different endpoint at "${input}"`;
 
-    if (body !== undefined && typeof body !== 'string')
-        throw `"body" must be a string value`;
+    if (body !== undefined) {
+        if (
+            typeof body !== 'string' &&
+            !Buffer.isBuffer(body) &&
+            !Validator.JSON(body) &&
+            !(body instanceof File) &&
+            !(body instanceof Blob)
+        ) throw `"body" must be any of string, buffer, object, File, Blob`;
+    }
+
+    const rawBody = stringify([(body instanceof File || body instanceof Blob) ? Buffer.from(await body.arrayBuffer()) : body]);
 
-    const reqId = niceHash(
+    const reqId = await niceHash(
         JSON.stringify([
             rawHeader,
-            body,
+            rawBody,
             !!disableAuth,
             input
         ])
@@ -131,19 +143,18 @@ export const mfetch = async (input = '', init, config) => {
                 await awaitRefreshToken(projectUrl);
 
             const mtoken = Scoped.AuthJWTToken[projectUrl];
-            const uglified = !!(!isBaseUrl && body && uglify);
             const initType = rawHeader['content-type'];
 
-            const [reqBuilder, [privateKey]] = uglified ? serializeE2E(body, mtoken, serverE2E_PublicKey) : [null, []];
+            const [reqBuilder, [privateKey]] = uglified ? await serializeE2E(rawBody, mtoken, serverE2E_PublicKey) : [null, []];
 
-            const f = await fetch(isBaseUrl ? input : `${projectUrl}/${input}`, {
+            const f = await fetch(isBaseUrl ? input : `${projectUrl}/${normalizeRoute(input)}`, {
                 ...isBaseUrl ? {} : { method: 'POST' },
                 ...init,
                 ...uglified ? { body: reqBuilder } : {},
                 cache: 'no-cache',
                 headers: {
-                    ...isBaseUrl ? {} : { 'Content-type': 'application/json' },
-                    ...headers,
+                    ...isBaseUrl ? {} : { 'content-type': 'application/json' },
+                    ...rawHeader,
                     ...uglified ? {
                         uglified,
                         'content-type': 'text/plain',
@@ -159,7 +170,7 @@ export const mfetch = async (input = '', init, config) => {
             if (!isBaseUrl && simple) throw { simpleError: JSON.parse(simple) };
 
             const base64 = uglified ?
-                Buffer.from(deserializeE2E(await f.text(), serverE2E_PublicKey, privateKey), 'base64') :
+                Buffer.from(await deserializeE2E(await f.text(), serverE2E_PublicKey, privateKey), 'base64') :
                 Buffer.from(await f.arrayBuffer()).toString('base64');
 
             const resObj = {

package/src/products/storage/index.js

@@ -1,9 +1,10 @@
 import EngineApi from "../../helpers/engine_api";
 import { encodeBinary } from "../../helpers/peripherals";
 import { Scoped } from "../../helpers/variables";
-import { awaitReachableServer, buildFetchInterface, simplifyError } from "../../helpers/utils";
+import { awaitReachableServer, buildFetchInterface } from "../../helpers/utils";
 import { awaitRefreshToken } from "../auth/accessor";
 import { Buffer } from "buffer";
+import { simplifyError } from "simplify-error";
 
 export class MTStorage {
     constructor(config) {
@@ -104,7 +105,7 @@ const deleteContent = async (builder, path, isFolder) => {
     try {
         const r = await (await fetch(
             EngineApi[isFolder ? '_deleteFolder' : '_deleteFile'](projectUrl, uglify),
-            buildFetchInterface({ path }, accessKey, Scoped.AuthJWTToken[projectUrl], 'DELETE')
+            await buildFetchInterface({ path }, accessKey, Scoped.AuthJWTToken[projectUrl], 'DELETE')
         )).json();
         if (r.simpleError) throw r;
         if (r.status !== 'success') throw 'operation not successful';