mosquito-transport 1.6.0 → 1.6.2

package/lib/helpers/e2e_worker.js

@@ -1,6 +1,7 @@
 import LimitTask from "limit-task";
 import naclPkg from 'tweetnacl-functional';
 import { availableParallelism } from 'os';
+import { Worker } from "worker_threads";
 
 function e2e_baseCode() {
     const serializeE2E = (data, clientPublicKey, serverPrivateKey) => {

package/lib/helpers/utils.js

@@ -43,13 +43,9 @@ export const niceTry = (promise) => new Promise(async resolve => {
     }
 });
 
-export const serializeEntity = obj => serialize(obj);
-
-export const deserializeEntity = deserialize;
-
-export const deserializeE2E = async (data = '', projectName) => {
+export const deserializeE2E = async (data, projectName) => {
     try {
-        const [clientPubKey, clientNonce, clientData] = data.split('.').map(v => new Uint8Array(Buffer.from(v, 'base64'))),
+        const [clientPubKey, clientNonce, clientData] = deserialize(data),
             [_, serverPrivateKey] = Scoped.InstancesData[projectName].E2E_BufferPair || [];
 
         if (!serverPrivateKey) throw '"e2eKeyPair" is required for decrypting an end-to-end messages';
@@ -62,7 +58,7 @@ export const deserializeE2E = async (data = '', projectName) => {
 
         if (!baseArray) throw 'The server was unable to decrypt the request body';
 
-        const decryptedData = JSON.parse(Buffer.from(baseArray).toString('utf8'));
+        const decryptedData = deserialize(baseArray);
 
         return [decryptedData[0], clientPubKey, decryptedData[1]];
     } catch (e) {
@@ -72,24 +68,24 @@ export const deserializeE2E = async (data = '', projectName) => {
 
 export const serializeE2E = async (message, clientPublicKey, projectName) => {
     const [_, serverPrivateKey] = Scoped.InstancesData[projectName].E2E_BufferPair || [];
-    const inputData = new Uint8Array(Buffer.from(JSON.stringify([message]), 'utf8'));
+    const inputData = serialize(message);
 
     if (inputData.byteLength > 10240) {
         const { data, nonce } = await e2e_worker.encrypt(inputData, clientPublicKey, serverPrivateKey);
-        return `${Buffer.from(nonce).toString('base64')}.${Buffer.from(data).toString('base64')}`;
+        return serialize([nonce, data]);
     }
 
-    const nonce = randomBytes(box.nonceLength),
-        nonceBase64 = Buffer.from(nonce).toString('base64');
+    const nonce = randomBytes(box.nonceLength);
 
-    return `${nonceBase64}.${Buffer.from(
+    return serialize([
+        nonce,
         box(
             inputData,
             nonce,
             clientPublicKey,
             serverPrivateKey
         )
-    ).toString('base64')}`;
+    ]);
 };
 
 export const getStringExtension = (url) => {

package/lib/index.d.ts

@@ -276,6 +276,7 @@ interface MSocketSnapshot {
 interface MSocketError {
     error: string;
     message: string;
+    socket: Socket;
 }
 
 interface TransformMediaOption {
@@ -358,7 +359,8 @@ interface MosquitoServerConfig {
 
     storageRules: (snapshot?: StorageRulesSnapshot) => Promise<void> | undefined;
     databaseRules: (snapshot?: DatabaseRulesSnapshot) => Promise<void> | undefined;
-    onSocketSnapshot?: (snapshot?: MSocketSnapshot, error?: MSocketError) => void;
+    onSocketSnapshot?: (snapshot?: MSocketSnapshot) => void;
+    onSocketError?: (error?: MSocketError) => void;
     /**
      * the port number you want mosquito-transport instance to be running on
      */
@@ -709,6 +711,13 @@ interface RawBodyRequest extends express.Request {
     rawBody: Buffer;
 }
 
+/**
+ * useful for avoiding encrypting data and extra overhead
+ */
+export class DoNotEncrypt {
+    value: any;
+}
+
 export default class MosquitoTransportServer {
     constructor(config: MosquitoServerConfig);
 

package/lib/index.js

@@ -4,7 +4,7 @@ import { databaseLivePath, databaseLiveRoutes, databaseRoutes, dbRoute, emitData
 import { authLivePath, authLiveRoutes, authRouteName, authRoutes } from "./products/auth/index.js";
 import { removeVideoFreezer, storageRouteName, storageRoutes, validateStoragePath } from "./products/storage/index.js";
 import { Scoped } from "./helpers/variables.js";
-import { decodeBinary, deserializeE2E, deserializeEntity, encodeBinary, ensureDir, getStringExtension, interpolate, niceTry, normalizeRoute, serializeE2E, serializeEntity } from "./helpers/utils.js";
+import { decodeBinary, deserializeE2E, encodeBinary, ensureDir, getStringExtension, interpolate, niceTry, normalizeRoute, serializeE2E } from "./helpers/utils.js";
 import { getDB } from "./products/database/base.js";
 import { releaseTokenSelfDestruction, validateJWT, verifyJWT } from "./products/auth/tokenizer.js";
 import { ADMIN_DB_NAME, ADMIN_DB_URL, AUTH_PROVIDER_ID, ERRORS, EnginePath, EngineRoutes, STORAGE_DIRS, STORAGE_PREFIX_PATH, STORAGE_ROUTE, one_hour, one_mb, one_minute } from "./helpers/values.js";
@@ -301,7 +301,7 @@ const InternalRoutes = [
 ];
 
 const useMosquitoServer = (app, config) => {
-    const { projectName, port, accessKey, corsOrigin, maxRequestBufferSize, onSocketSnapshot, enforceE2E_Encryption, preMiddlewares, onUserMounted } = config;
+    const { projectName, port, accessKey, corsOrigin, maxRequestBufferSize, onSocketSnapshot, onSocketError, enforceE2E_Encryption, preMiddlewares, onUserMounted } = config;
 
     app.disable("x-powered-by");
 
@@ -328,6 +328,7 @@ const useMosquitoServer = (app, config) => {
         serveStorage({ ...config }),
         express.json({ type: '*/json', limit: maxRequestBufferSize || '100MB' }),
         express.text({ type: 'text/plain', limit: maxRequestBufferSize || '100MB' }),
+        express.raw({ type: 'request/buffer', limit: maxRequestBufferSize || '100MB' }),
         (req, _, next) => {
             const authToken = req.headers['mosquito-token'];
             if (authToken) req.headers.mtoken = authToken;
@@ -352,7 +353,7 @@ const useMosquitoServer = (app, config) => {
                     if (chunk) chunks.push(Buffer.from(chunk));
                     const totalBuf = Buffer.concat(chunks);
                     const { __sender } = req;
-                    const transformedBody = typeof __sender === 'function' ? Buffer.from(await __sender(totalBuf), 'utf8') : totalBuf;
+                    const transformedBody = typeof __sender === 'function' ? await __sender(totalBuf) : totalBuf;
 
                     res.setHeader('Content-Length', Buffer.byteLength(transformedBody));
                     originalWrite.call(res, transformedBody);
@@ -443,7 +444,7 @@ const useMosquitoServer = (app, config) => {
             let mtoken, clientPublicKey, extraAuth;
 
             if (ugly) {
-                const [body, clientKey, atoken] = await deserializeE2E(e2e, projectName);
+                const [body, clientKey, atoken] = await deserializeE2E(Buffer.from(e2e, 'base64'), projectName);
                 mtoken = atoken;
                 clientPublicKey = clientKey;
                 if (body.accessKey !== accessKey)
@@ -468,7 +469,7 @@ const useMosquitoServer = (app, config) => {
                             callback?.(...[...arguments]);
                             return;
                         }
-                        const [emittion, emitable, ...rest] = [...arguments];
+                        const [[emittion, not_encrypted], emitable, ...rest] = [...arguments];
 
                         let reqBody, clientPublicKey;
 
@@ -480,9 +481,10 @@ const useMosquitoServer = (app, config) => {
                             ) throw 'tampered socket emittion';
                             if (ugly) {
                                 const [body, clientKey] = await deserializeE2E(emittion, projectName);
-                                reqBody = deserializeEntity(body);
+                                reqBody = body;
                                 clientPublicKey = clientKey;
                             } else reqBody = emittion;
+                            reqBody = discloseSocketArguments([reqBody, not_encrypted]);
                             if (!Array.isArray(reqBody))
                                 throw simplifyError('invalid_argument_result', 'The request body was not deserialized correctly');
                         } catch (e) {
@@ -490,19 +492,19 @@ const useMosquitoServer = (app, config) => {
                             onError?.({
                                 ...simplifyCaughtError(e)?.simpleError,
                                 auth: extraAuth,
-                                data: reqBody || emittion
+                                data: emittion
                             });
                             return;
                         }
 
                         callback?.(...reqBody, ...typeof emitable === 'function' ? [async function () {
-                            const args = [...arguments];
+                            const [args, not_encrypted] = encloseSocketArguments([...arguments]);
                             let res;
 
                             if (ugly) {
-                                res = await serializeE2E(serializeEntity(args), clientPublicKey, projectName);
+                                res = await serializeE2E(args, clientPublicKey, projectName);
                             } else res = args;
-                            emitable(res);
+                            emitable([res, not_encrypted]);
                         }] : []);
                     });
                 };
@@ -516,31 +518,31 @@ const useMosquitoServer = (app, config) => {
 
                 const lastEmit = restEmit.slice(-1)[0];
                 const hasEmitable = typeof lastEmit === 'function';
-                const mit = hasEmitable ? restEmit.slice(0, -1) : restEmit;
+                const [mit, not_encrypted] = encloseSocketArguments(hasEmitable ? restEmit.slice(0, -1) : restEmit);
 
                 if (hasEmitable && promise)
                     throw 'emitWithAck cannot have function in it argument';
 
-                const reqBuilder = ugly ? await serializeE2E(serializeEntity(mit), clientPublicKey, projectName) : null;
+                const reqBuilder = ugly ? await serializeE2E(mit, clientPublicKey, projectName) : null;
 
                 const result = await (timeout ? socket.timeout(timeout) : socket)[promise ? 'emitWithAck' : 'emit'](
                     route,
-                    ugly ? reqBuilder : mit,
-                    ...hasEmitable ? [function () {
-                        const [args] = [...arguments];
-                        (async function () {
-                            let res;
+                    [ugly ? reqBuilder : mit, not_encrypted],
+                    ...hasEmitable ? [async function () {
+                        const [[args, not_encrypted]] = [...arguments];
+                        let res;
 
-                            if (ugly) {
-                                res = deserializeEntity((await deserializeE2E(args, projectName))[0]);
-                            } else res = args;
+                        if (ugly) {
+                            res = (await deserializeE2E(args, projectName))[0];
+                        } else res = args;
 
-                            lastEmit(...res || []);
-                        })();
+                        lastEmit(...discloseSocketArguments([res, not_encrypted]));
                     }] : []
                 );
 
-                if (result && promise) return ugly ? deserializeEntity((await deserializeE2E(result, projectName))[0])[0] : result[0];
+                if (result && promise) {
+                    return discloseSocketArguments([ugly ? (await deserializeE2E(result[0], projectName))[0] : result[0], result[1]])[0];
+                }
             };
 
             const clonedSocket = {
@@ -569,7 +571,7 @@ const useMosquitoServer = (app, config) => {
             };
             onSocketSnapshot(clonedSocket);
         } catch (e) {
-            onSocketSnapshot?.(undefined, simplifyCaughtError(e).simpleError);
+            onSocketError?.(Object.assign(simplifyCaughtError(e).simpleError, { socket }));
         }
     });
 
@@ -578,6 +580,30 @@ const useMosquitoServer = (app, config) => {
     });
 }
 
+export class DoNotEncrypt {
+    constructor(value) {
+        this.value = value;
+    }
+};
+
+const encloseSocketArguments = (args) => {
+    const [encrypted, unencrypted] = [{}, {}];
+
+    args.forEach((v, i) => {
+        if (v instanceof DoNotEncrypt) {
+            unencrypted[i] = v.value;
+        } else encrypted[i] = v;
+    });
+    return [encrypted, unencrypted];
+}
+
+const discloseSocketArguments = (args = []) => {
+    return args.map((obj, i) => Object.entries(obj).map(v => i ? [v[0], new DoNotEncrypt(v[1])] : v)).flat()
+        .sort((a, b) => (a[0] * 1) - (b[0] * 1)).map((v, i) => {
+            if (v[0] * 1 !== i) throw 'corrupted socket arguments';
+            return v[1];
+        });
+}
 
 export default class MosquitoTransportServer {
     constructor(configx) {
@@ -753,14 +779,13 @@ export default class MosquitoTransportServer {
                         const [body, clientKey, atoken] = await deserializeE2E(req.body, this.projectName);
                         clientPublicKey = clientKey;
                         authToken = atoken;
-                        const rawBody = deserializeEntity(body)[0];
                         const initContentType = req.headers["init-content-type"];
 
                         if (initContentType === 'application/json') {
                             try {
-                                reqBody = JSON.parse(rawBody);
+                                reqBody = JSON.parse(body);
                             } catch (_) { }
-                        } else reqBody = rawBody;
+                        } else reqBody = body;
                     }
 
                     req.body = reqBody;
@@ -789,15 +814,20 @@ export default class MosquitoTransportServer {
                 try {
                     if (uglified) {
                         req.__sender = async (buffer) => {
-                            res.set('content-type', 'text/plain');
-                            return await serializeE2E(buffer.toString('base64'), clientPublicKey, this.projectName);
+                            res.set('content-type', 'application/octet-stream');
+                            return await serializeE2E(buffer, clientPublicKey, this.projectName);
                         };
                     }
 
                     await callback(req, res, auth ? { ...auth, token: authToken } : null);
                 } catch (e) {
                     if (hasErrorLogger) console.error(`errRoute: /${route} err:`, e);
-                    if (!res.headersSent) res.end();
+                    if (!res.headersSent) {
+                        res.setHeader(
+                            'simple_error',
+                            JSON.stringify(simplifyCaughtError(e).simpleError || {})
+                        );
+                    }
                 }
                 if (hasLogger) console.log(`${req.url} took: ${Date.now() - now}ms`);
             })
@@ -1155,7 +1185,9 @@ const validateServerConfig = (config, that) => {
         transformMediaCleanupTimeout,
         onUserMounted,
         ddosMap,
-        internals
+        internals,
+        onSocketSnapshot,
+        onSocketError
     } = config;
 
     if (
@@ -1225,6 +1257,12 @@ const validateServerConfig = (config, that) => {
     if (storageRules !== undefined && typeof storageRules !== 'function')
         throw `storageRules type must be function but got ${typeof storageRules}`;
 
+    if (onSocketSnapshot !== undefined && typeof onSocketSnapshot !== 'function')
+        throw `onSocketSnapshot type must be function but got ${typeof onSocketSnapshot}`;
+
+    if (onSocketError !== undefined && typeof onSocketError !== 'function')
+        throw `onSocketError type must be function but got ${typeof onSocketError}`;
+
     if (typeof signerKey !== 'string' || signerKey.trim().length !== 90)
         throw `signerKey must have string length equals to 90 characters without spaces`;
 

package/lib/products/auth/index.js

@@ -9,6 +9,7 @@ import { validateJWT } from "./tokenizer.js";
 import { Validator } from "guard-object";
 import { simplifyCaughtError } from 'simplify-error';
 import { statusErrorCode, useDDOS } from "../../helpers/ddos.js";
+import { serialize } from "entity-serializer";
 
 const {
     _listenUserVerification,
@@ -87,7 +88,7 @@ export const authRoutes = ({
             const { data, metadata: metax, token, r_token } = reqBody;
 
             const makeResult = async (b) => {
-                return ugly ? { e2e: await serializeE2E(b, clientPublicKey, projectName) } : b;
+                return ugly ? serialize([await serializeE2E(b, clientPublicKey, projectName)]) : b;
             }
 
             switch (route) {
@@ -154,7 +155,9 @@ export const authRoutes = ({
                 logger.includes('all') ||
                 logger.includes('error')
             ) console.error(`errRoute: /${route} err:`, e);
-            res.status(statusErrorCode(e)).send({ status: 'error', ...simplifyCaughtError(e) });
+            const result = { status: 'error', ...simplifyCaughtError(e) };
+
+            res.status(statusErrorCode(e)).send(ugly ? serialize([undefined, result]) : result);
         }
         if (hasLogger) console.log(`/${route} took: ${Date.now() - now}ms`);
     })
@@ -181,7 +184,7 @@ export const authLiveRoutes = ({ projectName, logger, enforceETE_Encryption, int
                 let mtoken = initAuthHandshake.mtoken, clientPublicKey;
 
                 if (ugly) {
-                    const [body, clientKey] = await deserializeE2E(initAuthHandshake.e2e, projectName);
+                    const [body, clientKey] = await deserializeE2E(Buffer.from(initAuthHandshake.e2e, 'base64'), projectName);
                     mtoken = body.mtoken;
                     clientPublicKey = clientKey;
                 }

package/lib/products/database/index.js

@@ -14,6 +14,7 @@ import { simplifyCaughtError, simplifyError } from 'simplify-error';
 import { deserializeBSON, serializeToBase64 } from "./bson.js";
 import cloneDeep from "lodash.clonedeep";
 import { statusErrorCode, useDDOS } from "../../helpers/ddos.js";
+import { serialize } from "entity-serializer";
 
 export const TIMESTAMP = { $timestamp: 'now' };
 
@@ -421,7 +422,7 @@ export const databaseRoutes = ({ projectName, logger, accessKey, enforceE2E_Encr
             }
 
             const makeResult = async (b) => {
-                return ugly ? { e2e: await serializeE2E(b, clientPublicKey, projectName) } : b;
+                return ugly ? serialize([await serializeE2E(b, clientPublicKey, projectName)]) : b;
             }
 
             switch (route) {
@@ -456,10 +457,9 @@ export const databaseRoutes = ({ projectName, logger, accessKey, enforceE2E_Encr
             }
         } catch (e) {
             if (logger.includes('all') || logger.includes('error')) console.error(`errRoute: /${route} err:`, e);
-            res.status(statusErrorCode(e)).send({
-                status: 'error',
-                ...simplifyCaughtError(e)
-            });
+            const result = { status: 'error', ...simplifyCaughtError(e) };
+
+            res.status(statusErrorCode(e)).send(ugly ? serialize([undefined, result]) : result);
         }
         if (hasLogger) console.log(`${route} took: ${Date.now() - now}ms`);
     })
@@ -504,7 +504,7 @@ export const databaseLiveRoutes = ({
 
         if (ugly) {
             try {
-                const [body, clientKey, atoken] = await deserializeE2E(initAuthshake.e2e, projectName);
+                const [body, clientKey, atoken] = await deserializeE2E(Buffer.from(initAuthshake.e2e, 'base64'), projectName);
 
                 mtoken = atoken;
                 authshakeObj = body;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mosquito-transport",
-  "version": "1.6.0",
+  "version": "1.6.2",
   "description": "MosquitoTransport is a powerful tool that helps persist and synchronize data between your MongoDB database and frontend applications",
   "main": "lib/index.js",
   "type": "module",
@@ -36,11 +36,11 @@
   },
   "homepage": "https://github.com/deflexable/mosquito-transport#readme",
   "dependencies": {
-    "@deflexable/bit-stream": "^1.0.2",
+    "@deflexable/bit-stream": "^1.0.4",
     "buffer": "^6.0.3",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
-    "entity-serializer": "^1.0.0",
+    "entity-serializer": "^1.0.1",
     "express": "^4.18.2",
     "google-auth-library": "^8.8.0",
     "guard-object": "^1.1.3",