mosquito-transport-js 0.4.0 → 0.4.2

package/TODO

@@ -1,9 +1,23 @@
-- fix local cache query on sqlite and fs
+- fix local cache query on sqlite ✅
 - fix and add all mongodb query and update operator
 - reauthenticate
 - change `Object` in d.ts to [key: string]: any
 - add `getServerTimeOffset` method
-- change null to undefined in `value`
-- `provide functionality to add extra header to MT instance (all outgoing request)`
+- change undefined to null in `value` ✅
+- `provide functionality to add extra header to MT instance (all outgoing request)` ✅
 - borrowToken
-- `add sqlite`
+- `add sqlite` ✅
+- minimize extraction data ✅
+- change `collection().onDisconnect()` to `collection().socket().onDisconnect()` and `collection().socket().onConnect()` ✅
+- add `_foreign_doc` to d.ts ✅
+- tree shake dependencies
+- dynamic import for fs ✅
+- new URL() work around ✅
+- fetchHttp, default retrieval if has body ✅
+- native hashing
+- TextEncoder
+- native storage upload
+- lodashes ✅
+- switch to events package
+- serverTimeOffset
+<!-- - error: "refreshToken retry limit exceeded" <--- no need -->

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mosquito-transport-js",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "Javascript web sdk for mosquito-transport (https://github.com/brainbehindx/mosquito-transport)",
   "main": "src/index.js",
   "type": "module",
@@ -31,14 +31,10 @@
     "@turf/turf": "^7.1.0",
     "bson": "^6.8.0",
     "buffer": "^6.0.3",
-    "crypto-js": "^4.2.0",
     "entity-serializer": "^1.0.2",
     "guard-object": "^1.1.3",
     "limit-task": "1.0.0",
-    "lodash.clonedeep": "^4.5.0",
-    "lodash.get": "^4.4.2",
-    "lodash.set": "^4.3.2",
-    "lodash.unset": "^4.5.2",
+    "lodash": "^4.17.21",
     "simplify-error": "^1.0.1",
     "socket.io-client": "^4.6.2",
     "subscription-listener": "^1.1.2",
@@ -57,4 +53,4 @@
   "engines": {
     "node": ">=14"
   }
-}
+}

package/src/helpers/engine_api.js

@@ -1,12 +1,7 @@
 import { encodeBinary } from './peripherals';
 
-const EngineApiBase = (baseApi, ugly, path) => {
-    const url = new URL(baseApi);
-    if (ugly) {
-        url.pathname = `/e2e/${encodeBinary(path)}`;
-    } else url.pathname = path;
-    return url.href;
-};
+const EngineApiBase = (baseApi, ugly, path) =>
+    ugly ? `${baseApi}/e2e/${encodeBinary(path)}` : `${baseApi}/${path}`;
 
 const apis = {
     _readDocument: (baseApi, ugly) => EngineApiBase(baseApi, ugly, '_readDocument'),

package/src/helpers/peripherals.js

@@ -1,13 +1,10 @@
 import { Buffer } from "buffer";
 import { ServerReachableListener } from "./listeners";
-import aes_pkg from 'crypto-js/aes.js';
-import Utf8Encoder from 'crypto-js/enc-utf8.js';
 import naclPkg from 'tweetnacl-functional';
-import getLodash from "lodash.get";
+import getLodash from "lodash/get";
 import e2e_worker from "./e2e_worker";
 import { deserialize, serialize } from "entity-serializer";
 
-const { encrypt, decrypt } = aes_pkg;
 const { box, randomBytes } = naclPkg;
 
 export const listenReachableServer = (callback, projectUrl) => {
@@ -43,7 +40,7 @@ export const normalizeRoute = (route = '') => route.split('').map((v, i, a) =>
 ).join('');
 
 export const shuffleArray = (n) => {
-    const array = [...n];
+    const array = n.slice(0);
     let currentIndex = array.length, randomIndex;
 
     while (currentIndex != 0) {
@@ -67,7 +64,7 @@ export function sortArrayByObjectKey(arr = [], key) {
     });
 };
 
-export async function niceHash(str) {
+export async function niceHash(str = '') {
     try {
         // Convert the string to a Uint8Array
         const encoder = new TextEncoder();
@@ -96,14 +93,6 @@ export const sameInstance = (var1, var2) => {
     }
 };
 
-export const encryptString = (txt, password, iv) => {
-    return encrypt(txt, `${password || ''}${iv || ''}`).toString();
-};
-
-export const decryptString = (txt, password, iv) => {
-    return decrypt(txt, `${password || ''}${iv || ''}`).toString(Utf8Encoder);
-};
-
 export const serializeE2E = async (data, auth_token, serverPublicKey) => {
     const inputData = serialize([data, auth_token]);
 

package/src/helpers/purger.js

@@ -0,0 +1,136 @@
+import { Validator } from "guard-object";
+import { incrementDatabaseSizeCore } from "../products/database/counter";
+import { incrementFetcherSizeCore } from "../products/http_callable/counter";
+import unsetLodash from 'lodash/unset';
+
+export const purgeRedundantRecords = async (data, builder) => {
+    const { maxLocalDatabaseSize = 10485760, maxLocalFetchHttpSize = 10485760 } = builder;
+
+    /**
+     * @type {import('./variables')['CacheStore']['DatabaseStats']}
+     */
+    const { _db_size, _fetcher_size } = data.DatabaseStats || {};
+
+    const purgeDatabase = () => {
+        if (!Validator.POSITIVE_NUMBER(_db_size) || !maxLocalDatabaseSize || _db_size < maxLocalDatabaseSize) return;
+        const DbListing = [];
+
+        breakDbMap(data.DatabaseStore, (projectUrl, dbUrl, dbName, path, value) => {
+            Object.entries(value.instance).forEach(([access_id, obj]) => {
+                DbListing.push({
+                    builder: { projectUrl, dbUrl, dbName },
+                    path,
+                    access_id,
+                    value: obj
+                });
+            });
+            Object.entries(value.episode).forEach(([access_id, limitObj]) => {
+                Object.entries(limitObj).forEach(([limit, obj]) => {
+                    DbListing.push({
+                        builder: { projectUrl, dbUrl, dbName },
+                        path,
+                        access_id,
+                        limit,
+                        value: obj,
+                        isEpisode: true
+                    });
+                });
+            });
+        });
+
+        breakDbMap(data.DatabaseCountResult, (projectUrl, dbUrl, dbName, path, value) => {
+            Object.entries(value).forEach(([access_id, obj]) => {
+                DbListing.push({
+                    builder: { projectUrl, dbUrl, dbName },
+                    path,
+                    access_id,
+                    value: obj,
+                    isCount: true
+                });
+            });
+        });
+
+        const redundantDbRanking = DbListing.sort((a, b) =>
+            a.value.touched - b.value.touched
+        );
+
+        const newSize = maxLocalDatabaseSize / 2;
+        let sizer = _db_size;
+        let cuts = 0;
+
+        for (let i = 0; i < redundantDbRanking.length; i++) {
+            sizer -= redundantDbRanking[i].value.size || 0;
+            ++cuts;
+            if (sizer < newSize) break;
+        }
+
+        console.warn(`purging ${cuts} of ${redundantDbRanking.length} db entities`);
+        redundantDbRanking.slice(0, cuts).forEach(({
+            builder,
+            path,
+            access_id,
+            isCount,
+            isEpisode,
+            limit,
+            value: { size }
+        }) => {
+            const { projectUrl, dbUrl, dbName } = builder;
+            if (isCount) {
+                unsetLodash(data.DatabaseCountResult, [projectUrl, dbUrl, dbName, path, access_id]);
+            } else {
+                incrementDatabaseSizeCore(data.DatabaseStats, builder, path, -size);
+                if (isEpisode) {
+                    unsetLodash(data.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'episode', access_id, limit]);
+                } else {
+                    unsetLodash(data.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', access_id]);
+                }
+            }
+        });
+    }
+    const purgeFetcher = () => {
+        if (!Validator.POSITIVE_NUMBER(_fetcher_size) || !maxLocalFetchHttpSize || _fetcher_size < maxLocalFetchHttpSize) return;
+        const redundantFetchRanking = Object.entries(data.FetchedStore).map(([projectUrl, access_id_Obj]) =>
+            Object.entries(access_id_Obj).map(([access_id, data]) => ({
+                access_id,
+                projectUrl,
+                data
+            }))
+        ).flat().sort(([a], [b]) =>
+            a.data.touched - b.data.touched
+        );
+
+        const newSize = maxLocalFetchHttpSize / 2;
+        let sizer = _fetcher_size;
+        let cuts = 0;
+
+        for (let i = 0; i < redundantFetchRanking.length; i++) {
+            sizer -= redundantFetchRanking[i].data.size || 0;
+            ++cuts;
+            if (sizer < newSize) break;
+        }
+
+        console.warn(`purging ${cuts} of ${redundantFetchRanking.length} fetcher entities`);
+        redundantFetchRanking.slice(0, cuts).forEach(({ access_id, data: { size }, projectUrl }) => {
+            incrementFetcherSizeCore(data.DatabaseStats, projectUrl, -size);
+            unsetLodash(data.FetchedStore, [projectUrl, access_id]);
+        });
+        console.log('fetcher purging complete');
+    }
+    purgeDatabase();
+    purgeFetcher();
+}
+
+const breakDbMap = (obj, callback) =>
+    Object.entries(obj).forEach(([projectUrl, dbUrlObj]) => {
+        Object.entries(dbUrlObj).forEach(([dbUrl, dbNameObj]) => {
+            Object.entries(dbNameObj).forEach(([dbName, pathObj]) => {
+                Object.entries(pathObj).forEach(([path, value]) => {
+                    callback(projectUrl, dbUrl, dbName, path, value);
+                });
+            });
+        });
+    });
+
+export const purgeInstance = (projectUrl) => {
+    // TODO: purge when signed-out
+}

package/src/helpers/utils.js

@@ -1,76 +1,74 @@
 import { ServerReachableListener, StoreReadyListener } from "./listeners";
-import { CACHE_PROTOCOL, CACHE_STORAGE_PATH, DEFAULT_CACHE_PASSWORD } from "./values";
+import { CACHE_STORAGE_PATH } from "./values";
 import { CacheStore, Scoped } from "./variables";
-import { decryptString, encryptString, serializeE2E } from "./peripherals";
+import { serializeE2E } from "./peripherals";
 import { deserializeBSON, serializeToBase64 } from "../products/database/bson";
 import { trySendPendingWrite } from "../products/database";
 import { deserialize } from "entity-serializer";
+import { purgeRedundantRecords } from "./purger";
 
-export const updateCacheStore = (timer = 300) => {
+export const updateCacheStore = (timer = 300, node) => {
     try { window } catch (_) { return; }
-    const {
-        cachePassword = DEFAULT_CACHE_PASSWORD,
-        cacheProtocol = CACHE_PROTOCOL.LOCAL_STORAGE,
-        io,
-        promoteCache
-    } = Scoped.ReleaseCacheData;
 
-    clearTimeout(Scoped.cacheStorageReducer);
-    Scoped.cacheStorageReducer = setTimeout(() => {
-        const { AuthStore, EmulatedAuth, PendingAuthPurge, DatabaseStore, PendingWrites, ...restStore } = CacheStore;
-
-        const txt = encryptString(
-            JSON.stringify({
-                AuthStore,
-                EmulatedAuth,
-                PendingAuthPurge,
-                ...promoteCache ? {
-                    DatabaseStore: serializeToBase64(DatabaseStore),
-                    PendingWrites: serializeToBase64(PendingWrites)
-                } : {},
-                ...promoteCache ? restStore : {}
-            }),
-            cachePassword,
-            cachePassword
-        );
+    const { io, promoteCache } = Scoped.ReleaseCacheData;
+
+    const doUpdate = async () => {
+        const {
+            AuthStore,
+            EmulatedAuth,
+            PendingAuthPurge,
+            DatabaseStore,
+            PendingWrites,
+            ...restStore
+        } = CacheStore;
+
+        const txt = JSON.stringify({
+            AuthStore,
+            EmulatedAuth,
+            PendingAuthPurge,
+            ...promoteCache ? {
+                DatabaseStore: serializeToBase64(DatabaseStore),
+                PendingWrites: serializeToBase64(PendingWrites)
+            } : {},
+            ...promoteCache ? restStore : {}
+        });
 
         if (io) {
-            io.output(txt);
-        } else if (cacheProtocol === CACHE_PROTOCOL.LOCAL_STORAGE) {
-            window.localStorage.setItem(CACHE_STORAGE_PATH, txt);
-        }
-    }, timer);
+            io.output(txt, node);
+        } else window.localStorage.setItem(CACHE_STORAGE_PATH, txt);
+    };
+
+    clearTimeout(Scoped.cacheStorageReducer);
+    if (timer) {
+        Scoped.cacheStorageReducer = setTimeout(doUpdate, timer);
+    } else doUpdate();
 };
 
 export const releaseCacheStore = async (builder) => {
     try { window } catch (_) { return; }
-    const {
-        cachePassword = DEFAULT_CACHE_PASSWORD,
-        cacheProtocol = CACHE_PROTOCOL.LOCAL_STORAGE,
-        io
-    } = builder;
-
-    let txt;
-
-    if (io) {
-        txt = await io.input();
-    } else if (cacheProtocol === CACHE_PROTOCOL.LOCAL_STORAGE) {
-        txt = window.localStorage.getItem(CACHE_STORAGE_PATH);
-    }
+    const { io } = builder;
+
+    let data = {};
 
-    const j = JSON.parse(decryptString(txt || '', cachePassword, cachePassword) || '{}');
-    const projectList = new Set([]);
+    try {
+        if (io) {
+            data = await io.input();
+        } else data = window.localStorage.getItem(CACHE_STORAGE_PATH);
+        data = JSON.parse(data || '{}');
+        await purgeRedundantRecords(data, builder);
+    } catch (error) {
+        console.error('releaseCacheStore err:', error);
+    }
 
-    Object.entries(j).forEach(([k, v]) => {
+    Object.entries(data).forEach(([k, v]) => {
         if (['DatabaseStore', 'PendingWrites'].includes(k)) {
             CacheStore[k] = deserializeBSON(v);
         } else CacheStore[k] = v;
-        projectList.add(k);
     });
     Object.entries(CacheStore.AuthStore).forEach(([key, value]) => {
         Scoped.AuthJWTToken[key] = value?.token;
     });
-    projectList.forEach(projectUrl => {
+    Object.keys(CacheStore.PendingWrites).forEach(projectUrl => {
         if (Scoped.IS_CONNECTED[projectUrl])
             trySendPendingWrite(projectUrl);
     });
@@ -119,18 +117,17 @@ export const getReachableServer = (projectUrl) => new Promise(resolve => {
     }, true);
 });
 
-export const buildFetchInterface = async ({ body, accessKey, authToken, method, uglify, serverE2E_PublicKey, extraHeaders }) => {
+export const buildFetchInterface = async ({ body, authToken, method, uglify, serverE2E_PublicKey, extraHeaders }) => {
     if (!uglify) body = JSON.stringify({ ...body });
     const [plate, keyPair] = uglify ? await serializeE2E(body, authToken, serverE2E_PublicKey) : [undefined, []];
 
     return [{
         body: uglify ? plate : body,
-        cache: 'no-cache',
+        // cache: 'no-cache',
         headers: {
+            ...extraHeaders,
             'Content-type': uglify ? 'request/buffer' : 'application/json',
-            'Authorization': accessKey,
-            ...(authToken && !uglify) ? { 'Mosquito-Token': authToken } : {},
-            ...extraHeaders
+            ...(authToken && !uglify) ? { 'Mosquito-Token': authToken } : {}
         },
         method: method || 'POST'
     }, keyPair];

package/src/helpers/values.js

@@ -1,17 +1,13 @@
 import { encodeBinary } from './peripherals';
 
-export const CACHE_STORAGE_PATH = encodeBinary('MOSQUITO_TRANSPORT_FREEZER'),
-    DEFAULT_CACHE_PASSWORD = encodeBinary('MOSQUITO_TRANSPORT_CACHE_PASSWORD');
-
-export const CACHE_PROTOCOL = {
-    LOCAL_STORAGE: 'local-storage'
-};
+export const CACHE_STORAGE_PATH = encodeBinary('MOSQUITO_TRANSPORT');
 
 export const RETRIEVAL = {
     STICKY: 'sticky',
     STICKY_NO_AWAIT: 'sticky-no-await',
     STICKY_RELOAD: 'sticky-reload',
     DEFAULT: 'default',
+    CACHE_AWAIT: 'cache-await',
     CACHE_NO_AWAIT: 'cache-no-await',
     NO_CACHE_NO_AWAIT: 'no-cache-no-await',
     NO_CACHE_AWAIT: 'no-cache-await'
@@ -19,11 +15,10 @@ export const RETRIEVAL = {
 
 export const DELIVERY = {
     DEFAULT: 'default',
-    NO_CACHE: 'no-cache',
-    NO_AWAIT: 'no-await',
-    NO_AWAIT_NO_CACHE: 'no-await-no-cache',
-    AWAIT_NO_CACHE: 'await-no-cache',
-    CACHE_NO_AWAIT: 'cache-no-await'
+    CACHE_AWAIT: 'cache-await',
+    CACHE_NO_AWAIT: 'cache-no-await',
+    NO_CACHE_NO_AWAIT: 'no-cache-no-await',
+    NO_CACHE_AWAIT: 'no-cache-await'
 };
 
 export const AUTH_PROVIDER_ID = {

package/src/helpers/variables.js

@@ -14,14 +14,8 @@ export const Scoped = {
     LastTokenRefreshRef: {},
     StorageProcessID: 0,
     InitiatedForcedToken: {},
-    PendingFetchCollective: {
-        pendingProcess: {},
-        pendingResolution: {}
-    },
-    PendingDbReadCollective: {
-        pendingProcess: {},
-        pendingResolution: {}
-    },
+    PendingFetchCollective: {},
+    PendingDbReadCollective: {},
     ActiveDatabaseListeners: {},
     OutgoingWrites: {},
     /**
@@ -34,7 +28,22 @@ export const Scoped = {
 export const CacheStore = {
     DatabaseStore: {},
     DatabaseCountResult: {},
-    DatabaseStats: {},
+    DatabaseStats: {
+        /**
+         * @type {{[projectUrl: string]: {[dbUrl: string]: {[dbName: string]: {[path: string]: number}}}}}
+         */
+        database: {},
+        /**
+         * @type {{[projectUrl: string]: {[dbUrl: string]: {[dbName: string]: {[path: string]: boolean}}}}}
+         */
+        counters: {},
+        /**
+         * @type {{[projectUrl: string]: number}}
+         */
+        fetchers: {},
+        _db_size: 0,
+        _fetcher_size: 0
+    },
     AuthStore: {},
     PendingAuthPurge: {},
     EmulatedAuth: {},