corebasic 1.0.205 → 1.0.208

package/libs/elabase.js

@@ -2,7 +2,7 @@ import axios from 'axios'
 import {default as dipper, shard_stats as ShardStats} from './dipper.js'
 import * as Utils from './utils.js'
 import * as Cpp from './cpp.js'
-
+import {applySuffixPolicy} from './dip/suffix/policy.js'
 
 
 
@@ -74,17 +74,17 @@ export const globalMeta = args => {
 }
 export const companyMeta = (company, args) => {
     if (typeof company !== "string" && !Array.isArray(company))
-        throw  {message: "Error: Invalid company provided in Dip.companyMeta()"}
+        throw new Error("Error: Invalid company provided in Dip.companyMeta()")
     if (!company || (typeof company === "string" && company.trim() === "") ||  company === "GLOBAL" || (Array.isArray(company) && (!company.length || company.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL")) ))
-        throw  {message: "Error: Invalid company provided in Dip.companyMeta()"}
+        throw new Error("Error: Invalid company provided in Dip.companyMeta()")
 
     return new DipMeta({...args, outlet: "GLOBAL", company})
 }
 export const outletMeta = (outlet, args) => {
     if (typeof outlet !== "string" && !Array.isArray(outlet))
-        throw  {message: "Error: Invalid outlet provided in Dip.outletMeta()"}
+        throw new Error("Error: Invalid outlet provided in Dip.outletMeta()")
     if (!outlet || (typeof outlet === "string" && outlet.trim() === "") ||  outlet === "GLOBAL" || (Array.isArray(outlet) && (!outlet.length || outlet.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL")) ))
-        throw  {message: "Error: Invalid outlet provided in Dip.outletMeta()"}
+        throw new Error("Error: Invalid outlet provided in Dip.outletMeta()")
 
     return new DipMeta({...args, company: "GLOBAL", outlet})
 }
@@ -92,38 +92,38 @@ export const outletMeta = (outlet, args) => {
 export const customMeta = (company, outlet, args) => {
 
     if (typeof company !== "string" && !Array.isArray(company))
-        throw  {message: "Error: Invalid company provided in Dip.customMeta()"}
+        throw new Error("Error: Invalid company provided in Dip.customMeta()")
     if (typeof outlet !== "string" && !Array.isArray(outlet))
-        throw  {message: "Error: Invalid outlet provided in Dip.customMeta()"}
+        throw new Error("Error: Invalid outlet provided in Dip.customMeta()")
 
     if (!company || (typeof company === "string" && company.trim() === "") ||  company === "GLOBAL" || (Array.isArray(company) && (!company.length || company.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL")) ))
-        throw  {message: "Error: Invalid company provided in Dip.customMeta()"}
+        throw new Error("Error: Invalid company provided in Dip.customMeta()"}
     if (!outlet || (typeof outlet === "string" && outlet.trim() === "") ||  outlet === "GLOBAL" || (Array.isArray(outlet) && (!outlet.length || outlet.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL")) ))
-        throw  {message: "Error: Invalid outlet provided in Dip.customMeta()"}
+        throw new Error("Error: Invalid outlet provided in Dip.customMeta()")
 
     return new DipMeta({...args, company, outlet})
 }
 
 function validate_collection(collection, fn_name) {
     if ((typeof collection === 'string' && collection.trim() === "") || (typeof collection !== 'string' && !Array.isArray(collection)) || (Array.isArray(collection) && (!collection.length || collection.some(item => typeof item !== "string" || item.trim() === "")))) {
-        throw {message: `Error: Invalid collection ${fn_name}`}
+        throw new Error(`Error: Invalid collection ${fn_name}`)
     }
 }
 
 function validate_meta(meta, fn_name) {
     if (!meta || typeof meta !== 'object') {
-        throw {message: `Error: Invalid meta ${fn_name}`}
+        throw new Error(`Error: Invalid meta ${fn_name}`)
     }
     if ((typeof meta.company === 'string' && meta.company.trim() === "") || (typeof meta.company !== 'string' && !Array.isArray(meta.company)) || (Array.isArray(meta.company) && (!meta.company.length || meta.company.some(item => typeof item !== "string"|| item.trim() === "")))) {
-        throw {message: `Error: Invalid meta.company ${fn_name}`}
+        throw new Error(`Error: Invalid meta.company ${fn_name}`)
     }
     if ((typeof meta.outlet === 'string' && meta.outlet.trim() === "") || (typeof meta.outlet !== 'string' && !Array.isArray(meta.outlet)) || (Array.isArray(meta.outlet) && (!meta.outlet.length || meta.outlet.some(item => typeof item !== "string" || item.trim() === "")))) {
-        throw {message: `Error: Invalid meta.outlet ${fn_name}`}
+        throw new Error(`Error: Invalid meta.outlet ${fn_name}`)
     }
     if ((meta.suffix !== undefined && typeof meta.suffix !== 'string' && !Array.isArray(meta.suffix)) || (Array.isArray(meta.suffix) && meta.suffix.some(item => typeof item !== "string" || item.trim() === ""))) {
-        throw {message: `Error: Invalid meta.suffix ${fn_name}`}
+        throw new Error(`Error: Invalid meta.suffix ${fn_name}`)
     } else if (typeof meta.suffix === 'string' && meta.suffix.trim() === "")
-        throw {message: `Error: Invalid meta.suffix ${fn_name}`}
+        throw new Error(`Error: Invalid meta.suffix ${fn_name}`)
 }
 
 function generate_suffix(meta) {
@@ -202,7 +202,7 @@ export const batch = async (callback, arg) => { // Convenience function with aut
 // Ported
 export function batchSubmit(batch_id, arg) {
     if (!batch_id || !batches[batch_id] || !batches[batch_id].length)
-        throw {message: `Error: Invalid batch in Dip.batchSubmit()`}
+        throw new Error(`Error: Invalid batch in Dip.batchSubmit()`)
     let txns = batches[batch_id]
     const executor = arg?.executor ?? batches[batch_id].find(txn => txn.executor)?.executor
     const syncMode = arg?.syncMode ?? batches[batch_id].find(txn => txn.syncMode)?.syncMode
@@ -231,7 +231,7 @@ export const schema = async (name) => {
     } catch (e) {
         // Delete the broken promise from cache so a retry can occur later
         delete SCHEMAS_JSON_PROMISES[name]
-        throw {message: `Error: Failed to load schema ${name} in Dip`}
+        throw new Error(`Error: Failed to load schema ${name} in Dip`)
     }
 }
 
@@ -248,13 +248,20 @@ function init_collections_config() {
     for (let collection in COLLECTIONS_JSON) {
         let config = COLLECTIONS_JSON[collection]
         if (config.suffix !== undefined && (typeof config.suffix !== "object" || Array.isArray(config.suffix)) ) {
-            throw {message: `Error: Initiating collection configs failed. Invalid suffix in collection: ${collection}`}
+            throw new Error(`Error: Initiating collection configs failed. Invalid suffix in collection: ${collection}`)
         }
     }
 }
 if (COLLECTIONS_JSON)
     init_collections_config()
 
+const EXCLUDED_COLLECTIONS_CONFIG = {}
+export function excludeCollectionConfig(collection) {
+    EXCLUDED_COLLECTIONS_CONFIG[collection] = true
+}
+export function includeCollectionConfig(collection) {
+    delete EXCLUDED_COLLECTIONS_CONFIG[collection]
+}
 
 function validate_topology(meta, collection, fn_name) {
 
@@ -262,15 +269,16 @@ function validate_topology(meta, collection, fn_name) {
 
     let config = COLLECTIONS_JSON?.[collection]
     if (!config) {
-        console.warn(`Warn: Missing config for collection: ${collection}${fn_name ? ` ${fn_name}` : ''}`)
+        if (!EXCLUDED_COLLECTIONS_CONFIG[collection])
+            console.warn(`Warn: Missing config for collection: ${collection}${fn_name ? ` ${fn_name}` : ''}`)
         return new_meta
     }
 
     function check_core_meta(config, meta, suffix) {
         if (config.company !== meta.company && typeof config.company !== "object")
-            throw {message: `Error: Incompatible meta.company config for collection: ${collection}${suffix ? ' suffix: ' + suffix : ''}${fn_name ? ` ${fn_name}` : ''}`}
+            throw new Error(`Error: Incompatible meta.company config for collection: ${collection}${suffix ? ' suffix: ' + suffix : ''}${fn_name ? ` ${fn_name}` : ''}`)
         if (config.outlet !== meta.outlet && typeof config.outlet !== "object")
-            throw {message: `Error: Incompatible meta.outlet config for collection: ${collection}${suffix ? ' suffix: ' + suffix : ''}${fn_name ? ` ${fn_name}` : ''}`}
+            throw new Error(`Error: Incompatible meta.outlet config for collection: ${collection}${suffix ? ' suffix: ' + suffix : ''}${fn_name ? ` ${fn_name}` : ''}`)
     }
 
     check_core_meta(config, meta)
@@ -315,7 +323,8 @@ export const insert = async (meta, collection, value, options, extras) => {
         ...(meta.DIP_DB ? {db: meta.DIP_DB} : {}),
     }
     arg = Object.assign(arg, extras)
-    arg.suffix = generate_suffix(meta)
+    arg.suffix = await applySuffixPolicy(COLLECTIONS_JSON, collection, arg.insert, true, arg)
+    arg.suffix = (arg.suffix ?? []).concat(generate_suffix(meta))
     arg.executor = meta.executor ?? "native"
     arg = Object.assign({}, topology_meta, arg)
     if (options?._id) {
@@ -325,7 +334,7 @@ export const insert = async (meta, collection, value, options, extras) => {
 
     if (meta.batch) {
         if (!batches[meta.batch])
-            throw {message: 'Error: Invalid batch in Dip.insert()'}
+            throw new Error('Error: Invalid batch in Dip.insert()')
         batches[meta.batch].push(arg)
         return
     }
@@ -357,12 +366,13 @@ export const query = async (meta, collection, query, options, extras) => {
         ...(meta.DIP_DB ? {db: meta.DIP_DB} : {}),
     }
     arg = Object.assign(arg, extras)
-    arg.suffix = generate_suffix(meta)
+    arg.suffix = await applySuffixPolicy(COLLECTIONS_JSON, collection, arg.query, false, arg)
+    arg.suffix = (arg.suffix ?? []).concat(generate_suffix(meta))
     arg.executor = meta.executor ?? "native"
     arg = Object.assign({}, topology_meta, arg)
     if (meta.batch) {
         if (!batches[meta.batch])
-            throw {message: 'Error: Invalid batch in Dip.query()'}
+            throw new Error('Error: Invalid batch in Dip.query()')
         batches[meta.batch].push(arg)
         return
     }
@@ -408,12 +418,13 @@ export const update = async (meta, collection, query, update, options, extras) =
         ...(meta.DIP_DB ? {db: meta.DIP_DB} : {}),
     }
     arg = Object.assign(arg, extras)
-    arg.suffix = generate_suffix(meta)
+    arg.suffix = await applySuffixPolicy(COLLECTIONS_JSON, collection, arg.query, false, arg)
+    arg.suffix = (arg.suffix ?? []).concat(generate_suffix(meta))
     arg.executor = meta.executor ?? "native"
     arg = Object.assign({}, topology_meta, arg)
     if (meta.batch) {
         if (!batches[meta.batch])
-            throw {message: 'Error: Invalid batch in Dip.update()'}
+            throw new Error('Error: Invalid batch in Dip.update()')
         batches[meta.batch].push(arg)
         return
     }
@@ -433,7 +444,7 @@ export const remove = async (meta, collection, query, options, extras) => {
     validate_collection(collection, 'in Dip.remove()')
 
     if (Utils.isEmpty(query._id) && !meta.allowDangerousRemove)
-        throw { message: "Error: Dip.remove() without _id blocked by default (Dangerous Operation). Set meta.allowDangerousRemove if intentional." }
+        throw new Error("Error: Dip.remove() without _id blocked by default (Dangerous Operation). Set meta.allowDangerousRemove if intentional.")
 
     validate_meta(meta, 'in Dip.remove()')
 
@@ -449,12 +460,13 @@ export const remove = async (meta, collection, query, options, extras) => {
         ...(meta.DIP_DB ? {db: meta.DIP_DB} : {}),
     }
     arg = Object.assign(arg, extras)
-    arg.suffix = generate_suffix(meta)
+    arg.suffix = await applySuffixPolicy(COLLECTIONS_JSON, collection, arg.query, false, arg)
+    arg.suffix = (arg.suffix ?? []).concat(generate_suffix(meta))
     arg.executor = meta.executor ?? "native"
     arg = Object.assign({}, topology_meta, arg)
     if (meta.batch) {
         if (!batches[meta.batch])
-            throw {message: 'Error: Invalid batch in Dip.query()'}
+            throw new Error('Error: Invalid batch in Dip.query()')
         batches[meta.batch].push(arg)
         return
     }
@@ -551,7 +563,7 @@ function parseBinaryResponse(result) {
         return {metadata: result.body, content: result.body.items}
     }
     if (!contentType.includes("application/dip")) {
-        throw {message: "Error: Dip unsupported media. Invalid content-type."}
+        throw new Error("Error: Dip unsupported media. Invalid content-type.")
     }
 
     const arrayBuffer = result.body

package/libs/features.js

@@ -43,13 +43,13 @@ let SERVICE_ADDRESS = process.env.APP_ENDPOINT || 'http://127.0.0.1:3000'
 
 
 export const get = feature => {
-    const throwError = () => {throw {message: `Feature ${feature} not found in internal or external list during Features.subscribe call`}}
+    const throwError = () => {throw new Error(`Feature ${feature} not found in internal or external list during Features.subscribe call`)}
     let {api, service = SERVICE_ADDRESS, topic} = getFeature(feature) ?? SLYP_FEATURES_LIST[feature] ?? throwError()
     return {api, service, topic}
 }
 
 export const send = async (meta, feature, data, params) => {
-    const throwError = () => {throw {message: `Feature ${feature} not found in internal or external list during inter feature call`}}
+    const throwError = () => {throw new Error(`Feature ${feature} not found in internal or external list during inter feature call`)}
     let {api, service = SERVICE_ADDRESS} = getFeature(feature) ?? SLYP_FEATURES_LIST[feature] ?? throwError()
     let method = getFeatureMethod(api)
     let url = getFeatureUrl(api)
@@ -58,7 +58,7 @@ export const send = async (meta, feature, data, params) => {
         for (let key in params)
             url = url.replace(`:${key}`, params[key])
         if (url.includes(':'))
-            throw {feature, when: 'api/params/substitution', message: "Internal Feature Call" }
+            throw new Error(`Error: Internal feature call send on feature: ${feature} when api/params/substitution`)
     }
 
     let payload = { ...meta, data, feature, txn: Utils.uid() }
@@ -217,6 +217,9 @@ function registerApi() {
 }
 
 export const start = async (app, url, file) => {
+    const globalMeta = Dip.globalMeta()
+    Dip.excludeCollectionConfig("users.txns")
+    Dip.excludeCollectionConfig("Features.txns")
     ExpressApp = app
     PROJECT_ROOT_URL = url
     features = await Utils.fileToJson(url, file)
@@ -257,9 +260,12 @@ export const start = async (app, url, file) => {
             topic = topic.replace(/^.*Features./,'')
             while (true) {
                 try {
-                    await Dip.update(message.meta, `users.txns.${message.user}`, {_id: message.txn}, { $setOnInsert: { user: message.user, feature: message.feature, date: message.date, created: message.date, updated: message.date, status: "Queued" } }, {upsert: true}) // Can always update the status later
+                    // const iso_date = new Date(message.date).toISOString()
+                    // // TODO: use $useChunks: [] once support is added in dip insert
+                    // await Dip.insert(globalMeta, `users.txns`, { _id: `${message.user}_${iso_date}_${message.txn}`, user: message.user, feature: message.feature, date: message.date, created: message.date, updated: message.date, status: "Queued" }, {idempotent: true}) // Can always update the status later
+                    // TODO: Avoid duplicate processing
                     await features[message.feature].handler(topic, message)
-                    await Dip.insert(message.meta, "Features.txns", {_id: message.txn, status: "Processed"})
+                    await Dip.insert(globalMeta, "Features.txns", {_id: message.txn, status: "Processed"}, {idempotent: true})
                     break
                 } catch(err) {
                     if (process.env.DEBUG_MODE)
@@ -276,7 +282,7 @@ export const start = async (app, url, file) => {
     // Transaction status api
     app.get('/transactions/:id', async (req, res) => { // Front end calls this as a regular feature call. so `req.meta` exists
         try {
-            let items = await Dip.query(req.meta, "Features.txns", {_id: req.params.id})
+            let items = await Dip.query(globalMeta, "Features.txns", {_id: req.params.id})
             if (items.length)
                 res.json({ data: { ...items[0], txn: items._id } })
             else
@@ -285,13 +291,17 @@ export const start = async (app, url, file) => {
             res.status(500).json({ message: "Failed to retreive data" })
         }
     })
-    app.get('/users/:id/transactions', async (req, res) => {
-        try {
-            res.json({ data: await Dip.query(req.meta, `users.txns.${req.params.id}`, {}, { $orderby: { date: -1 } }) })
-        } catch {
-            res.status(500).json({ message: "Failed to retreive data" })
-        }
-    })
+    // app.get('/users/:id/transactions', async (req, res) => {
+    //     try {
+    //         const idPrefix = `${req.params.id}_`
+    //         // TODO: use $useChunks: [] once support is added in dip insert
+    //         // TODO: remove $orderby: { date: -1 } and remove $orderby: { _id: -1 }
+    //         // res.json({ data: await Dip.query(globalMeta, `users.txns`, {}, { $rangeFrom: idPrefix, $rangeTo: idPrefix, $rangeDirection: -1, $rangeInclusive: true, $orderby: { date: -1 } }) })
+    //         res.json({ data: await Dip.query(globalMeta, `users.txns`, {}, { $rangeFrom: idPrefix, $rangeTo: idPrefix, $rangeDirection: -1, $rangeInclusive: true, $orderby: { _id: -1 } }) })
+    //     } catch {
+    //         res.status(500).json({ message: "Failed to retreive data" })
+    //     }
+    // })
 
     try { await Messaging.start(app) } catch { console.log('Messaging failed to start. Maybe missing redis') }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "corebasic",
   "type": "module",
-  "version": "1.0.205",
+  "version": "1.0.208",
   "description": "",
   "main": "index.js",
   "scripts": {