@cocreate/mongodb 1.7.2 → 1.9.0

package/src/index.js

@@ -20,52 +20,44 @@ async function dbClient(data) {
     return
 }
 
-async function databaseStats(data) {
-    const client = await dbClient(data)
-    if (!client) return
-    const db = client.db(data.organization_id)
-    const stats = await db.stats()
-    stats.storage = data.storageName
-    if (!data.stats)
-        data.stats = [stats]
-    else
-        data.stats.push(stats)
-    return data
-}
-
-function createDatabase(data) {
-    return database('createDatabase', data)
-}
-
-function readDatabase(data) {
-    return database('readDatabase', data)
-}
-
-function updateDatabase(data) {
-    return database('updateDatabase', data)
-}
-
-function deleteDatabase(data) {
-    return database('deleteDatabase', data)
+function send(data) {
+    let [method, type] = data.method.split('.')
+    // let action = data.method.replace(/\.([a-z])/g, (_, match) => match.toUpperCase());
+
+    if (type === 'database')
+        return database(method + type.charAt(0).toUpperCase() + type.slice(1), data)
+    if (type === 'array')
+        return array(method + type.charAt(0).toUpperCase() + type.slice(1), data)
+    if (type === 'object')
+        return object(method + type.charAt(0).toUpperCase() + type.slice(1), data)
 }
 
 function database(action, data) {
     return new Promise(async (resolve, reject) => {
         let type = 'database'
         let databaseArray = []
+        let dataTransferedIn = 0
+        let dataTransferedOut = 0
 
         try {
             const client = await dbClient(data)
             if (!client) return
             if (action == 'readDatabase') {
                 const db = client.db().admin();
+                // TODO: support if a database name is defined then return the database details and stats
+                // else apply filter and return dbs for which each should have stats and details
+                // if (data.database) {
+                //     const db = client.db(data.organization_id)
+                //     const stats = await db.stats()
+                // }
 
                 // List all the available databases
                 db.listDatabases(function (err, dbs) {
+                    dataTransferedIn += getBytes(dbs)
 
                     for (let database of dbs.databases) {
-                        if (data.filter && data.filter.query) {
-                            let isFilter = queryData(database, data.filter.query)
+                        if (data.$filter && data.$filter.query) {
+                            let isFilter = queryData(database, data.$filter.query)
                             if (isFilter)
                                 databaseArray.push({ database, storage: data.storageName })
                         } else
@@ -77,8 +69,9 @@ function database(action, data) {
             }
             if (action == 'deleteDatabase') {
                 const db = client.db(data.database);
-                db.dropDatabase().then(response => {
-                    resolve(response)
+                db.dropDatabase().then(restult => {
+                    dataTransferedIn += getBytes(restult)
+                    resolve(restult)
                 })
             }
         } catch (error) {
@@ -92,27 +85,12 @@ function database(action, data) {
     });
 }
 
-
-function createCollection(data) {
-    return array('createCollection', data)
-}
-
-function readCollection(data) {
-    return array('readCollection', data)
-}
-
-function updateCollection(data) {
-    return array('updateCollection', data)
-}
-
-function deleteCollection(data) {
-    return array('deleteCollection', data)
-}
-
 function array(action, data) {
     return new Promise(async (resolve, reject) => {
         let type = 'array'
         let arrayArray = [];
+        let dataTransferedIn = 0
+        let dataTransferedOut = 0
 
         try {
             const client = await dbClient(data)
@@ -131,16 +109,18 @@ function array(action, data) {
 
                 if (action == 'readCollection') {
 
-                    let { query, sort } = getFilters(data);
+                    let { query, sort } = await createFilter(data);
 
                     db.listCollections().toArray(function (error, result) {
                         if (error)
                             errorHandler(data, error, database)
 
                         if (result) {
+                            dataTransferedIn += getBytes(result)
+
                             for (let res of result) {
-                                if (data.filter && data.filter.query) {
-                                    let isFilter = queryData(res, data.filter.query)
+                                if (data.$filter && data.$filter.query) {
+                                    let isFilter = queryData(res, data.$filter.query)
                                     if (isFilter)
                                         arrayArray.push({ name: res.name, database, storage: data.storageName })
                                 } else
@@ -150,7 +130,7 @@ function array(action, data) {
 
                         databasesLength -= 1
                         if (!databasesLength) {
-                            data = createData(data, arrayArray, type)
+                            data = createData(data, arrayArray, type, dataTransferedIn, dataTransferedOut)
                             resolve(data)
                         }
                     })
@@ -169,19 +149,22 @@ function array(action, data) {
                     for (let array of arrays) {
 
                         if (action == 'createCollection') {
+                            dataTransferedOut += getBytes(array)
                             db.createCollection(array, function (error, result) {
                                 if (error)
                                     errorHandler(data, error, database, array)
 
-                                if (result)
+                                if (result) {
+                                    dataTransferedIn += getBytes(result)
                                     arrayArray.push({ name: array, database, storage: data.storageName })
+                                }
 
                                 arraysLength -= 1
                                 if (!arraysLength)
                                     databasesLength -= 1
 
                                 if (!databasesLength && !arraysLength) {
-                                    data = createData(data, arrayArray, type)
+                                    data = createData(data, arrayArray, type, dataTransferedIn, dataTransferedOut)
                                     resolve(data)
                                 }
                             })
@@ -193,19 +176,21 @@ function array(action, data) {
                             const arrayObj = db.collection(array);
 
                             if (action == 'updateCollection') {
+                                dataTransferedOut += getBytes(result)
                                 arrayObj.rename(value, function (error, result) {
                                     if (error)
                                         errorHandler(data, error, database, array)
 
-                                    if (result)
+                                    if (result) {
+                                        dataTransferedIn += getBytes(result)
                                         arrayArray.push({ name: value, oldName: array, database, storage: data.storageName })
-
+                                    }
                                     arraysLength -= 1
                                     if (!arraysLength)
                                         databasesLength -= 1
 
                                     if (!databasesLength && !arraysLength) {
-                                        data = createData(data, arrayArray, type)
+                                        data = createData(data, arrayArray, type, dataTransferedIn, dataTransferedOut)
                                         resolve(data)
                                     }
 
@@ -217,15 +202,17 @@ function array(action, data) {
                                     if (error)
                                         errorHandler(data, error, database, array)
 
-                                    if (result)
+                                    if (result) {
+                                        dataTransferedOut += getBytes(result)
                                         arrayArray.push({ name: array, database, storage: data.storageName })
+                                    }
 
                                     arraysLength -= 1
                                     if (!arraysLength)
                                         databasesLength -= 1
 
                                     if (!databasesLength && !arraysLength) {
-                                        data = createData(data, arrayArray, type)
+                                        data = createData(data, arrayArray, type, dataTransferedIn, dataTransferedOut)
                                         resolve(data)
                                     }
 
@@ -248,324 +235,169 @@ function array(action, data) {
     });
 }
 
-
-function createObject(data) {
-    return object('createObject', data)
-}
-
-function readObject(data) {
-    return object('readObject', data)
-}
-
-function updateObject(data) {
-    return object('updateObject', data)
-}
-
-function deleteObject(data) {
-    return object('deleteObject', data)
-}
-
 function object(action, data) {
     return new Promise(async (resolve, reject) => {
         try {
             const client = await dbClient(data)
             if (!client) return
-
+            let dataTransferedIn = 0
+            let dataTransferedOut = 0
             let type = 'object'
             let documents = [];
 
             if (data.request)
                 data[type] = data.request
 
-
             if (!data['timeStamp'])
                 data['timeStamp'] = new Date().toISOString()
 
-
-            let isFilter
-            if (data.filter && data.filter.query)
-                isFilter = true
-
             let databases = data.database;
             if (!Array.isArray(databases))
                 databases = [databases]
 
-            let databasesLength = databases.length
             for (let database of databases) {
                 let arrays = data.array;
                 if (!Array.isArray(arrays))
                     arrays = [arrays]
 
-                let arraysLength = arrays.length
                 for (let array of arrays) {
                     const db = client.db(database);
                     const arrayObj = db.collection(array);
+                    const reference = { $storage: data.storageName, $database: database, $array: array }
 
-                    let { query, sort } = getFilters(data);
-                    if (data['organization_id']) {
-                        query['organization_id'] = { $eq: data['organization_id'] }
-                    }
-
-                    let _ids = []
-                    let update_ids = []
-                    let updateData = {}
-
-                    if (data[type]) {
-                        if (!Array.isArray(data[type]))
-                            data[type] = [data[type]]
-                        for (let i = 0; i < data[type].length; i++) {
-                            data[type][i] = replaceArray(data[type][i])
-                            data[type][i]['organization_id'] = data['organization_id'];
+                    if (data[type] && !Array.isArray(data[type]))
+                        data[type] = [data[type]]
 
+                    let isFilter
+                    if (data.$filter && data.$filter.query)
+                        isFilter = true
 
-                            if (action == 'createObject') {
-                                data[type][i] = dotNotationToObject(data[type][i])
+                    let filter = await createFilter(data, arrayObj);
 
-                                if (!data[type][i]._id)
-                                    data[type][i]._id = ObjectId()
-                                else
-                                    data[type][i]._id = ObjectId(data[type][i]._id)
-                                data[type][i]['created'] = { on: data.timeStamp, by: data.user_id || data.clientId }
-                            }
-                            if (action == 'readObject' && data[type][i]._id) {
-                                _ids.push(ObjectId(data[type][i]._id))
-                            }
-                            if (action == 'updateObject') {
-                                if (data[type][i]._id)
-                                    update_ids.push({ _id: data[type][i]._id, updateDoc: data[type][i], updateType: '_id' })
+                    let projections = {}, projection = {}, update = {}, options = {}
 
-                                if (!data[type][i]._id)
-                                    updateData = createUpdate({ object: [data[type][i]] }, type)
+                    if (isFilter && (!data[type] || !data[type].length))
+                        data[type] = [{}]
 
-                                data[type][i]['modified'] = { on: data.timeStamp, by: data.user_id || data.clientId }
+                    if (action === 'updateObject')
+                        createUpdate(update, options, data, true)
 
-                            }
-                            if (action == 'deleteObject') {
-                                if (data[type][i]._id) {
-                                    _ids.push(ObjectId(data[type][i]._id))
-                                    documents.push({ _id: data[type][i]._id, storage: data.storageName, database, array })
-                                }
-                            }
+                    for (let i = 0; i < data[type].length; i++) {
+                        if (action !== 'createObject' && data[type][i].$filter) {
+                            isFilter = true
+                            reference['$filter'] = data[type][i].$filter
+                            filter = await createFilter({ $filter: data[type][i].$filter }, arrayObj)
                         }
-                        if (_ids.length == 1)
-                            query['_id'] = ObjectId(_ids[0])
-                        else if (_ids.length > 0)
-                            query['_id'] = { $in: _ids }
-                    }
-
-
-                    if (action == 'createObject') {
-                        arrayObj.insertMany(data[type], function (error, result) {
-                            if (error)
-                                errorHandler(data, error, database, array)
-
-                            for (let i = 0; i < data[type].length; i++)
-                                documents.push({ storage: data.storageName, database, array, ...data[type][i] })
-
-                            arraysLength -= 1
-                            if (!arraysLength)
-                                databasesLength -= 1
 
-                            if (!databasesLength && !arraysLength) {
-                                data = createData(data, documents, type)
-                                resolve(data)
-                            }
-                        });
-                    }
+                        let { query, sort, index, limit } = filter
 
-                    if (action == 'readObject') {
-                        let index = 0, limit = 0
-                        if (data.filter) {
-                            const count = await arrayObj.estimatedDocumentCount()
-                            data.filter.count = count
-
-                            if (data.filter.index)
-                                index = data.filter.index
-                            if (data.filter.limit)
-                                limit = data.filter.limit
-                            if (limit)
-                                limit = index + limit;
+                        if (action === 'createObject') {
+                            data[type][i] = replaceArray(data[type][i])
+                            data[type][i] = dotNotationToObject(data[type][i])
+                            data[type][i]._id = ObjectId(data[type][i]._id)
+                            data[type][i]['organization_id'] = data['organization_id'];
+                            data[type][i]['created'] = { on: data.timeStamp, by: data.user_id || data.clientId }
+                        } else if (action === 'readObject') {
+                            projection = createProjection(data[type][i])
+                        } else if (action === 'updateObject') {
+                            data[type][i].modified = { on: data.timeStamp, by: data.user_id || data.clientId }
+                            data[type][i].organization_id = data.organization_id
+                            createUpdate(update, options, data[type][i])
                         }
 
-                        arrayObj.find(query).skip(index).limit(limit).sort(sort).toArray(function (error, result) {
-                            if (error)
-                                errorHandler(data, error, database, array)
-
-                            if (result) {
-                                // TODO: forEach at cursor
-                                for (let doc of result) {
-                                    let isMatch = true
-                                    if (data.filter && data.filter['search'])
-                                        isMatch = searchData(doc, data.filter['search'])
-                                    if (isMatch) {
-                                        doc.storage = 'mongodb'
-                                        doc.database = database
-                                        doc.array = array
-                                        doc._id = doc._id.toString()
-
-                                        if (data.returnObject == false) {
-                                            let tempDoc = {};
-                                            let docs = new Map(data[type].map((obj) => [obj._id, obj]));
-                                            let doc1 = docs.get(doc._id)
-                                            if (doc1) {
-                                                tempDoc._id = tempDoc
-                                                for (let key of Object.keys(doc1)) {
-                                                    tempDoc[key] = doc[key]
-                                                }
-                                                doc = tempDoc
-                                            }
-                                        }
+                        if (data[type][i]._id) {
+                            try {
+                                if (action !== 'createObject') {
+                                    query._id = ObjectId(data[type][i]._id)
+                                }
 
-                                        documents.push(doc)
-                                    }
+                                dataTransferedOut += getBytes({ query, update, projection, options })
+
+                                let result
+                                if (action === 'createObject') {
+                                    let _id = data[type][i]._id.toString()
+                                    // TODO: type error occuring when pushing the item pushes but throws an error
+                                    documents.push({ ...data[type][i], ...reference, _id })
+                                } else if (action === 'readObject') {
+                                    result = await arrayObj.findOne(query, projection);
+                                    result._id = result._id.toString()
+                                    documents.push({ ...result, ...reference })
+                                } else if (action === 'updateObject') {
+                                    result = await arrayObj.updateOne(query, update, options);
+                                    documents.push({ ...data[type][i], ...reference })
+                                } else if (action === 'deleteObject') {
+                                    result = await arrayObj.deleteOne(query);
+                                    documents.push({ ...reference, _id: data[type][i]._id })
                                 }
+                                dataTransferedIn += getBytes(result)
 
-                                // if (index && limit) {
-                                //     documents = documents.slice(index, limit)
-                                // }
+                            } catch (error) {
+                                errorHandler(data, error, database, array)
                             }
+                        } else if (isFilter) {
+                            try {
+                                if (action === 'readObject')
+                                    projection = { ...projections, ...projection }
 
-                            arraysLength -= 1
-                            if (!arraysLength)
-                                databasesLength -= 1
-
-                            if (!databasesLength && !arraysLength) {
-                                data = createData(data, documents, type)
-                                resolve(data)
-                            }
-                        });
-                    }
+                                delete query._id
 
-                    if (action == 'updateObject' || action == 'deleteObject') {
-                        const queryDocs = () => {
-                            return new Promise(async (resolve, reject) => {
+                                dataTransferedOut += getBytes({ query, projection, sort, index, limit })
+                                let document = ''
 
-                                arrayObj.find(query).sort(sort).toArray(function (error, result) {
-                                    if (error)
-                                        errorHandler(data, error, database, array)
+                                const cursor = arrayObj.find(query, projection).sort(sort).skip(index).limit(limit);
+                                if (!(await cursor.hasNext()) && action === 'updateObject' && data.upsert)
+                                    document = { _id: ObjectId(data[type][i]._id) }
+                                while (await cursor.hasNext() || document) {
+                                    if (!document)
+                                        document = await cursor.next();
 
-                                    if (data.filter && data.filter.search) {
-                                        let searchResult = []
+                                    dataTransferedIn += getBytes(document)
 
-                                        for (let doc of result) {
-                                            let isMatch = searchData(doc, data.filter.search)
-                                            if (isMatch)
-                                                searchResult.push(doc)
-                                        }
-                                        result = searchResult
+                                    if (data.$filter && data.$filter.search) {
+                                        let isMatch = searchData(document, data.$filter.search)
+                                        if (!isMatch)
+                                            continue;
                                     }
-                                    resolve(result)
-                                })
-                            }, (err) => {
-                                console.log(err);
-                            });
-                        }
 
-                        let Result, $update, update, projection;
+                                    if (action === 'readObject') {
+                                        documents.push({ ...document, ...reference, _id: document._id.toString() })
+                                    } else {
+                                        dataTransferedOut += getBytes({ _id: document._id, update, options })
 
-                        if (isFilter && data.returnObject != false)
-                            if (action == 'deleteObject' || action == 'updateObject' && updateData.update)
-                                Result = await queryDocs()
-
-                        if (Result) {
-                            for (let doc of Result) {
-                                if (action == 'deleteObject')
-                                    documents.push({ _id: doc._id, storage: data.storageName, database, array })
-                                else
-                                    doc['modified'] = { on: data.timeStamp, by: data.user_id || data.clientId }
-
-                                _ids.push(doc._id)
-                            }
-                            update_ids.push({ updateType: 'filter' })
-                        }
-
-                        if (action == 'updateObject') {
-                            let docsLength = update_ids.length
-                            for (let { updateDoc, updateType } of update_ids) {
-
-                                if (updateType == '_id') {
-                                    let update_id = updateDoc._id
-                                    query['_id'] = ObjectId(update_id)
-                                    $update = createUpdate({ object: [updateDoc] }, type)
-                                    update = $update.update
-                                    projection = $update.projection
-                                    documents.push({ _id: update_id, storage: data.storageName, database, array, ...update['$set'] })
-                                }
-
-                                if (updateType == 'filter') {
-                                    query['_id'] = { $in: _ids }
-                                    $update = updateData
-                                    update = $update.update
-                                    projection = $update.projection
-                                    for (let _id of _ids)
-                                        documents.push({ _id, storage: data.storageName, database, array, ...update['$set'] })
-
-                                }
-
-                                update['$set']['organization_id'] = data.organization_id
-
-                                arrayObj.updateMany(query, update, {
-                                    upsert: data.upsert,
-                                    projection
-                                }).then((result) => {
-
-                                }).catch((error) => {
-                                    errorHandler(data, error, database, array)
-                                    console.log(action, 'error', error);
-                                }).finally((error) => {
-                                    docsLength -= 1
-                                    if (!docsLength)
-                                        arraysLength -= 1
-
-                                    if (!arraysLength)
-                                        databasesLength -= 1
+                                        let result
+                                        if (action === 'updateObject') {
+                                            result = await arrayObj.updateOne({ _id: document._id }, update, options);
+                                        } else if (action === 'deleteObject') {
+                                            result = await arrayObj.deleteOne({ _id: document._id });
+                                        }
 
-                                    if (!databasesLength && !arraysLength) {
-                                        data = createData(data, documents, type)
-                                        resolve(data)
+                                        dataTransferedIn += getBytes(result)
+                                        documents.push({ ...data[type][i], ...reference, _id: document._id.toString() })
                                     }
-                                })
-                            }
-
-                            if (!update_ids.length) {
-                                docsLength -= 1
-                                if (!docsLength)
-                                    arraysLength -= 1
-
-                                if (!arraysLength)
-                                    databasesLength -= 1
-
-                                if (!databasesLength && !arraysLength) {
-                                    data = createData(data, documents, type)
-                                    resolve(data)
+                                    document = ''
                                 }
+                            } catch (error) {
+                                errorHandler(data, error, database, array)
                             }
-
                         }
+                    }
 
-                        if (action == 'deleteObject') {
-                            if (_ids.length == 1)
-                                query['_id'] = ObjectId(_ids[0])
-                            else if (_ids.length > 0)
-                                query['_id'] = { $in: _ids }
-                            arrayObj.deleteMany(query, function (error, result) {
-                                arraysLength -= 1
-                                if (!arraysLength)
-                                    databasesLength -= 1
-
-                                if (!databasesLength && !arraysLength) {
-                                    data = createData(data, documents, type)
-                                    resolve(data)
-                                }
-
-                            })
+                    if (action === 'createObject') {
+                        try {
+                            dataTransferedOut += getBytes(data[type])
+                            const result = await arrayObj.insertMany(data[type]);
+                            dataTransferedIn += getBytes(result)
+                        } catch (error) {
+                            errorHandler(data, error, database, array)
                         }
-
                     }
 
                 }
             }
 
+            data = createData(data, documents, type, dataTransferedIn, dataTransferedOut)
+            resolve(data)
+
         } catch (error) {
             errorHandler(data, error)
             console.log(action, 'error', error);
@@ -577,110 +409,136 @@ function object(action, data) {
 
 }
 
-function createUpdate(data, type) {
-    let update = {}, projection = {};
-    if (data[type][0]) {
-        update['$set'] = data[type][0]
-        // update['$set']['organization_id'] = data['organization_id'];
-        if (update['$set']['_id'])
-            delete update['$set']['_id']
-        Object.keys(update['$set']).forEach(key => {
-
-            if (key.includes('[u]')) {
-                update['$addToSet'] = { [key.replace('[u]', '')]: update['$set'][key] }
-                delete update['$set'][key]
-            } else if (key.includes('[]')) {
-                if (!Array.isArray(update['$set'][key]))
-                    update['$set'][key] = [update['$set'][key]]
-
-                update['$push'] = {
-                    [key.replace('[]', '')]: { $each: update['$set'][key] }
-                }
-                delete update['$set'][key]
-            }
-            // { $push: { "skills": { $each: ["Sports", "Acting"] } } })
-            // { $addToSet: { "skills": "GST" } }) // adds "GST"to all arrays if not exist
-            projection[key] = 1
-        })
-    }
+function createUpdate(update, options, data, isGlobal) {
+    if (data.upsert)
+        options.upsert = data.upsert
+    if (data.$upsert)
+        options.upsert = data.$upsert
+
+    Object.keys(data).forEach(key => {
+        if (isGlobal && !key.startsWith('$') || key === '_id')
+            return
+
+        let operator
+        if (key.endsWith(']')) {
+            const regex = /^(.*(?:\[\d+\].*?)?)\[(.*?)\](?:\[\])?$/;
+            var match = key.match(regex);
+            var index = parseInt(match[2], 10);
+            if (index === NaN)
+                operator = match[2]
+            var arrayKey = match[1].replace(/\[(\d+)\]/g, '.$1');
+        }
 
-    if (data['deleteName']) {
-        update['$unset'] = replaceArray(data['deleteName']);
-    }
+        if (key.startsWith('$'))
+            operator = key.split('.')[0]
+        else if (!operator && typeof data[key] === 'string' && data[key].startsWith('$'))
+            operator = data[key]
 
-    if (data['updateName']) {
-        update['$rename'] = replaceArray(data['updateName'])
-        for (const [key, value] of Object.entries(update['$rename'])) {
-            if (/\.([0-9]*)/g.test(key) || /\[([0-9]*)\]/g.test(value)) {
-                console.log('key is array', /\[([0-9]*)\]/g.test(value), /\.([0-9]*)/g.test(key))
-            } else {
-                let newValue = replaceArray({ [value]: value })
-                let oldkey = key;
-                for (const [key] of Object.entries(newValue)) {
-                    update['$rename'][oldkey] = key
+        if (!update['$set'])
+            update['$set'] = {}
+
+        let originalKey = key
+        key = key.replace(/\[(\d+)\]/g, '.$1');
+
+        let operators = ['$rename', '$inc', '$push', '$splice', '$unset', '$delete', '$slice', '$pop']
+        if (!operators.includes(operator) && typeof index !== 'number') {
+            if (!isGlobal)
+                update['$set'][key] = data[originalKey]
+            return
+        }
+
+        let updates = {}
+        if (operator === '$rename') {
+            if (key === '$rename')
+                for (let oldkey of Object.keys(data[originalKey])) {
+                    key = '$rename.' + oldkey
+                    updates[key] = data[originalKey][oldkey].replace(/\[(\d+)\]/g, '.$1')
                 }
-            }
+            else
+                updates[key] = data[originalKey].replace(/\[(\d+)\]/g, '.$1')
+
+        } else if (operator === '$delete' || operator === '$unset' || operator === '$slice') {
+            operator = '$unset'
+            updates[key] = 1
+            if (!updates['$pull'])
+                updates['$pull'] = {}
+            updates['$pull'][key] = null
+        } else if (operator === '$pop') {
+            key = arrayKey
+            updates[key] = index || 1
+        } else if (operator === '$addToSet' || operator === '$pull') {
+            key = arrayKey
+            updates[key] = data[originalKey]
+        } else if (operator === '$push' || typeof index === 'number') {
+            if (!Array.isArray(data[originalKey]))
+                updates[key] = [data[originalKey]]
+            else
+                updates[key] = data[originalKey]
+
+            let insert = { $each: updates[key] }
+            if (typeof index === 'number' && index >= 0)
+                insert.$postion = index
+
+            updates[key] = insert
+        } else if (operator === '$inc') {
+            updates[key] = data[originalKey]
         }
-    }
 
-    return { update, projection }
+        if (!update[operator])
+            update[operator] = {}
 
+        if (key === operator)
+            update[operator] = { ...update[operator], ...replaceArray(updates[key]) }
+        else
+            update[operator][key.replace(operator, '')] = updates[key]
+    })
 }
 
-function createData(data, array, type) {
-    if (!data.request)
-        data.request = data[type] || {}
+async function createFilter(data, arrayObj) {
+    let query = {}, sort = {}, index = 0, limit = 0, count
 
-    if (data.filter && data.filter.sort)
-        data[type] = sortData(array, data.filter.sort)
-    else
-        data[type] = array
+    if (data.$filter) {
+        if (data.$filter.query)
+            query = createQuery(data.$filter.query);
 
-    if (data.returnLog) {
-        if (!data.log)
-            data.log = []
-        data.log.push(...data[type])
-    }
 
-    return data
-}
+        if (data.$filter.sort) {
+            for (let i = 0; i < data.$filter.sort.length; i++) {
+                let direction = data.$filter.sort[i].direction
+                if (direction == 'desc' || direction == -1)
+                    direction = -1;
+                else
+                    direction = 1;
 
-function getFilters(data) {
-    let query = {}, sort = {}
-    let filter = {
-        query: [],
-        sort: [],
-        search: {
-            value: [],
-            type: "or"
-        },
-        index: 0,
-        ...data.filter
-    };
-
-    query = createQuery(filter.query);
-
-
-    if (filter.sort) {
-        for (let i = 0; i < filter.sort.length; i++) {
-            let direction = filter.sort[i].direction
-            if (direction == 'desc' || direction == -1)
-                direction = -1;
-            else
-                direction = 1;
+                sort[data.$filter.sort[i].key] = direction
+            }
+        }
 
-            sort[filter.sort[i].key] = filter.sort[i].direction
+        if (arrayObj) {
+            count = await arrayObj.estimatedDocumentCount()
+            data.$filter.count = count
         }
+
+        if (data.$filter.index)
+            index = data.$filter.index
+        if (data.$filter.limit)
+            limit = data.$filter.limit
+        if (limit)
+            limit = index + limit;
     }
-    return { query, sort }
+
+    if (data['organization_id'])
+        query['organization_id'] = { $eq: data['organization_id'] }
+
+    return { query, sort, index, limit, count }
 }
 
-// TODO: create impved mongodb query to cover many cases
-function createQuery(filters) {
-    let query = new Object();
 
+// TODO: improve mongodb query to cover many cases
+function createQuery(queries) {
+    let query = new Object();
 
-    for (let item of filters) {
+    for (let item of queries) {
 
         if (!item.key)
             continue
@@ -731,6 +589,13 @@ function createQuery(filters) {
                 else
                     query[key] = { $in: item.value }
                 break;
+            case '$or':
+                if (!query[item.operator])
+                    query[item.operator] = [{ [key]: item.value }];
+                else
+                    query[item.operator].push({ [key]: item.value })
+                delete query[key]
+                break;
             case '$geoWithin':
                 try {
                     let value = JSON.parse(item.value);
@@ -755,6 +620,63 @@ function createQuery(filters) {
     return query;
 }
 
+function createProjection(data) {
+    let projection = {}
+
+    Object.keys(data).forEach((key) => {
+        if (!['_id', 'organization_id'].includes(key) && !key.startsWith('$'))
+            projection[key.replace(/\[(\d+)\]/g, '.$1')] = 1
+    });
+
+    return projection;
+}
+
+function replaceArray(data) {
+    let object = {}
+
+    Object.keys(data).forEach((key) => {
+        object[key.replace(/\[(\d+)\]/g, '.$1')] = data[key]
+    });
+
+    return object;
+}
+
+function createData(data, array, type, dataTransferedIn, dataTransferedOut) {
+    if (dataTransferedIn) {
+        process.emit("setBandwidth", {
+            type: 'in',
+            data: dataTransferedIn,
+            organization_id: data.organization_id
+        });
+    }
+
+    if (dataTransferedOut) {
+        process.emit("setBandwidth", {
+            type: 'out',
+            data: dataTransferedOut,
+            organization_id: data.organization_id
+        });
+    }
+
+    if (!data.request)
+        data.request = data[type] || {}
+
+    data[type] = array
+
+    if (data.returnLog) {
+        if (!data.log)
+            data.log = []
+        data.log.push(...data[type])
+    }
+
+    return data
+}
+
+function getBytes(data) {
+    const jsonString = JSON.stringify(data);
+    return Buffer.byteLength(jsonString, 'utf8');
+}
+
 function errorHandler(data, error, database, array) {
     if (typeof error == 'object')
         error['storage'] = 'mongodb'
@@ -771,40 +693,4 @@ function errorHandler(data, error, database, array) {
         data.error = [error]
 }
 
-function replaceArray(data) {
-    let keys = Object.keys(data);
-    let objectData = {};
-
-    keys.forEach((k) => {
-        let nk = k
-        if (/\[([0-9]*)\]/g.test(k)) {
-            nk = nk.replace(/\[/g, '.');
-            if (nk.endsWith(']'))
-                nk = nk.slice(0, -1)
-            nk = nk.replace(/\]./g, '.');
-            nk = nk.replace(/\]/g, '.');
-        }
-        objectData[nk] = data[k];
-    });
-
-    return objectData;
-}
-
-
-module.exports = {
-    databaseStats,
-    createDatabase,
-    readDatabase,
-    updateDatabase,
-    deleteDatabase,
-
-    createCollection,
-    readCollection,
-    updateCollection,
-    deleteCollection,
-
-    createObject,
-    readObject,
-    updateObject,
-    deleteObject,
-}
+module.exports = { send }