file-obj-queue 2.0.5 → 3.0.0

package/.github/workflows/{github-actions-demo.yml → github-actions.yml}

@@ -15,4 +15,4 @@ jobs:
       - name: List files in the repository
         run: |
           ls ${{ github.workspace }}
-      - run: echo "🍏 This job's status is ${{ job.status }}."
+      - run: echo "🍏 This job's status is ${{ job.status }}."

package/.github/workflows/node.js.yml

@@ -79,3 +79,4 @@ jobs:
 
     - run: npm run build --if-present
     - run: npm run test_files
+    - run: npm run test_all

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 jman717
+Copyright (c) 2021 jman717
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -1,11 +1,30 @@
-[![npm Package](https://img.shields.io/npm/v/file-obj-queue.svg)](https://www.npmjs.org/package/file-obj-queue)
-[![License](https://img.shields.io/npm/l/file-obj-queue.svg)](https://github.com/jman717/file-obj-queue/blob/master/LICENSE)
-[![CodeQL](https://github.com/jman717/file-obj-queue/actions/workflows/github-actions-demo.yml/badge.svg)](https://github.com/jman717/file-obj-queue/actions/workflows/github-actions-demo.yml)
-[![Node.js CI](https://github.com/jman717/file-obj-queue/actions/workflows/node.js.yml/badge.svg)](https://github.com/jman717/file-obj-queue/actions/workflows/node.js.yml)
+[![npm Package](https://img.shields.io/npm/v/queueobj.svg)](https://www.npmjs.org/package/queueobj)
+[![License](https://img.shields.io/npm/l/queueobj.svg)](https://github.com/jman717/queueobj/blob/master/LICENSE)
+[![CodeQL](https://github.com/jman717/QueueObj/actions/workflows/github-actions.yml/badge.svg)](https://github.com/jman717/QueueObj/blob/main/.github/workflows/github-actions.yml)
+[![Node.js CI](https://github.com/jman717/QueueObj/actions/workflows/node.js.yml/badge.svg)](https://github.com/jman717/QueueObj/actions/workflows/node.js.yml)
 
-[![NPM](https://nodei.co/npm/file-obj-queue.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/file-obj-queue/)
+[![NPM](https://nodei.co/npm/queueobj.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/queueobj/)
 
-An array of objects containing file path information. One use is for dynamically import require modules.
+Queue javascript objects dynamically then process the queue according to the appender. All appenders are handled synchronously.
+
+Included tag appenders:
+
+* files - See if inputed files exists.
+* all - process all added objects.
+* top_one - process only the object in the 0(zero) position of the process array.
+* bottom_one - process only the object in the last position of the process array.
+* func_all - process custom function names in created objects. Custom processing names can be used in any appender, this is just one example.
+* status - queue and process all objects by status.
+* name - queue and process all objects by name.
+* version - queue and process all objects by version.
+
+* json_all - process a class object per json input array variables.
+* json_top_one - process a class object per the first json input array variable.
+* json_bottom_one - process a class object per the last json input array variable.
+* json_func_all - process custom function names in created objects. Custom processing names can be used in any json appender, this is just one example.
+* json_status - queue and process all objects by status.
+* json_name - queue and process all objects by name.
+* json_version - queue and process all objects by version.
 
 Mocha Test
 ---------
@@ -17,5 +36,111 @@ General Setup Test
 ---------
 ```
 npm run test_files
+npm run test_all
+npm run test_top_one
+npm run test_bottom_one
+npm run test_func_all
+npm run test_status_matching
+npm run test_status_non_matching
+npm run test_name_matching
+npm run test_name_non_matching
+npm run test_version_matching
+npm run test_version_non_matching
+npm run test_json_all
+npm run test_json_top_one
+npm run test_json_bottom_one
+npm run test_json_func_all
+npm run test_json_status_matching
+npm run test_json_status_non_matching
+npm run test_json_version_matching
+npm run test_json_version_non_matching
+npm run test_json_name_matching
+npm run test_json_name_non_matching
+
+```
+
+Usage
+---------
+```js
+
+var queue = require("queueobj");
+
+var tst1 = class test1 {
+  constructor(props) {
+    let t = this, fname = "test_all.test1.constructor"
+    t.log = props.log
+    t.id = props.id
+  }
+
+  process(callback) {
+    let t = this, fname = "test_all.test1.process"
+    t.log({ msg: `This object (${fname}) is id (${t.id}). Do stuff here`.bgBrightGreen, type: "info" })
+    callback({ success: { msg: `processing all test1` } })
+  }
+}
+
+var tst2 = class test2 {
+  constructor(props) {
+    let t = this, fname = "test_all.test2.constructor"
+    t.log = props.log
+    t.id = props.id
+  }
+
+  process(callback) {
+    let t = this, fname = "test_all.test2.process"
+    t.log({ msg: `This object (${fname}) is id (${t.id}). Do stuff here`.bgBrightGreen, type: "info" })
+    setTimeout(() => {
+      callback({ success: { msg: `processing all test2` } })
+    }, 4000)
+  }
+}
+
+var tst3 = class test3 {
+  constructor(props) {
+    let t = this, fname = "test_all.test3.constructor"
+    t.log = props.log
+    t.id = props.id
+  }
+
+  process(callback) {
+    let t = this, fname = "test_all.test3.process"
+    t.log({ msg: `This object (${fname}) is id (${t.id}). Do stuff here`.bgBrightGreen, type: "info" })
+    // callback({success: { msg: `processing all test3` }})
+    callback({ error: { msg: `there is some problem thrown here on test3` } })
+  }
+}
+
+var tst4 = class test4 {
+  constructor(props) {
+    let t = this, fname = "test_all.test4.constructor"
+    t.log = props.log
+    t.id = props.id
+
+  }
+
+  process(callback) {
+    let t = this, fname = "test_all.test4.process"
+    t.log({ msg: `This object (${fname}) is id (${t.id}). Do stuff here`.bgBrightGreen, type: "info" })
+    callback({ success: { msg: `processing all test4` } })
+  }
+}
+
+var qObj = new queue()
+
+qObj.init().process({
+  appender: "all",
+  exclude_logMsg: ["debug"],   /* example ["debug", "info"] */
+  process_objects: [tst1, tst2, tst3, tst4]
+}).then((success) => {
+  qObj.logMsg({ msg: `test success: {msg: "all objects processed with no errors"}`.success.italic.bold, type: "success" })
+}, (error) => {
+  if (typeof error == "string") {
+    qObj.logMsg({ msg: `error: ${error}`.error.italic.bold, type: "error" })
+  } else {
+    let add_s = (error.error_count > 1) ? 's' : ''
+    qObj.logMsg({ msg: `${error.error_count} error${add_s} detected`.error.italic.bold, type: "error" })
+  }
+})
+
+```
 
-```

package/app.js

@@ -3,70 +3,185 @@
 * Main processing app
 */
 
-let colors = require('colors'),
-    base_queue = require("./base_queue/app")
-
-colors.setTheme({
-    silly: 'rainbow',
-    input: 'grey',
-    verbose: 'cyan',
-    prompt: 'grey',
-    info: 'green',
-    data: 'grey',
-    help: 'cyan',
-    warn: 'yellow',
-    debug: 'blue',
-    error: 'red',
-    success: 'green'
-});
-
-exports = module.exports = class FilesQueue {
+let base_queue = require("base-queue"),
+    log_queue = require("log-queue"),
+    fs = require('fs'),
+    validPath = require('valid-path')
+
+const { memoryUsage } = require('process')
+
+var log_data = [
+    { props: { id: 100, name: "console_basic", absolute_path: __filename, check: true } }
+]
+
+
+var log_object = class log_obj {
+    constructor(props) {
+        let t = this
+        t.id = props.id
+        t.log = props.log
+        t.name = props.name
+        t.path = props.relative_path
+        t.absolute_path = props.absolute_path
+        t.status = 'init'
+        t.errors = false
+        t.error_msg = 'none'
+
+        t.process = t.process.bind(t)
+        t.do_checks = t.do_checks.bind(t)
+
+        if (props.check) {
+            t.do_checks()
+        }
+
+        return t
+    }
+
+    do_checks() {
+        let t = this, path_to_check,
+            last_item = t.absolute_path.split("\\").pop(),
+            check_file = t.absolute_path.split(last_item)[0], check_path = t.path.split('../')
+
+        check_file = check_file.replace(/\\/g, "/");
+        path_to_check = validPath(t.path);
+
+        if (!path_to_check.valid) {
+            t.errors = true
+            t.error_msg = `id = ${t.id} name(${t.name}) Error in ${path_to_check.data.input}: ${path_to_check.error})`
+        }
+
+        // check_path.map((dat, i) => {
+        //     if (/^[a-zA-Z._]+$/.test(dat)) {
+        //         if (dat != '.' || dat != '..')
+        //             check_file += dat + '/'
+        //     }
+        // })
+        // check_file = check_file.slice(0, -1)
+        check_file += check_path[1]
+        check_file += `${t.name}.js`
+        try {
+            if (!fs.existsSync(check_file)) {
+                t.errors = true
+                t.error_msg = `id = ${t.id} name(${t.name}) file (${check_file} does not exist)`
+            }
+        } catch (e) {
+            e.message = "file_obj do_checks error: " + e.message
+            throw (e)
+        }
+    }
+
+    process(callback) {
+        let t = this
+        if (t.errors)
+            callback({ error: {msg: t.error_msg} })
+        else {
+            callback({ success: {msg: `id = ${t.id} name(${t.name}) success`} })
+        }
+    }
+}
+
+exports = module.exports = class QueueObj {
     constructor() {
+        var t = this, fname = 'QueueObj.constructor'
         try {
-            var t = this
-            t.app_resolve = null
-            t.app_reject = null
+            t.promise = null
+            t.resolve = null
+            t.reject = null
+            t.promise_2q = null
+            t.resolve_2q = null
+            t.reject_2q = null
             t.successMsg = ''
+            t.log_queue = null
 
             t.logMsg = t.logMsg.bind(t)
+            t.init = t.init.bind(t)
             t.process = t.process.bind(t)
             t.getFileObject = t.getFileObject.bind(t)
 
-            t.logMsg(`FilesQueue.constructor`.debug)
+            // t.logMsg(`QueueObj.constructor`.debug)
 
             return t
         } catch (e) {
-            e.message = `FilesQueue app.js constructor error: ${e.message}`.error
-            throw (e)
+            e.message = `QueueObj app.js constructor error: ${e.message}`.error
+            console.log(e.message)
         }
     }
 
-    process(props = {}) {
-        let t = this
+    init() {
+        var t = this, fname = `QueueObj.init`
+        try {
+            t.promise = new Promise((resolve, reject) => {
+                t.resolve = resolve
+                t.reject = reject
+            })
+            t.promise_2q= new Promise((resolve, reject) => {
+                t.resolve_2q = resolve
+                t.reject_2q = reject
+            })
+            return t
+        } catch (e) {
+            t.logMsg({ msg: `${fname} error: (${e.message})`.error, type: 'error' })
+            // e.message = `QueueObj app.js constructor error: ${e.message}`.error
+            // console.log(e.message)
+        }
+    }
 
-        return new Promise((resolve, reject) => {
-            t.app_resolve = resolve
-            t.app_reject = reject
+    process(props = {}) {
+        let t = this, fname = `app.process`
 
-            if (typeof props.data_to_process_array == 'undefined')
-                t.app_reject('base_queue no props.data_to_process_array')
+        // if (typeof props.data_to_process_array == 'undefined')
+        //     t.reject('base_queue no props.data_to_process_array')
 
-            if (typeof props.appender == 'undefined')
-                t.app_reject('base_queue no props.appender')
+        if (typeof props.appender == 'undefined')
+            t.reject('base_queue no props.appender')
 
-            if (typeof props.process_objects == 'undefined')
-                t.app_reject(`props.process_objects not defined`)
+        if (typeof props.process_objects == 'undefined')
+            t.reject(`props.process_objects not defined`)
 
-            props.parent = t
-            t.base_queue = new base_queue().load(props).process()
-        })
+        try {
+            t.log_queue = new log_queue({
+                parent: t,
+                relative_path: "../base_queue/appenders/log/",
+                exclude_logMsg: props.exclude_logMsg,
+                resolve: t.resolve,
+                reject: t.reject
+            }).init({
+                appender: "console_basic",
+                process_objects: [log_object],
+                data_to_process_array: log_data
+            })
+            t.base_queue = new base_queue({   
+                parent: t,
+                relative_path: "./appenders/",
+                logMsg: t.logMsg,
+                resolve: t.resolve_2q,
+                reject: t.reject_2q
+            }).load(props).process()
+            return t.promise_2q
+        } catch (e) {
+            console.log(`${fname} error: (${e.message})`)
+        }
     }
 
     getFileObject() {
         return this.qJson.get_class_obj_array()
     }
 
-    logMsg(msg, props = {}) {
-        console.log(msg)
+    logMsg(props = { msg: '', type: '' }) {   
+        let t = this, fname = "QueueObj.logMsg"
+        try {
+            if (typeof props.msg == "undefined")
+                throw new Error(`no msg property in (${JSON.stringify(props)}) `)
+            if (typeof props.type == "undefined")
+                throw new Error(`no type property in (${JSON.stringify(props)}) `)
+            if (t.log_queue != null && typeof t.log_queue.logMsg != 'undefined') {
+                t.log_queue.logMsg(props)
+            } else {
+                throw new Error(`t.log_queue does not exist`)
+            }
+        } catch (e) {
+            e.message = `${fname} error: ${e.message})`
+            t.reject(e.message)
+        }
     }
 }

package/lib/appenders/all.js

@@ -0,0 +1,17 @@
+/*
+* @author Jim Manton: jrman@risebroadband.net
+* @since 2023-2-5l
+* lib/appenders/all.js
+*/
+
+var base = require('./base.js')
+
+exports = module.exports = class all extends base{
+	constructor(props) {
+		super(props)
+        var t = this
+		t.aname = 'all'
+		t.pro_types = ['items']
+		return t
+    }
+}

package/lib/appenders/base.js

@@ -0,0 +1,189 @@
+/*
+* @author Jim Manton: jrman@risebroadband.net
+* @since 2023-02-06
+* lib/appenders/base.js
+*/
+
+var colors = require('colors')
+
+class process_all_obj {
+    constructor(props) {
+        let t = this
+        t.parent = props.parent
+        t.objs = t.parent.getParent().getObjs()
+        t.status = 'process'
+        t.process_objs_item = 0
+        t.process_props_item = 0
+        t.await_item = 0
+        t.any_errors = false
+
+        t.continueProcessing = t.continueProcessing.bind(t)
+    }
+
+    process () {
+        let t = this
+        try {
+            if (t.process_objs_item >= t.objs.length) {
+                if (t.any_errors)
+                    t.setStatus('finish with errors')
+                else
+                    t.setStatus('finish')
+                return
+            }
+            try {
+                let obj, pro, itm
+                try {
+                    obj = t.parent.getParent().getItemToProcess(t.process_objs_item)
+                } catch (e) {
+                    e.message = `base error: (${e.message})`
+                    throw e
+                }
+                try {
+                    pro = (typeof obj == 'function') ? obj : obj.process;
+                } catch (e) {
+                    t.parent.getParent()(`pro error: (${e.message})`.red)
+                    throw e
+                }
+                if (typeof t.parent.pro_props != 'undefined' &&
+                    typeof t.parent.pro_props.property != 'undefined' &&
+                    typeof t.parent.pro_props.items != 'undefined' &&
+                    typeof obj._getProperty == 'function') {
+                    let skip = true
+                    for (let q = 0; q < t.parent.pro_props.items.length; q++) {
+                        itm = t.parent.pro_props.items[q]
+                        if (itm == obj._getProperty(obj)) {
+                            skip = false
+                            break
+                        }
+                    }
+                    if (skip) {
+                        t.continueProcessing()
+                        return
+                    }
+                }
+                t.setStatus('wait')
+                pro((obj_props) => {
+                    try {
+                        if (typeof obj_props != 'undefined' && typeof obj_props.error != 'undefined') {
+                            t.any_errors = true
+                            t.parent.getParent().logMsg(obj_props)
+                        } else {
+                            t.parent.getParent().logMsg(obj_props)
+                        }
+                        t.parent.results_array.push(obj_props)
+                        t.continueProcessing()
+                    } catch (e) {
+                        t.parent.getParent().logMsg({msg: `pro obj error: (${e.message})`.error, type: "error"})
+                        throw e
+                    }
+                })
+            } catch (e) {
+                e.message = `error: ${e.message} `
+                throw e
+            }
+        } catch (e) {
+            e.message = `process_all_obj error: ${e.message} `
+            throw e
+        }
+    }
+
+    continueProcessing () {
+        let t = this
+        t.process_objs_item++
+        t.setStatus('process')
+        t.parent.process_all()
+    }
+
+    getStatus () {
+        return this.status
+    }
+
+    setStatus (v) {
+        this.status = v
+    }
+}
+
+exports = module.exports = class base {
+    constructor(props) {
+        let t = this
+        t.await_array = []
+        t.resolve_array = []
+        t.reject_array = []
+        t.results_array = []
+        t.getParent = props.getParent
+        t.dt_start = null
+        t.dt_end = null
+        t.process_all_obj = null
+        t.pro_props = []
+
+        t.process = t.process.bind(this)
+        t.process_all = t.process_all.bind(this)
+    }
+
+    // await(props) {
+    //     let t = this
+    //     t.await_array.push(props)
+    //     return new Promise((resolve, reject) => {
+    //         t.resolve_array.push(resolve)
+    //         t.reject_array.push(reject)
+    //     });
+    // }
+
+    process(props) {
+        let t = this
+        t.dt_start = new Date(); // start measuring time
+
+        t.pro_props = props
+
+        return new Promise((resolve, reject) => {
+            t.resolve_array.push(resolve)
+            t.reject_array.push(reject)
+            t.process_all()
+        });
+    }
+
+    getStats() {
+        let t = this
+        t.dt_end = new Date();
+        let ret_str = '', st = t.dt_start, ed = t.dt_end, ml = t.dt_end - t.dt_start
+        // ret_str += JSON.stringify(t.results_array)
+        if (t.getParent().getStats()) {
+            ret_str += `start (${st}) end(${ed}) milliseconds(${ml})`
+        }
+        return ret_str   
+    }
+
+    process_all () {
+        let t = this, _continue
+        if (t.process_all_obj == null) {
+            t.process_all_obj = new process_all_obj({ parent: t })
+        }
+
+        _continue = false
+        try {
+            switch (t.process_all_obj.getStatus()) {
+                case 'process':
+                    t.process_all_obj.process()
+                    _continue = true
+                    break
+                case 'finish with errors':
+                    t.reject_array[t.reject_array.length - 1](t.getStats())
+                    break
+                case 'finish':
+                    t.resolve_array[t.resolve_array.length - 1](t.getStats())
+                    break
+                case 'wait':
+                    return
+                default:
+                    throw new Error(`status(${t.process_all_obj.getStatus()}) does not exist`)
+            }
+
+            if (_continue)
+                t.process_all()
+        } catch (e) {
+            e.message = `process_all error: ${e.message} `
+            throw e
+        }
+    }
+}
+

package/lib/appenders/bottom_one.js

@@ -0,0 +1,17 @@
+/*
+* @author Jim Manton: jrman@risebroadband.net
+* @since 2023-2-5
+* lib/appenders/bottom_one.js
+*/
+
+var base = require('./base.js')
+
+exports = module.exports = class bottom_one extends base{
+	constructor(props) {
+		super(props)
+        var t = this
+		t.aname = 'bottom_one'
+		t.pro_types = ['items']
+		return t
+    }
+}

package/lib/appenders/func_all.js

@@ -0,0 +1,17 @@
+/*
+* @author Jim Manton: jrman@risebroadband.net
+* @since 2023-2-5
+* lib/appenders/func_all.js
+*/
+
+var base = require('./base.js')
+
+exports = module.exports = class func_all extends base {
+    constructor(props) {
+        super(props)
+        var t = this
+        t.aname = 'func_all'
+		t.pro_types = ['items']
+        return t
+    }
+}

package/lib/appenders/name.js

@@ -0,0 +1,17 @@
+/*
+* @author Jim Manton: jrman@risebroadband.net
+* @since 2023-01-16
+* lib/appenders/name.js
+*/
+
+var base = require('./base.js')
+
+exports = module.exports = class name extends base {
+    constructor(props) {
+        super(props)
+        var t = this
+        t.aname = 'name'
+		t.pro_types = ['name']
+        return t
+    }
+}

package/lib/appenders/status.js

@@ -0,0 +1,17 @@
+/*
+* @author Jim Manton: jrman@risebroadband.net
+* @since 2023-01-16
+* lib/appenders/status.js
+*/
+
+var base = require('./base.js')
+
+exports = module.exports = class status extends base {
+    constructor(props) {
+        super(props)
+        var t = this
+        t.aname = 'status'
+		t.pro_types = ['status']
+        return t
+    }
+}