multi-tasks 1.0.2 → 1.0.4

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Lei Zhang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,2 @@
+# multi-tasks
+multi-tasks

package/example/RunnerConsumer.js

@@ -1,4 +1,4 @@
-let SuperConsumer = require('multi-task').SuperConsumer;
+let SuperConsumer = require('../index').SuperConsumer;
 
 
 let _do = (mainTaskInfo, actions, callback)=>{

package/example/RunnerProducer.js

@@ -1,4 +1,4 @@
-let SuperProducer = require('multi-task').SuperProducer;
+let SuperProducer = require('../index').SuperProducer;
 class TestProducer extends SuperProducer {
   
     // constructor

package/example/run.js

@@ -1,5 +1,4 @@
 let fs = require('fs');
-let _ = require('lodash');
 let pathutil = require('path');
 let RunnerProducer = require('./RunnerProducer')
 let RunnerConsumer = require('./RunnerConsumer')
@@ -9,7 +8,7 @@ let multi_task_parent_folder = pathutil.resolve(__dirname, `../../../tasks-searc
 
 let producer = new RunnerProducer(`workers_of_${taskId}`, multi_task_parent_folder)
 let consumer = new RunnerConsumer();
-let master = require('multi-task').master;
+let master = require('../index').master;
 
 console.log(producer.getTaskName())
 //return;

package/example2-dynamicIncreasing/RunnerConsumer.js

@@ -1,6 +1,14 @@
 let fs = require('fs');
+const jsonFormat = require('json-format');
 let pathutil = require('path');
+const { spawn, exec } = require('child_process');
 let SuperConsumer = require('../index').SuperConsumer;
+let localUtil = require('./util');
+const makeDir = require('make-dir');
+
+let thisroot = pathutil.parse(__filename).dir;
+let garbagedir = pathutil.resolve(thisroot,`./tmp-garbage`);
+makeDir.sync(garbagedir);
 
 let loadTaskFile = (dir)=>{
     var results = []
@@ -12,40 +20,138 @@ let loadTaskFile = (dir)=>{
         }catch(e){
             //Must try-catch, the file maybe deleted by other process
         }
-        if (stat && !stat.isDirectory()) results.push({file, fpath: pathutil.resolve(dir, file)});
+        if (stat && !stat.isDirectory()) results.push({
+            file, 
+            fpath: pathutil.resolve(dir, file)
+        });
     })
     //console.log(results);
     return results;
 };
-let mvTask = (oldPath, newPath, oldfname, callback)=>{
-    if(typeof callback === 'undefined') callback = ()=>{};
-    //console.log('>>>', oldPath, newPath, oldfname)
-    let oldfPath = pathutil.resolve(oldPath, oldfname);
-    let newfPath = pathutil.resolve(newPath, oldfname);
-    //if(fs.existsSync(newfPath)) fs.rmSync(newfPath, {force: true});
-    fs.rename(oldfPath, newfPath, (err) => {
-        if (err) throw err;
-        callback(newfPath);
+let processTask = (moduleName, requestVersion, callback)=>{
+    let outputs = [];
+    let ls = spawn(`ls`, [], {cwd: garbagedir, shell: true});
+    ls.stdout.on('data', (data) => {
+        let stdout = data.toString();
+        outputs.push(stdout);
+        //console.log('[stdout]', stdout)
+    });    
+    ls.stderr.on('data', (data) => {
+        let stdout = data.toString();
+        //console.log('[stderr]', stdout)
+        outputs.push(stdout);
+    });    
+    ls.on('close', (code) => {
+        //console.log('[close]', code)
+        // if(code === 1) {
+        //     console.log('[error], checkout failed!!')
+        // }
+        let succ = true;
+        let logTxt = outputs.join('');
+        
+        if(logTxt.indexOf('ERR!')>=0 && logTxt.match(/ERR\!/g)) {
+            succ = false;
+        }
+        let logJson;
+        try{
+            logJson = JSON.parse(logTxt);
+        }catch(e){
+            console.log(e)
+            console.log(`at===`, moduleName, requestVersion)            
+            console.log(`logTxt===`, logTxt)
+            succ = false;
+        }
+        if(succ && logJson){
+            if(!Array.isArray(logJson)){
+                logJson = [logJson];
+            };
+            let data = {
+                response: logJson,
+                __info__: {
+                    moduleName,
+                    requestVersion,
+                }
+            };
+            callback(true, data);
+        }else{
+            callback(false, logTxt);
+        }
     });
-};
+
+}
 
 let _do = (mainTaskInfo, actions, callback)=>{
     let{dynamicTasksFolderNew, dynamicTasksFolderSucc, dynamicTasksFolderFail} = mainTaskInfo.defination;
+    let tasks = loadTaskFile(dynamicTasksFolderNew);  
+    if(tasks.length === 0) return callback({data:'hahaha'+Math.random()});//stop!
 
-    let tasks = loadTaskFile(dynamicTasksFolderNew);
-    
-    if(tasks.length === 0) return callback({data:'hahaha'+Math.random()});
-    let task = tasks[0];
-    let fpath = task.fpath;
+    tasks = localUtil.shuffleAnArray(tasks);//avoid file IO conflicts
 
-    mvTask(dynamicTasksFolderNew, dynamicTasksFolderSucc, task.file);
-   
-    { 
-        //processing...
-        console.log('processing...', fpath);
+    let task;
+    let data;
+    while(!data){
+        task = tasks.shift();
+        try{
+            data = JSON.parse(fs.readFileSync(task.fpath, 'utf8'));  
+        }catch(e){
+            console.log('xxxx')
+        }
     }
 
-    _do(mainTaskInfo, actions, callback);
+    if(task && data){
+        let fpath = task.fpath;
+        { 
+            //processing...
+            let {moduleName, requestVersion, fullnameMd5} = data.info;            
+            let oldfPath = fpath;
+            let newfPath = pathutil.resolve(dynamicTasksFolderNew, `${fullnameMd5}.json`);
+            let succfPath = pathutil.resolve(dynamicTasksFolderSucc, `${fullnameMd5}.json`);
+            let errfPath = pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`);
+            if(fs.existsSync(succfPath) || fs.existsSync(errfPath)) {
+                console.log('exist...', process.pid, fullnameMd5);
+                if(fs.existsSync(newfPath))
+                try{
+                    fs.unlinkSync(newfPath);
+                }catch(e){
+                    console.log('remove failed', process.pid, newfPath, e);
+                }
+                _do(mainTaskInfo, actions, callback);
+            }else {
+                console.log('processing...', process.pid, fpath);
+                processTask(moduleName, requestVersion, (succ, logJson)=>{
+                    if(succ){      
+                        fs.writeFileSync(succfPath, jsonFormat(logJson));  
+                        logJson.response.forEach((item)=>{                            
+                            if(item.dependencies){
+                                for(let module2 in item.dependencies){
+                                    let version2 = item.dependencies[module2];
+                                    console.log('found new task', module2, version2);
+                                    localUtil.createNewTask(module2, version2, dynamicTasksFolderNew);//find new one, create a new task file
+                                }
+                            }
+                        });
+                    }else{
+                        console.log('ERR! failed', moduleName, requestVersion);
+                        fs.writeFileSync(pathutil.resolve(dynamicTasksFolderFail, `${fullnameMd5}.json`), jsonFormat({
+                            moduleName, requestVersion,
+                            msg: logJson
+                        }));
+                    }
+                    _do(mainTaskInfo, actions, callback);
+                });
+            }          
+            if(fs.existsSync(oldfPath)){
+                try{
+                    fs.unlinkSync(oldfPath);
+                }catch(e){
+                    console.log('remove failed', process.pid, oldfPath, e);
+                }
+            }            
+        }
+    }else{        
+        console.log('__done2', process.pid)
+        return callback({data:'hahaha'+Math.random()});//stop!
+    }
 };
 
 

package/example2-dynamicIncreasing/RunnerProducer.js

@@ -1,4 +1,24 @@
 let SuperProducer = require('../index').SuperProducer;
+const fs = require('fs');
+let pathutil = require('path');
+const jsonFormat = require('json-format');
+
+let thisroot = pathutil.parse(__filename).dir;
+
+let localUtil = require('./util')
+
+
+let createInitialTasks = (info)=>{    
+    let succList = JSON.parse(fs.readFileSync(pathutil.resolve(thisroot,'../tmp/version_succList.json'), 'utf8'));
+    //succList = succList.slice(0, 20);
+    succList.forEach((item)=>{
+        let {moduleName, requestVersion} = item;
+        localUtil.createNewTask(moduleName, requestVersion, info.dynamicTasksFolderNew);
+    });
+};
+
+
+
 class TestProducer extends SuperProducer {
   
     // constructor
@@ -10,7 +30,7 @@ class TestProducer extends SuperProducer {
         this.trunkInfo = trunkInfo;
     }
     getAllActions() {
-
+        createInitialTasks(this.trunkInfo);
         return new Promise((resolve, reject)=>{
             let trunkInfo = this.trunkInfo;
             let {NUMBER_OF_WORKS} = trunkInfo;
@@ -18,7 +38,7 @@ class TestProducer extends SuperProducer {
            //throw repos.length+'ff'+repos2.length
             //here, IGNORED_repoRegexStringList
             let alltasks = []
-            for(let i=0;i<NUMBER_OF_WORKS;i++){
+            for(let i=0;i<NUMBER_OF_WORKS;i++){//the task was driven by files, not here
                 alltasks.push({
                     num:i
                 })

package/example2-dynamicIncreasing/run.js

@@ -6,7 +6,7 @@ let RunnerConsumer = require('./RunnerConsumer')
 
 let thisroot = pathutil.parse(__filename).dir;
 let taskId = `workers_of_${Math.random()}`;
-let multi_task_parent_folder = pathutil.resolve(thisroot, `../../../multi-tasks-example2-dynamicIncreasing`);
+let multi_task_parent_folder = pathutil.resolve(thisroot, `../../example2_dynamicly_increasing`);
 
 let producer = new RunnerProducer(taskId, multi_task_parent_folder)
 let consumer = new RunnerConsumer();
@@ -15,7 +15,7 @@ let master = require('../index').master;
 console.log(producer.getTaskName())
 //return;
 
-let NUMBER_OF_WORKS = 6;//LOCK NUMBER OF WORKERS
+let NUMBER_OF_WORKS = 5;//LOCK NUMBER OF WORKERS
 
 let taskConfig = {
     NUMBER_OF_WORKS,
@@ -23,10 +23,9 @@ let taskConfig = {
     dynamicTasksFolderSucc: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-SUCC`),
     dynamicTasksFolderFail: pathutil.resolve(multi_task_parent_folder, `./dynamicTasks-FAIL`),
 }
-
-if(!fs.existsSync(taskConfig.dynamicTasksFolderNew)) throw "Must create folder, DO NOT create by code!";
-if(!fs.existsSync(taskConfig.dynamicTasksFolderSucc)) throw "Must create folder, DO NOT create by code!";
-if(!fs.existsSync(taskConfig.dynamicTasksFolderFail)) throw "Must create folder, DO NOT create by code!";
+makeDir.sync(taskConfig.dynamicTasksFolderNew)
+makeDir.sync(taskConfig.dynamicTasksFolderSucc)
+makeDir.sync(taskConfig.dynamicTasksFolderFail)
 
 
 producer.setTaskTrunkInfo(taskConfig);

package/example2-dynamicIncreasing/util.js

@@ -0,0 +1,47 @@
+const md5 = require('md5');
+const fs = require('fs');
+let pathutil = require('path');
+const jsonFormat = require('json-format');
+
+let getFileName = (moduleName, requestVersion)=>{
+    let fullname = `${moduleName}@${requestVersion}`;
+    let fullnameMd5 = md5(fullname);
+    return fullnameMd5;
+};
+let createNewTask = (moduleName, requestVersion, dynamicTasksFolderNew)=>{
+    let fullnameMd5 = getFileName(moduleName, requestVersion);
+    fs.writeFileSync(pathutil.resolve(dynamicTasksFolderNew, `./${fullnameMd5}.json`), jsonFormat({
+        info:{
+            moduleName,
+            requestVersion,
+            fullnameMd5,
+        }
+    }));
+};
+function shuffleAnArray(array) {
+    if(array.length === 0 || array.length === 1) return array;
+    let currentIndex = array.length,  randomIndex;
+  
+    // While there remain elements to shuffle.
+    while (currentIndex > 0) {
+  
+      // Pick a remaining element.
+      randomIndex = Math.floor(Math.random() * currentIndex);
+      currentIndex--;
+
+      // And swap it with the current element.
+      let tmp = array[currentIndex];
+      array[currentIndex] = array[randomIndex];
+      array[randomIndex] = tmp;
+  
+    //   [array[currentIndex], array[randomIndex]] = [
+    //     array[randomIndex], array[currentIndex]];
+    }
+  
+    return array;
+  }
+module.exports = {
+    createNewTask,
+    getFileName,
+    shuffleAnArray
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "",
   "main": "index.js",
   "directories": {

package/workers/master.js

@@ -62,12 +62,19 @@ let run = (currentProducer, currentConsumer, opt)=>{
               if(message === MSG_REQUEST_TASKID){
                 if(taskStack.length > 0){     
                   let subTaskInfo = taskStack.shift();
-                  worker.send({
-                    startTimestamp: new Date()*1,
-                    mainTaskInfo,
-                    subTaskInfo,
-                    totalTaskNum
-                  })
+                  let _sendMsg = ()=>{
+                    worker.send({
+                      startTimestamp: new Date()*1,
+                      mainTaskInfo,
+                      subTaskInfo,
+                      totalTaskNum
+                    },()=>{
+                      //handler
+                    }, {
+                      keepOpen: (typeof opt.keepOpen === 'undefined') ? true : opt.keepOpen
+                    })
+                  };
+                  _sendMsg();
                 }else{
                   removeWorker(worker.id)
                   worker.kill();