multi-tasks 2.0.1 → 2.0.3

package/workers/TaskMgr.js

@@ -1,182 +1,213 @@
-const fs = require('fs');
-const pathutil = require('path');
-const makeDir = require('../utils/makedir');
-const randomsUtil = require('../utils/randoms');
-
-let taskCounter = 0;
-
-let taskFolder;
-let progressRootFolder;
-let resultsRootFolder;
-
-let taskFolderNew;
-let taskFolderRunning;
-let taskFolderFinished;;
-let taskFolderFailed;
-
-let taskFolderResult;
-let taskFolderResultFailed;
-
-let allSubTasks = {
-    new:[],
-    running:[],
-    finished:[],
-    finished_with_errors:[]
-};
-
-let main_config;
-
-const TaskMgr = {
-    load:(config)=>{
-        main_config = config;
-        return TaskMgr;
-    },
-    initFolders:()=>{
-        let {taskRootFolder, masterTaskId} = main_config;
-        taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
-        if(fs.existsSync(taskFolder)){
-            let files = fs.readdirSync(taskFolder);
-            if(files.length > 0){
-                console.log('[FATAL] taskRootFolder already exist and not empty, delete it and try again: ', taskFolder);
-                return process.exit(0);
-            }
-        };
-        //let taskfolder = pathutil.parse(__filename).dir;
-        //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
-        progressRootFolder = pathutil.resolve(taskFolder, './progress');
-        resultsRootFolder = pathutil.resolve(taskFolder, './results');
-
-        taskFolderNew = pathutil.resolve(progressRootFolder, './new');
-        taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
-        taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
-        taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
-
-        taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
-        taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
-        //let mainRptFolder = pathutil.resolve(taskfolder, './report');
-    
-        //makeDir.sync(mainRptFolder);
-        makeDir.sync(taskFolderNew);
-        makeDir.sync(taskFolderRunning);
-        makeDir.sync(taskFolderFinished);
-        makeDir.sync(taskFolderFailed);
-        makeDir.sync(taskFolderResult);
-        makeDir.sync(taskFolderResultFailed);
-
-        main_config = {
-            ...main_config,
-            taskFolder,
-            progressRootFolder,
-            resultsRootFolder,
-
-            taskFolderNew,
-            taskFolderRunning,
-            taskFolderFinished,
-            taskFolderFailed,
-            taskFolderResult,
-            taskFolderResultFailed,
-        };
-
-        //for restart a failed task
-        let main_config2 = JSON.parse(JSON.stringify(main_config));
-        delete main_config2.tasks;//will load from folder 'new', so no need the early tasks data
-        fs.writeFileSync(pathutil.resolve(taskFolder, 'task_config.json'), JSON.stringify(main_config2));
-        return main_config;
-    },
-    createNewTask:(task)=>{
-        const {masterTaskId, taskFolderNew} = main_config;
-        const subid = randomsUtil.getDateTimeTxt() + Math.random();
-
-        task.masterTaskId = masterTaskId;
-        task.subid = `${masterTaskId}-${subid}`;
-        let taskfilepath = pathutil.resolve(taskFolderNew, `./${task.subid}.task.json`);
-        fs.writeFileSync(taskfilepath, JSON.stringify(task));
-    },
-    loadNewTasks:()=>{
-        const {masterTaskId, taskFolderNew} = main_config;
-        return new Promise((resolve)=>{
-            console.log('get1')
-            makeDir.getDirectFiles(taskFolderNew, (taskFiles)=>{
-                taskFiles.forEach((filename)=>{
-                    let fpath = pathutil.resolve(taskFolderNew, filename);
-                    let fcontent;
-                        fcontent = fs.readFileSync(fpath, 'utf-8');
-
-                    if(fcontent){
-                        let task = JSON.parse(fcontent);
-                        allSubTasks.new.push(task);
-                    }
-                });
-                resolve()
-            });
-        });
-    },
-    updateNewTasks:()=>{
-        let p;
-        if(allSubTasks.new.length === 0){
-            p = TaskMgr.loadNewTasks();//load again
-        }else{
-            p = Promise.resolve();
-        }
-        return p;
-    },
-    popNewTask:()=>{
-        return new Promise((resolve)=>{
-            let newtask = allSubTasks.new.shift();
-            if(newtask){
-                TaskMgr.mvTask(newtask.subid, 'new', 'running', (newPath)=>{ 
-                    resolve(newtask);
-                });
-            }else{
-                resolve(null);
-            }
-        });
-    },
-    getTask:(type, id)=>{
-        let result;
-        let arr = allSubTasks[type];
-        for(let i=0;i<arr.length;i++){
-            let task = arr[i];
-            if(task.subid === id){
-                result = task;
-                break;
-            } 
-        }
-        return result;
-    },
-    getTasks:(type)=>{
-        return allSubTasks[type];
-    },
-    getTaskFiles:(type, callback)=>{
-        let rootdir = main_config.progressRootFolder;
-        let targetFolder = pathutil.resolve(rootdir, `./${type}`);
-        console.log('get2')
-        makeDir.getDirectFiles(targetFolder, (files)=>{
-            callback(files);
-        });
-    },
-    mvTask: (subid, from, to, callback)=> {
-        //console.log('moooooov=======', subid, from, to)
-        let {progressRootFolder} = main_config;
-        let oldPath = pathutil.resolve(progressRootFolder, `./${from}/${subid}.task.json`);
-        let newPath = pathutil.resolve(progressRootFolder, `./${to}/${subid}.task.json`);
-
-            fs.rename(oldPath, newPath, (err) => {
-                //if (err) throw err;
-                callback(newPath);
-            });
-    },
-    saveTaskResult:(subid, hasError, data)=>{        
-        let {taskFolderResult, taskFolderResultFailed} = main_config;
-        let folder = hasError ? taskFolderResultFailed : taskFolderResult;
-        let fpath = pathutil.resolve(folder, `${subid}.json`);
-        let fcontent;
-        if(typeof data === 'undefined') {
-            fcontent = 'undefined';
-        }else{
-            fcontent = JSON.stringify(data);
-        };
-        fs.writeFileSync(fpath, fcontent);
-    }
-};
+const fs = require('fs');
+const pathutil = require('path');
+const makeDir = require('../utils/makedir');
+const randomsUtil = require('../utils/randoms');
+
+let taskCounter = 0;
+
+let taskFolder;
+let progressRootFolder;
+let resultsRootFolder;
+
+let taskFolderNew;
+let taskFolderRunning;
+let taskFolderFinished;;
+let taskFolderFailed;
+
+let taskFolderResult;
+let taskFolderResultFailed;
+
+let allSubTasks = {
+    new:[],
+    running:[],
+    finished:[],
+    finished_with_errors:[]
+};
+
+let main_config;
+
+const TaskMgr = {
+    load:(config)=>{
+        main_config = config;
+        return TaskMgr;
+    },
+    initFolders:()=>{
+        let {taskRootFolder, masterTaskId} = main_config;
+        taskFolder = pathutil.resolve(taskRootFolder, masterTaskId);
+        if(fs.existsSync(taskFolder) && !(main_config.__resume || main_config.__restart)){
+            let files = fs.readdirSync(taskFolder);
+            if(files.length > 0){
+                console.log('[FATAL] taskRootFolder already exist and not empty, delete it and try again: ', taskFolder);
+                return process.exit(0);
+            }
+        };
+        //let taskfolder = pathutil.parse(__filename).dir;
+        //let taskFolder = pathutil.resolve(taskfolder, `./tasks/${masterTaskId}`);
+        progressRootFolder = pathutil.resolve(taskFolder, './progress');
+        resultsRootFolder = pathutil.resolve(taskFolder, './results');
+
+        taskFolderNew = pathutil.resolve(progressRootFolder, './new');
+        taskFolderRunning = pathutil.resolve(progressRootFolder, './running');
+        taskFolderFinished = pathutil.resolve(progressRootFolder, './finished');
+        taskFolderFailed= pathutil.resolve(progressRootFolder, './finished_with_errors');
+
+        taskFolderResult = pathutil.resolve(resultsRootFolder, './succ');
+        taskFolderResultFailed = pathutil.resolve(resultsRootFolder, './errors');
+        //let mainRptFolder = pathutil.resolve(taskfolder, './report');
+    
+        //makeDir.sync(mainRptFolder);
+        makeDir.sync(taskFolderNew);
+        makeDir.sync(taskFolderRunning);
+        makeDir.sync(taskFolderFinished);
+        makeDir.sync(taskFolderFailed);
+        makeDir.sync(taskFolderResult);
+        makeDir.sync(taskFolderResultFailed);
+
+        main_config = {
+            ...main_config,
+            taskFolder,
+            progressRootFolder,
+            resultsRootFolder,
+
+            taskFolderNew,
+            taskFolderRunning,
+            taskFolderFinished,
+            taskFolderFailed,
+            taskFolderResult,
+            taskFolderResultFailed,
+        };
+
+        //for restart a failed task
+        let main_config2 = JSON.parse(JSON.stringify(main_config));
+        delete main_config2.tasks;//will load from folder 'new', so no need the early tasks data
+        fs.writeFileSync(pathutil.resolve(taskFolder, 'task_config.json'), JSON.stringify(main_config2));
+        return main_config;
+    },
+    createNewTask:(task)=>{
+        const {masterTaskId, taskFolderNew} = main_config;
+        const subid = randomsUtil.get36Id();
+
+        task.masterTaskId = masterTaskId;
+        task.subid = `${masterTaskId}--${subid}`;
+        let taskfilepath = pathutil.resolve(taskFolderNew, `./${task.subid}.task.json`);
+        fs.writeFileSync(taskfilepath, JSON.stringify(task));
+    },
+    loadAllTasks:(loadtype)=>{
+        const {progressRootFolder} = main_config;
+        let load = (type)=>{
+            allSubTasks[type] = [];
+            return new Promise((resolve)=>{
+                    let folder = pathutil.resolve(progressRootFolder, type);
+                    makeDir.getDirectFiles(folder, (taskFiles)=>{
+                        taskFiles.forEach((filename)=>{
+                            let fpath = pathutil.resolve(folder, filename);
+                            let fcontent;
+                            try{
+                                fcontent = fs.readFileSync(fpath, 'utf-8');
+                            }catch(e){
+                                console.log('[WARN]', `file not found`, fpath)
+                            }
+                            if(fcontent){
+                                let task = JSON.parse(fcontent);
+                                allSubTasks[type].push(task);
+                            }
+                        });
+                        resolve(allSubTasks[type]);
+                    });                
+            });
+        };
+        let promises = [];
+        if(typeof loadtype === 'undefined'){
+            for(let type in allSubTasks){
+                promises.push(load(type));
+            }
+        }else{
+            promises.push(load(loadtype));
+        }
+        return Promise.all(promises);
+    },
+    loadNewTasks:()=>{
+        return TaskMgr.loadAllTasks('new');
+    },
+    updateNewTasks:()=>{
+        let p;
+        if(allSubTasks.new.length === 0){
+            p = TaskMgr.loadNewTasks();//load again
+        }else{
+            p = Promise.resolve();
+        }
+        return p;
+    },
+    popNewTask:()=>{
+        return new Promise((resolve)=>{
+            let newtask = allSubTasks.new.shift();
+            if(newtask){
+                TaskMgr.mvTask(newtask.subid, 'new', 'running', (newPath)=>{ 
+                    resolve(newtask);
+                });
+            }else{
+                resolve(null);
+            }
+        });
+    },
+    getTask:(type, id)=>{
+        let result;
+        let arr = allSubTasks[type];
+        for(let i=0;i<arr.length;i++){
+            let task = arr[i];
+            if(task.subid === id){
+                result = task;
+                break;
+            } 
+        }
+        return result;
+    },
+    getTasks:(type)=>{
+        return allSubTasks[type];
+    },
+    getTaskFiles:(type, callback)=>{
+        let rootdir = main_config.progressRootFolder;
+        let targetFolder = pathutil.resolve(rootdir, `./${type}`);
+        makeDir.getDirectFiles(targetFolder, (files)=>{
+            callback(files);
+        });
+    },
+    mvTaskPromise: (subid, from, to)=> {
+        return new Promise((resolve, reject)=>{
+            TaskMgr.mvTask(subid, from, to, (newPath, succ)=>{
+                succ ? resolve(newPath) : reject(newPath);
+            });
+        });
+    },
+    mvTask: (subid, from, to, callback)=> {
+        console.log('Move task:', subid, from, to, callback)
+        let {progressRootFolder} = main_config;
+        let oldPath = pathutil.resolve(progressRootFolder, `./${from}/${subid}.task.json`);
+        let newPath = pathutil.resolve(progressRootFolder, `./${to}/${subid}.task.json`);
+        try{
+            fs.rename(oldPath, newPath, (err) => {
+                if (err) {
+                    callback(newPath, false);
+                }else{
+                    callback(newPath, true);
+                }
+            });
+        }catch(e){
+            console.log(e);
+            callback(newPath, false);
+        }
+    },
+    saveTaskResult:(subid, hasError, data)=>{        
+        let {taskFolderResult, taskFolderResultFailed} = main_config;
+        let folder = hasError ? taskFolderResultFailed : taskFolderResult;
+        let fpath = pathutil.resolve(folder, `${subid}.json`);
+        let fcontent;
+        if(typeof data === 'undefined') {
+            fcontent = 'undefined';
+        }else{
+            fcontent = JSON.stringify(data);
+        };
+        fs.writeFileSync(fpath, fcontent);
+    }
+};
 module.exports = TaskMgr;

package/workers/WorkerMgr.js

@@ -1,31 +1,31 @@
-const fs = require('fs');
-const pathutil = require('path');
-const makeDir = require('../utils/makedir');
-
-const BUSY = 1;
-const IDLE = 0;
-
-let workers = {};
-let status = {};
-
-const mgr = {
-    addWorker:(worker)=>{
-        let id = `${worker.id}`;
-        if(workers[id] || status[id]){
-            throw 'FATAL: duplicated worker!';
-        }
-        status[id] = {isBusy: true};
-        console.log('add worker', worker.id);
-        workers[id] = worker;
-    },
-    getWorker:(id)=>{
-        return workers[`${id}`];
-    },
-    setBusy:(id, isBusy)=>{
-
-    },
-    isBusy:(id)=>{
-
-    }
-};
+const fs = require('fs');
+const pathutil = require('path');
+const makeDir = require('../utils/makedir');
+
+const BUSY = 1;
+const IDLE = 0;
+
+let workers = {};
+let status = {};
+
+const mgr = {
+    addWorker:(worker)=>{
+        let id = `${worker.id}`;
+        if(workers[id] || status[id]){
+            throw 'FATAL: duplicated worker!';
+        }
+        status[id] = {isBusy: true};
+        console.log('add worker', worker.id);
+        workers[id] = worker;
+    },
+    getWorker:(id)=>{
+        return workers[`${id}`];
+    },
+    setBusy:(id, isBusy)=>{
+
+    },
+    isBusy:(id)=>{
+
+    }
+};
 module.exports = mgr;

package/workers/asMaster.js

@@ -1,71 +1,123 @@
-var fs = require('fs');
-var pathutil = require('path');
-const numCPUs = require('os').cpus().length;
-
-let MSG_REQUEST_TASKID = 'request_taskid';
-
-const TaskMgr = require('./TaskMgr');
-const Multithread = require('./Multithread');
-
-
-const fmtDigit = (n)=>{
-    return n > 9 ? "" + n: "0" + n;
-}
-const getDateTimeTxt = ()=>{
-    let d=new Date();
-    let YYYY=d.getFullYear()
-    let MM=fmtDigit(d.getMonth()+1);
-    let DD=fmtDigit(d.getDate());
-    let HH=fmtDigit(d.getHours());
-    let mm=fmtDigit(d.getMinutes());
-    let ss=fmtDigit(d.getSeconds());
-    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
-};
-
-const init = (config)=>{
-    if(typeof config === 'undefined') config = {}; 
-    if(!config.tasks){ throw 'Please provide tasks data!';  };
-    if(!config.processTask){ throw 'Please provide processTask function!'; };
-    if(!config.tasks){ throw 'Must assign initial tasks!'; };
-
-    if(typeof config.numberOfWorkers === 'undefined') config.numberOfWorkers = numCPUs - 1;
-
-    let {taskRootFolder, taskName} = config;
-
-    if(taskName && !taskName.match(/[a-zA-Z0-9_-]/)){ throw 'Error: taskName can only contain letters and numbers'; }
-    if(!taskName){ taskName = 'multitasks'; }
-
-    let {round, pow, random} = Math;
-    let dateTimeText = getDateTimeTxt();//moment().format('YYYY-MM-DD_HH-mm-ss');
-    let randomNum = round((random() * pow(10,8)),8).toString(36);
-    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
-
-    console.log(`  masterTaskId===${masterTaskId}`);
-
-    config.taskRootFolder = taskRootFolder;
-    config.masterTaskId = masterTaskId;
-
-
-    TaskMgr.load(config);
-    config = TaskMgr.initFolders(config);
-
-    config.tasks.forEach((task)=>{
-        TaskMgr.createNewTask(task);
-    });
-
-    // let tasks = TaskMgr.getTasks('new')
-    // let task =  tasks.shift();
-    // console.log(tasks)
-    // TaskMgr.mvTask(task.subid, 'new', 'finished',()=>{})
-
-    //to reload is for to reStart a collapsed task
-    return new Promise((resolve)=>{
-        TaskMgr.loadNewTasks().then(()=>{
-            console.log('******start multi-thread')
-            resolve(config);
-        });
-    });
-};
-module.exports = {
-    init
+var fs = require('fs');
+var pathUtil = require('path');
+const numCPUs = require('os').cpus().length;
+
+let MSG_REQUEST_TASKID = 'request_taskid';
+
+const TaskMgr = require('./TaskMgr');
+const Multithread = require('./Multithread');
+
+
+const fmtDigit = (n)=>{
+    return n > 9 ? "" + n: "0" + n;
+}
+const getDateTimeTxt = ()=>{
+    let d=new Date();
+    let YYYY=d.getFullYear()
+    let MM=fmtDigit(d.getMonth()+1);
+    let DD=fmtDigit(d.getDate());
+    let HH=fmtDigit(d.getHours());
+    let mm=fmtDigit(d.getMinutes());
+    let ss=fmtDigit(d.getSeconds());
+    return `${YYYY}-${MM}-${DD}_${HH}-${mm}-${ss}`;
+};
+
+const resume = (config)=>{
+    let {taskFolder} = config;
+    if(!taskFolder){
+        console.log('[ERROR]', 'method resume need param "taskFolder"')
+        throw ('[ERROR]', 'method resume need param "taskFolder"');
+    }
+    let pathinfo = pathUtil.parse(taskFolder);
+    let taskId = pathinfo.name;
+    let taskRootFolder = pathinfo.dir;
+
+    //load from config file
+    let task_config = JSON.parse(fs.readFileSync(pathUtil.resolve(taskFolder, 'task_config.json')));
+    config = {
+        ...config,
+        ...task_config
+    }
+
+    // config.taskId = taskId;
+    // config.taskRootFolder = taskRootFolder;
+    // config.masterTaskId = taskId;
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    //console.log(config)
+
+    return new Promise((resolve)=>{
+
+        TaskMgr.getTaskFiles('running', (taskfiles)=>{
+            let promises = [];
+            taskfiles.forEach((fname)=>{
+                let subid = fname.replace(/\.task\.json$/, '');
+                promises.push(TaskMgr.mvTaskPromise(subid, 'running', 'new'));
+            });
+            Promise.all(promises).then(()=>{
+                TaskMgr.loadAllTasks().then((types)=>{
+                    console.log('******resume multi-thread')
+                    setTimeout(()=>{
+                        resolve(config);
+                    }, 1000);
+                });
+
+            }).catch((e)=>{
+                console.log('failed', e)
+            });
+        });
+    });
+
+
+};
+
+const init = (config)=>{
+    if(typeof config === 'undefined') config = {}; 
+    if(!config.tasks){ throw 'Please provide tasks data!';  };
+    if(!config.processTask){ throw 'Please provide processTask function!'; };
+    if(!config.tasks){ throw 'Must assign initial tasks!'; };
+
+    if(typeof config.numberOfWorkers === 'undefined') config.numberOfWorkers = numCPUs - 1;
+
+    let {taskRootFolder, taskName} = config;
+
+    if(taskName && !taskName.match(/[a-zA-Z0-9_-]/)){ throw 'Error: taskName can only contain letters and numbers'; }
+    if(!taskName){ taskName = 'multitasks'; }
+
+    let {round, pow, random} = Math;
+    let dateTimeText = getDateTimeTxt();//moment().format('YYYY-MM-DD_HH-mm-ss');
+    let randomNum = round((random() * pow(10,8)),8).toString(36);
+    let masterTaskId = config.taskId ? config.taskId : `${taskName}${dateTimeText}--${randomNum}`;
+
+    console.log(`  masterTaskId===${masterTaskId}`);
+
+    config.taskRootFolder = taskRootFolder;
+    config.masterTaskId = masterTaskId;
+
+
+    TaskMgr.load(config);
+    config = TaskMgr.initFolders(config);
+
+    config.tasks.forEach((task)=>{
+        TaskMgr.createNewTask(task);
+    });
+
+    // let tasks = TaskMgr.getTasks('new')
+    // let task =  tasks.shift();
+    // console.log(tasks)
+    // TaskMgr.mvTask(task.subid, 'new', 'finished',()=>{})
+
+    //to reload is for to reStart a collapsed task
+    return new Promise((resolve)=>{
+        TaskMgr.loadNewTasks().then(()=>{
+            console.log('******start multi-thread')
+            resolve(config);
+        });
+    });
+};
+module.exports = {
+    init,
+    resume
 };