multi-tasks 2.0.0 → 2.0.1

package/README.md

@@ -113,6 +113,7 @@ node examples/example1/run
 
 Changelog:
 
+- 2.0.1 Avoid possible I/O conflicts.
 - 2.0.0 Rewritten with a new architecture to support dynamic tasks.
 - 1.2.8 Fix: create task folder failed on MacOS
 - 1.2.7 Small updates

package/examples/example0/run.js

@@ -46,7 +46,7 @@ multiTasks({
     taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
     taskId: 'my-task',
     numberOfWorkers: 3, //Assign how many workers are working in parallel
-    //autoCloseAfterCompletion: true,
+    autoCloseAfterCompletion: true,
     onFinish: (report)=>{
         console.log('finish callback', report);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "",
   "main": "index.js",
   "directories": {

package/utils/makedir.js

@@ -39,14 +39,10 @@ function getDirectFiles(dir, callback) {
         files.forEach(file => {  
             const filePath = path.join(dir, file);  
             //console.log(filePath)
-            try{
                 let stats = fs.statSync(filePath);
                 if (stats.isFile()) {  
                     childrenfiles.push(file); 
                 }  
-            }catch(e){
-                //
-            }
         });
         callback(childrenfiles);
     });  

package/workers/Multithread.js

@@ -53,9 +53,11 @@ const start = (config)=>{
                 WorkerMgr.addWorker(worker);
                 worker.on('message', function(message) {
                     if(message === MSG_REQUEST_NEW_TASK){
-                        queue.enqueue(TaskMgr.updateNewTasks);//use queue to make sure no I/O conflict
+                        queue.enqueue(async ()=>{
+                            return TaskMgr.updateNewTasks();
+                        });//use queue to make sure no I/O conflict
                         queue.enqueue(async () => {
-                            TaskMgr.popNewTask().then((subTask)=>{
+                            return TaskMgr.popNewTask().then((subTask)=>{
                                 if(!subTask){
                                     //no tasks left
                                     console.log('[Master]: no tasks left!');
@@ -99,7 +101,7 @@ const start = (config)=>{
                 setTimeout(()=>{
                     console.log('[Master]: all workers are done, process exit.')
                     process.exit(0);
-                }, 1000);
+                }, 10);
             }
         }); 
     }else{

package/workers/TaskMgr.js

@@ -95,15 +95,13 @@ const TaskMgr = {
     loadNewTasks:()=>{
         const {masterTaskId, taskFolderNew} = main_config;
         return new Promise((resolve)=>{
+            console.log('get1')
             makeDir.getDirectFiles(taskFolderNew, (taskFiles)=>{
                 taskFiles.forEach((filename)=>{
                     let fpath = pathutil.resolve(taskFolderNew, filename);
                     let fcontent;
-                    try{
                         fcontent = fs.readFileSync(fpath, 'utf-8');
-                    }catch(e){
-                        console.log('[WARN]', `file not found`, fpath)
-                    }
+
                     if(fcontent){
                         let task = JSON.parse(fcontent);
                         allSubTasks.new.push(task);
@@ -152,6 +150,7 @@ const TaskMgr = {
     getTaskFiles:(type, callback)=>{
         let rootdir = main_config.progressRootFolder;
         let targetFolder = pathutil.resolve(rootdir, `./${type}`);
+        console.log('get2')
         makeDir.getDirectFiles(targetFolder, (files)=>{
             callback(files);
         });
@@ -161,14 +160,11 @@ const TaskMgr = {
         let {progressRootFolder} = main_config;
         let oldPath = pathutil.resolve(progressRootFolder, `./${from}/${subid}.task.json`);
         let newPath = pathutil.resolve(progressRootFolder, `./${to}/${subid}.task.json`);
-        try{
+
             fs.rename(oldPath, newPath, (err) => {
                 //if (err) throw err;
                 callback(newPath);
             });
-        }catch(e){
-            callback(newPath);
-        }
     },
     saveTaskResult:(subid, hasError, data)=>{        
         let {taskFolderResult, taskFolderResultFailed} = main_config;