multi-tasks 1.2.4 → 1.2.6

package/README.md

@@ -43,7 +43,10 @@ multiTasks({
     processTask, 
     multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
     quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
-    numberOfWorks: 6 //Assign how many workers are working in parallel, default is the number of cpu cores minus one
+    numberOfWorkers: 6, //Assign how many workers are working in parallel, default is the number of cpu cores minus one
+    onFinish: (report)=>{
+        console.log(report);//the execution report can be viewed here
+    }
 });
 
 ```
@@ -58,7 +61,7 @@ let RunnerConsumer;//See examples, please provide a Consumer class to process a
 master.start(RunnerProducer, RunnerConsumer, {
     multi_task_parent_folder: `C:/tmp-demo`,
     quotaOfEachTask:2, //The number of tasks assigned to each worker
-    numberOfWorks: 6
+    numberOfWorkers: 6
 });
 ```
 
@@ -71,7 +74,9 @@ node examples/example1/run
 
 Changelog:
 
-- 1.2.4 Fix: opt.numberOfWorks not work
+- 1.2.6 Support onFinish event
+- 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
+- 1.2.4 Fix: opt.numberOfWorkers not work
 - 1.2.3 Update README
 - 1.2.2 Update README and examples
 - 1.2.1 Handle exceptions and errors in subtasks

package/examples/example0/run.js

@@ -31,5 +31,8 @@ multiTasks({
     processTask, 
     multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
     quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
-    numberOfWorks: 6 //Assign how many workers are working in parallel
+    numberOfWorkers: 6, //Assign how many workers are working in parallel
+    onFinish: (report)=>{
+        console.log(report);
+    }
 });

package/examples/example1/run.js

@@ -5,5 +5,5 @@ let master = require('../../index').master;
 master.start(RunnerProducer, RunnerConsumer, {
     multi_task_parent_folder: `C:/tmp-demo`,
     quotaOfEachTask:2, //The number of tasks assigned to each worker
-    numberOfWorks: 6
+    numberOfWorkers: 6
 });

package/examples/example2/run.js

@@ -19,5 +19,5 @@ producer.setTaskTrunkInfo(taskConfig);
 consumer.setTaskTrunkInfo(taskConfig)
 master.run(producer, consumer, {
     quotaOfEachTask:2,
-    numberOfWorks: 6
+    numberOfWorkers: 6
 });

package/examples/example3-dynamicIncreasing/run.js

@@ -32,5 +32,5 @@ producer.setTaskTrunkInfo(taskConfig);
 consumer.setTaskTrunkInfo(taskConfig)
 master.run(producer, consumer, {
     quotaOfEachTask:1, //<--Lock as one
-    numberOfWorks:NUMBER_OF_WORKS
+    numberOfWorkers:NUMBER_OF_WORKS
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "1.2.4",
+  "version": "1.2.6",
   "description": "",
   "main": "index.js",
   "directories": {

package/workers/master.js

@@ -2,6 +2,7 @@ var fs = require('fs');
 var pathutil = require('path');
 const cluster = require('cluster');
 const numCPUs = require('os').cpus().length;
+const report_status = require('./report/report_status');
 
 let MSG_REQUEST_TASKID = 'request_taskid';
 /**
@@ -41,7 +42,8 @@ let run = (currentProducer, currentConsumer, opt)=>{
     if(typeof opt === 'undefined') opt = {};    
     let masterId = currentProducer.getTaskName();
     if(typeof opt.quotaOfEachTask === 'undefined') opt.quotaOfEachTask = 1;
-    if(typeof opt.numberOfWorks === 'undefined') opt.numberOfWorks = defaultCPUs;
+    if(opt.numberOfWorks && !opt.numberOfWorkers) opt.numberOfWorkers = opt.numberOfWorks; //Backward compatible with the wrong variable name
+    if(typeof opt.numberOfWorkers === 'undefined') opt.numberOfWorkers = defaultCPUs;
     console.log('master1, masterId=', masterId)
 
 
@@ -50,7 +52,7 @@ let run = (currentProducer, currentConsumer, opt)=>{
     console.log('start', getDateTimeTxt())
 
     if (cluster.isMaster) {//init
-      let numWorks = opt.numberOfWorks;
+      let numWorks = opt.numberOfWorkers;
       console.log(`numCPUs ${numCPUs}`);
       console.log(`numWorks ${numWorks}`);
       console.log(`Master=${process.pid}`);
@@ -95,7 +97,12 @@ let run = (currentProducer, currentConsumer, opt)=>{
                     };
                     let dataPath = pathutil.resolve(taskFolder, `./extractTaskData-${masterId}.json`);
                     fs.writeFileSync(dataPath, JSON.stringify(dataForNextMission))
-                    console.log('dataForNextMission=', dataPath)
+                    console.log('dataForNextMission=', dataPath);
+                    if(opt.onFinish) {
+                      opt.taskFolder = taskFolder;
+                      let report = report_status.getReport(opt);
+                      opt.onFinish(report);
+                    }
                   };
                 }
               }      

package/workers/report/report_status.js

@@ -0,0 +1,52 @@
+var fs = require('fs');
+var pathutil = require('path');
+
+function countDirectChildFiles(directoryPath) {
+    const files = fs.readdirSync(directoryPath);
+    const directChildFiles = files.filter(file => {
+      const filePath = pathutil.join(directoryPath, file);
+      return fs.statSync(filePath).isFile();
+    });
+    return directChildFiles;
+};
+
+let getReport = (taskConfig)=>{
+    let {taskFolder} = taskConfig;
+    let folder_result_errors = pathutil.resolve(taskFolder, './results/errors');
+    let folder_result_succ = pathutil.resolve(taskFolder, './results/succ');
+
+    let folder_new = pathutil.resolve(taskFolder, './progress/new');
+    let folder_running = pathutil.resolve(taskFolder, './progress/running');
+    let folder_finished = pathutil.resolve(taskFolder, './progress/finished');
+    let folder_finished_with_errors = pathutil.resolve(taskFolder, './progress/finished_with_errors');
+
+    let files_new = countDirectChildFiles(folder_new);
+    let files_running = countDirectChildFiles(folder_running);
+    let files_finished = countDirectChildFiles(folder_finished);
+    let files_finished_with_errors = countDirectChildFiles(folder_finished_with_errors);
+
+    let report = {
+        taskConfig,
+        task_folders:{
+            folder_new,
+            folder_running,
+            folder_finished,
+            folder_finished_with_errors
+        },
+        result_folders:{
+            folder_result_errors,
+            folder_result_succ
+        },
+        tasks:{
+            new: files_new.length,
+            running: files_running.length,
+            finished: files_finished.length,
+            finished_with_errors: files_finished_with_errors.length,
+        }
+    };
+    return report;
+};
+
+module.exports = {
+    getReport
+}