multi-tasks 1.2.5 → 1.2.7

package/README.md

@@ -43,12 +43,15 @@ multiTasks({
     processTask, 
     multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
     quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
-    numberOfWorkers: 6 //Assign how many workers are working in parallel, default is the number of cpu cores minus one
+    numberOfWorkers: 6, //Assign how many workers are working in parallel, default is the number of cpu cores minus one
+    onFinish: (report)=>{
+        console.log(report);//the report can be viewed here
+    }
 });
 
 ```
 
-Or, to provide the Producer & Consumer
+Or the older way, to provide the Producer & Consumer
 
 ```javascript
 let master = require('multi-tasks').master;
@@ -56,9 +59,10 @@ let RunnerProducer;//See examples, please provide a Producer class to specify al
 let RunnerConsumer;//See examples, please provide a Consumer class to process a certain sub-task and return the processed data
 
 master.start(RunnerProducer, RunnerConsumer, {
-    multi_task_parent_folder: `C:/tmp-demo`,
-    quotaOfEachTask:2, //The number of tasks assigned to each worker
-    numberOfWorkers: 6
+    multi_task_parent_folder,
+    quotaOfEachTask,
+    numberOfWorkers,
+    onFinish
 });
 ```
 
@@ -71,6 +75,8 @@ node examples/example1/run
 
 Changelog:
 
+- 1.2.7 Small updates
+- 1.2.6 Support onFinish event
 - 1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
 - 1.2.4 Fix: opt.numberOfWorkers not work
 - 1.2.3 Update README

package/examples/example0/run.js

@@ -31,5 +31,8 @@ multiTasks({
     processTask, 
     multi_task_parent_folder: `C:/tmp-demo`, //a directory to store progress and results files, you can check the progress here
     quotaOfEachTask:1, //The number of subtasks assigned to each worker at one time, the default is 1
-    numberOfWorkers: 6 //Assign how many workers are working in parallel
+    numberOfWorkers: 6, //Assign how many workers are working in parallel
+    onFinish: (report)=>{
+        console.log(report);
+    }
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "multi-tasks",
-  "version": "1.2.5",
+  "version": "1.2.7",
   "description": "",
   "main": "index.js",
   "directories": {

package/workers/master.js

@@ -2,6 +2,7 @@ var fs = require('fs');
 var pathutil = require('path');
 const cluster = require('cluster');
 const numCPUs = require('os').cpus().length;
+const report_status = require('./report/report_status');
 
 let MSG_REQUEST_TASKID = 'request_taskid';
 /**
@@ -96,7 +97,12 @@ let run = (currentProducer, currentConsumer, opt)=>{
                     };
                     let dataPath = pathutil.resolve(taskFolder, `./extractTaskData-${masterId}.json`);
                     fs.writeFileSync(dataPath, JSON.stringify(dataForNextMission))
-                    console.log('dataForNextMission=', dataPath)
+                    console.log('dataForNextMission=', dataPath);
+                    if(opt.onFinish) {
+                      opt.taskFolder = taskFolder;
+                      let report = report_status.getReport(opt);
+                      opt.onFinish(report);
+                    }
                   };
                 }
               }      

package/workers/report/report_status.js

@@ -0,0 +1,52 @@
+var fs = require('fs');
+var pathutil = require('path');
+
+function countDirectChildFiles(directoryPath) {
+    const files = fs.readdirSync(directoryPath);
+    const directChildFiles = files.filter(file => {
+      const filePath = pathutil.join(directoryPath, file);
+      return fs.statSync(filePath).isFile();
+    });
+    return directChildFiles;
+};
+
+let getReport = (taskConfig)=>{
+    let {taskFolder} = taskConfig;
+    let folder_result_errors = pathutil.resolve(taskFolder, './results/errors');
+    let folder_result_succ = pathutil.resolve(taskFolder, './results/succ');
+
+    let folder_new = pathutil.resolve(taskFolder, './progress/new');
+    let folder_running = pathutil.resolve(taskFolder, './progress/running');
+    let folder_finished = pathutil.resolve(taskFolder, './progress/finished');
+    let folder_finished_with_errors = pathutil.resolve(taskFolder, './progress/finished_with_errors');
+
+    let files_new = countDirectChildFiles(folder_new);
+    let files_running = countDirectChildFiles(folder_running);
+    let files_finished = countDirectChildFiles(folder_finished);
+    let files_finished_with_errors = countDirectChildFiles(folder_finished_with_errors);
+
+    let report = {
+        taskConfig,
+        task_folders:{
+            folder_new,
+            folder_running,
+            folder_finished,
+            folder_finished_with_errors
+        },
+        result_folders:{
+            folder_result_errors,
+            folder_result_succ
+        },
+        tasks:{
+            new: files_new.length,
+            running: files_running.length,
+            finished: files_finished.length,
+            finished_with_errors: files_finished_with_errors.length,
+        }
+    };
+    return report;
+};
+
+module.exports = {
+    getReport
+}