concurrency.js 0.0.3 → 0.0.5

package/.todo

@@ -6,7 +6,7 @@
 [*] Easy Cluster (Processess) functionality
 [*] Add Cluster numbers to Concurrency Threading
 [*] Add Demos for all functions
-[] Add Tests for all functions
+[*] Add Tests for all functions
 [] Add thread numbers to Concurrency Threading. Create different function
 [] Add Process numbers to Concurrency Threading. Create different function
 [] Add Thread Async numbers to Concurrency Async Threading. Create different function

package/README.md

@@ -2,139 +2,120 @@
 
 npm module to work with concurrency - worker threads and worker processes easily using simple functions and script files
 
-Find the demos in the [demos folder](./demos)
 
+# Installation
 
-#### Cluster Methods
-
-
-`_concurrencyClusters(filename = __filename, num = cpus().length, options = {}, greet = false)`
 
 ```
-
-const path = require("path");
-let { _concurrencyClusters } = require("concurrency.js");
-
-function concurrency() {
-    return new Promise(function (resolve, reject) {
-        _concurrencyClusters(
-            path.join("C:\\Users\\GB\\Documents\\projects\\requireurl\\concurrency\\src\\worker.cluster.js"),
-            8,
-            {
-                data: {
-                    data: "Testing parent data",
-                    url: "https://www.google.com"
-                },
-                childData: "Test data from child"
-            }
-        ).then((d) => {
-            console.log("Data fetched", JSON.stringify(d));
-            resolve(d);
-        }).catch((e) => {
-            console.log(e.toString());
-            reject(e);
-        });
-    });
-}
-concurrency();
-
+npm install concurrency.js --save
 ```
 
+Find the demos in the [demos folder](./demos). Works but is experimental. Use at your own risk. Please provide as much feedbacks as possible
 
-#### Process Methods
 
+<br/>
 
-`_concurrencyProcesses(filename = __filename, options = {}, greet = false)`
+Run tasks in async mode using promises as simple as below:
 
 ```
-
-const path = require("path");
-let { _concurrencyProcesses } = require("concurrency.js");
-_concurrencyProcesses(
-    path.join(
-    "C:\\Users\\GB\\Documents\\projects\\requireurl\\concurrency\\src\\worker.process.js"),
-    {
-        data: {
-            message: "Testing data",
-            url: "https://www.google.com"
-        }
-    },
-    true
-    ).then((d) => { console.log("Data fetched: ", JSON.stringify(d)); })
-    .catch((e) => { console.log(e.toString()); setTimeout(() => { process.exit(e); }, 5000) })
-
+// This repository's root folder
+const { runPromiseTasks } = require("concurrency.js") // require("../tasks.async.js");
+
+// --- Execution Plan using String Names ---
+const executionPlan = [
+    "taskone",                       // Index 0
+    ["tasktwo", "taskthree", "taskone"], // Index 1 (Parallel)
+    "taskfour",                      // Index 2
+    "taskthree",                     // Index 3
+    "taskone"                        // Index 4
+];
+
+const resultContext = {};
+// This repository's demos folder
+const taskLibraryPath = './demos/libmap.js';
+
+runPromiseTasks(executionPlan, resultContext, taskLibraryPath)
+    .then((finalContext) => {
+        console.log("\n--- Promises Based Final Accumulated Results Context ---");
+        console.log(JSON.stringify(finalContext, null, 2));
+    })
+    .catch((err) => {
+        console.error("Task execution failed:", err);
+    });
 ```
 
-
-![Process Execution Functions](./docs/Concurrency.js.Process.jpg)
-
-
-#### Threads Methods
-
-
-`_concurrencyThreads(filename = __filename, options = {}, greet = false)`
+Run tasks in a mix of nodejs code and run threaded code of nodejs execution plan
 
 ```
-
-const path = require("path");
-let { _concurrencyThreads } = require("concurrency.js");
-_concurrencyThreads(
-    __filename,
-    {
-        data: {
-            url: "https://www.google.com",
-            data: "Testing data"
-        },
-        childData: "Testing child data"
-    },
-    true
-).then((d) => console.log(JSON.stringify(d)));
-
+// This repository's root folder
+const {runThreadedTasks, createWorkerPool} = require("concurrency.js")  // require("../tasks.thread")
+
+// --- Execution Plan ---
+const executionPlan = [
+  "taskone",
+  ["tasktwo", "taskthree", "taskone"], // Threaded parallel
+  "taskfour",
+  "taskthree",
+  "taskone"
+];
+
+const resultContext = { results: [] };
+// This repository's demos folder
+const taskFileName = './demos/libmap.js'; 
+
+// The file to be imported dynamically
+//      from concurrent-tasks\worker.js
+
+runThreadedTasks(executionPlan, resultContext, taskFileName)
+  .then(final => {
+    console.log("\n--- Threads Based Final Accumulated Results ---");
+    console.log(JSON.stringify(final, null, 2));
+    process.exit(0);
+  })
+  .catch(err => {
+    console.error("Fatal Error:", err);
+    process.exit(1);
+  });
 ```
 
-
-![Threads Execution Functions](./docs/Concurrency.js.Threads.jpg)
-
-
-#### Thread Async Methods
-
-
-`_concurrencyThreadsAsync(command, options)`
-
+Run tasks in a mix of nodejs code and run process worker code of nodejs execution plan
 
 ```
-
-const path = require("path");
-let { _concurrencyThreadsAsync } = require("concurrency.js");
-
-let threads = _concurrencyThreadsAsync(
-    "C:\\Users\\GB\\Documents\\projects\\requireurl\\concurrency\\demos\\demos.threads.js",
-    {
-        data: {
-            data: "Testing parent data",
-            url: "https://www.google.com"
-        },
-        childData: "Test data from child"
-    }
-);
+// This repository's root folder
+const { runProcessTasks, runInProcess }  = require("concurrency.js")  // require("../tasks.process");
+
+// --- Execution Plan ---
+const executionPlan = [
+  "taskone",                       // Index 0
+  ["tasktwo", "taskthree", "taskone"], // Index 1 (Parallel Processes)
+  "taskfour",                      // Index 2
+  "taskthree",                     // Index 3
+  "taskone"                        // Index 4
+];
+
+const resultContext = { results: [] };
+const taskLibraryPath = './demos/libmap.js';
+// This repository's demos folder
+
+runProcessTasks(executionPlan, resultContext, taskLibraryPath)
+  .then(final => {
+    console.log("\n--- Final Accumulated Results (Process-Based) ---");
+    console.log(JSON.stringify(final, null, 2));
+  })
+  .catch(err => {
+    console.error("Execution Error:", err);
+  });
 
 ```
 
-
-
-### Contributions
+## Contributions
 
 Contributions, Feature Improvements, Bugs, and Issues are invited. [raising an issue](https://github.com/ganeshkbhat/concurrency.js/issues)
 
-
-
-### TODO
+## TODO
 
 [Todo](./todo)
 
-
-
 # License
 
 [MIT License](./LICENSE)
-

package/demos/concurrency.async.js

@@ -0,0 +1,22 @@
+const { runPromiseTasks } = require("../tasks.async.js");
+
+// --- Execution Plan using String Names ---
+const executionPlan = [
+    "taskone",                       // Index 0
+    ["tasktwo", "taskthree", "taskone"], // Index 1 (Parallel)
+    "taskfour",                      // Index 2
+    "taskthree",                     // Index 3
+    "taskone"                        // Index 4
+];
+
+const resultContext = {};
+const taskLibraryPath = './demos/libmap.js';
+
+runPromiseTasks(executionPlan, resultContext, taskLibraryPath)
+    .then((finalContext) => {
+        console.log("\n--- Promises Based Final Accumulated Results Context ---");
+        console.log(JSON.stringify(finalContext, null, 2));
+    })
+    .catch((err) => {
+        console.error("Task execution failed:", err);
+    });

package/demos/concurrency.process.js

@@ -0,0 +1,24 @@
+
+const { runProcessTasks, runInProcess }  = require("../tasks.process");
+
+// --- Execution Plan ---
+const executionPlan = [
+  "taskone",                       // Index 0
+  ["tasktwo", "taskthree", "taskone"], // Index 1 (Parallel Processes)
+  "taskfour",                      // Index 2
+  "taskthree",                     // Index 3
+  "taskone"                        // Index 4
+];
+
+const resultContext = { results: [] };
+const taskLibraryPath = './demos/libmap.js';
+
+runProcessTasks(executionPlan, resultContext, taskLibraryPath)
+  .then(final => {
+    console.log("\n--- Final Accumulated Results (Process-Based) ---");
+    console.log(JSON.stringify(final, null, 2));
+  })
+  .catch(err => {
+    console.error("Execution Error:", err);
+  });
+

package/demos/concurrency.threaded.js

@@ -0,0 +1,25 @@
+
+const {runThreadedTasks, createWorkerPool} = require("../tasks.thread")
+
+// --- Execution Plan ---
+const executionPlan = [
+  "taskone",
+  ["tasktwo", "taskthree", "taskone"],
+  "taskfour",
+  "taskthree",
+  "taskone"
+];
+
+const resultContext = { results: [] };
+const taskFileName = './demos/libmap.js'; // The file to be imported dynamically
+// concurrent-tasks\worker.js
+runThreadedTasks(executionPlan, resultContext, taskFileName)
+  .then(final => {
+    console.log("\n--- Threads Based Final Accumulated Results ---");
+    console.log(JSON.stringify(final, null, 2));
+    process.exit(0);
+  })
+  .catch(err => {
+    console.error("Fatal Error:", err);
+    process.exit(1);
+  });

package/demos/libmap.js

@@ -0,0 +1,30 @@
+
+// --- Task Definitions ---
+const taskone = async (context, index) => {
+  console.log(`[Step ${index}] Running: taskone`);
+  await new Promise(resolve => setTimeout(resolve, 50));
+  return `Data from taskone`;
+};
+
+const tasktwo = async (context, index) => {
+  console.log(`[Step ${index}] Running: tasktwo`);
+  await new Promise(resolve => setTimeout(resolve, 50));
+  return "Data from tasktwo";
+};
+
+const taskthree = async (context, index) => {
+  console.log(`[Step ${index}] Running: taskthree`);
+  await new Promise(resolve => setTimeout(resolve, 50));
+  return "Data from taskthree";
+};
+
+const taskfour = async (context, index) => {
+  console.log(`[Step ${index}] Running: taskfour`);
+  await new Promise(resolve => setTimeout(resolve, 50));
+  return "Data from taskfour";
+};
+
+// Map of tasks for the Worker to reference by name
+const taskMap = { taskone, tasktwo, taskthree, taskfour };
+
+module.exports = taskMap;

package/index.js

@@ -1,46 +1,12 @@
-/**
- * 
- * Package: concurrency.js
- * Author: Ganesh B
- * Description: npm module to work with concurrency - worker threads and worker processes easily using simple functions and script files
- * Install: npm i concurrency.js --save
- * Github: https://github.com/ganeshkbhat/concurrency
- * npmjs Link: https://www.npmjs.com/package/
- * File: index.js
- * File Description: 
- * 
-*/
-
-/* eslint no-console: 0 */
-
-'use strict';
-
-
-const { _concurrencyThreads } = require("./src/worker.threads.js");
-const { _concurrencyProcesses } = require("./src/worker.process.js");
-const { _concurrencyClusters } = require("./src/worker.cluster.js");
-const { _concurrencyThreadsAsync } = require("./src/worker.thread.async.js");
-
-
-module.exports._concurrencyThreads = _concurrencyThreads;
-module.exports._concurrencyProcesses = _concurrencyProcesses;
-module.exports._concurrencyClusters = _concurrencyClusters;
-module.exports._concurrencyThreadsAsync = _concurrencyThreadsAsync;
-
-module.exports.concurrencyThreads = _concurrencyThreads;
-module.exports.concurrencyProcesses = _concurrencyProcesses;
-module.exports.concurrencyClusters = _concurrencyClusters;
-module.exports.concurrencyThreadsAsync = _concurrencyThreadsAsync;
-
-module.exports.default = {
-    _concurrencyThreads,
-    _concurrencyProcesses,
-    _concurrencyClusters,
-    _concurrencyThreadsAsync,
-
-    concurrencyThreads: _concurrencyThreads,
-    concurrencyProcesses: _concurrencyProcesses,
-    concurrencyClusters: _concurrencyClusters,
-    concurrencyThreadsAsync: _concurrencyThreadsAsync
-    
-};
+const { runThreadedTasks, createWorkerPool } = require("./tasks.thread");
+const { runPromiseTasks } = require("./tasks.async");
+const { runProcessTasks, runInProcess }  = require("./tasks.process");
+
+// concurrent-tasks\worker_process.js
+// concurrent-tasks\worker.js
+
+module.exports = {
+    runThreadedTasks, createWorkerPool,
+    runPromiseTasks, runProcessTasks,
+    runInProcess
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "concurrency.js",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "description": "npm module to work with concurrency - worker threads and worker processes (currrently only fork method) easily using simple functions and script files",
   "main": "index.js",
   "directories": {
@@ -11,8 +11,9 @@
     "import": "./index.mjs"
   },
   "devDependencies": {
-    "chai": "^4.3.6",
-    "mocha": "^10.2.0",
+    "chai": "^4.5.0",
+    "express": "^4.18.2",
+    "mocha": "^10.8.2",
     "npm-check": "^6.0.1",
     "sinon": "^14.0.0",
     "unimported": "^1.22.0"
@@ -34,5 +35,8 @@
     "processes"
   ],
   "author": "Ganesh B",
-  "license": "MIT"
+  "license": "MIT",
+  "dependencies": {
+    "loadbalancerjs": "^0.0.1"
+  }
 }

package/tasks.async.js

@@ -0,0 +1,72 @@
+const path = require('path');
+
+/**
+ * Task Runner for Serial and Parallel Execution
+ * * Logic:
+ * - Resolves function names provided as strings using the taskMap.
+ * - Maintains strict order based on the executionPlan.
+ * - Passes the 'resultContext' and current 'index' into every task.
+ * - Accumulates results in a generic list to avoid key-overwriting.
+ */
+async function runPromiseTasks(taskList, resultContext, taskFilePath) {
+  console.log("Starting task execution sequence...");
+
+  // Dynamically load the task map from the provided library file
+  const taskMap = require(path.resolve(taskFilePath));
+
+  // Initialize the results array in context if it doesn't exist
+  if (!resultContext.results) {
+    resultContext.results = [];
+  }
+
+  for (let i = 0; i < taskList.length; i++) {
+    const step = taskList[i];
+
+    if (Array.isArray(step)) {
+      // Parallel Execution
+      console.log(`Step Index ${i}: Executing Parallel Block`);
+      
+      const parallelPromises = step.map(async (taskName) => {
+        const taskFunction = taskMap[taskName];
+        if (!taskFunction) {
+          throw new Error(`Task '${taskName}' not found in ${taskFilePath}`);
+        }
+        
+        // Execute task and capture result with metadata
+        const data = await taskFunction(resultContext, i);
+        return {
+          index: i,
+          name: taskName,
+          result: data
+        };
+      });
+
+      // Wait for all parallel tasks in this block to complete
+      const parallelResults = await Promise.all(parallelPromises);
+      resultContext.results.push(...parallelResults);
+    } else {
+      // Serial Execution
+      console.log(`Step Index ${i}: Executing Serial Task (${step})`);
+      
+      const taskFunction = taskMap[step];
+      if (!taskFunction) {
+        throw new Error(`Task '${step}' not found in ${taskFilePath}`);
+      }
+
+      const data = await taskFunction(resultContext, i);
+      
+      // Accumulate serial result into the context immediately
+      resultContext.results.push({
+        index: i,
+        name: step,
+        result: data
+      });
+    }
+  }
+
+  return resultContext;
+}
+
+module.exports = {
+  runPromiseTasks
+};

package/tasks.process.js

@@ -0,0 +1,65 @@
+// concurrency.async.js
+const { fork } = require('child_process');
+const path = require('path');
+
+/**
+ * Helper to run a task in a separate Process
+ */
+function runInProcess(taskName, context, index, taskFilePath) {
+  return new Promise((resolve, reject) => {
+    // Spawns a new Node.js process
+    const child = fork(path.join(__dirname, 'worker_process.js'));
+
+    child.on('message', (msg) => {
+      if (msg.status === 'success') resolve(msg.result);
+      else reject(new Error(msg.error));
+    });
+
+    child.on('error', reject);
+    
+    child.on('exit', (code) => {
+      if (code !== 0 && code !== null) {
+        reject(new Error(`Process exited with code ${code}`));
+      }
+    });
+
+    // Send the task details to the child process
+    child.send({ taskName, context, index, taskFilePath });
+  });
+}
+
+/**
+ * Main Task Runner
+ */
+async function runProcessTasks(taskList, resultContext, taskFilePath) {
+  console.log("--- Starting Process-Based Execution ---");
+  if (!resultContext.results) resultContext.results = [];
+
+  for (let i = 0; i < taskList.length; i++) {
+    const step = taskList[i];
+
+    if (Array.isArray(step)) {
+      console.log(`Step ${i}: Dispatching Parallel Processes...`);
+      
+      // Execute each string in the array as a separate process
+      const parallelPromises = step.map(taskName => 
+        runInProcess(taskName, resultContext, i, taskFilePath)
+      );
+
+      const outputs = await Promise.all(parallelPromises);
+      resultContext.results.push(...outputs);
+    } else {
+      console.log(`Step ${i}: Running Serial Task (${step})`);
+      
+      // Serial tasks run in a process to maintain total isolation
+      const output = await runInProcess(step, resultContext, i, taskFilePath);
+      resultContext.results.push(output);
+    }
+  }
+  return resultContext;
+}
+
+module.exports = {
+  runProcessTasks,
+  runInProcess
+}

package/tasks.thread.js

@@ -0,0 +1,92 @@
+const { Worker } = require('worker_threads');
+const path = require('path');
+const os = require('os');
+
+/**
+ * Functional Worker Pool Factory
+ * @param {string} taskFilePath - Path to the file containing task functions
+ */
+function createWorkerPool(size, taskFilePath) {
+  const workers = [];
+  const queue = [];
+
+  for (let i = 0; i < size; i++) {
+    const worker = new Worker(path.join(__dirname, 'worker.js'), { 
+      stdout: true,
+      workerData: { taskFilePath } // Sending the filename here
+    });
+    
+    worker.stdout.on('data', data => process.stdout.write(data));
+    workers.push({ instance: worker, inUse: false });
+  }
+
+  const execute = (workerObj, taskName, context, index, resolve, reject) => {
+    workerObj.inUse = true;
+
+    const onMessage = (msg) => {
+      workerObj.instance.off('message', onMessage);
+      workerObj.inUse = false;
+
+      if (msg.status === 'success') resolve(msg.result);
+      else reject(new Error(msg.error));
+
+      if (queue.length > 0) {
+        const next = queue.shift();
+        execute(workerObj, next.taskName, next.context, next.index, next.resolve, next.reject);
+      }
+    };
+
+    workerObj.instance.on('message', onMessage);
+    workerObj.instance.postMessage({ taskName, context, index });
+  };
+
+  return {
+    runTask: (taskName, context, index) => {
+      return new Promise((resolve, reject) => {
+        const availableWorker = workers.find(w => !w.inUse);
+        if (availableWorker) {
+          execute(availableWorker, taskName, context, index, resolve, reject);
+        } else {
+          queue.push({ taskName, context, index, resolve, reject });
+        }
+      });
+    },
+    destroy: () => {
+      workers.forEach(w => w.instance.terminate());
+    }
+  };
+}
+
+async function runThreadedTasks(taskList, resultContext, taskFileName) {
+  const poolSize = Math.max(2, Math.floor(os.cpus().length / 2));
+  const pool = createWorkerPool(poolSize, taskFileName);
+  
+  console.log(`--- Starting Dynamic Execution (File: ${taskFileName}) ---`);
+  if (!resultContext.results) resultContext.results = [];
+
+  try {
+    for (let i = 0; i < taskList.length; i++) {
+      const taskEntry = taskList[i];
+
+      if (Array.isArray(taskEntry)) {
+        console.log(`[Main] Dispatching Parallel Block at Index ${i}`);
+        const outputs = await Promise.all(
+          taskEntry.map(name => pool.runTask(name, resultContext, i))
+        );
+        resultContext.results.push(...outputs);
+      } else {
+        console.log(`[Main] Running Serial Task '${taskEntry}' at Index ${i}`);
+        const output = await pool.runTask(taskEntry, resultContext, i);
+        resultContext.results.push(output);
+      }
+    }
+  } finally {
+    pool.destroy();
+  }
+  return resultContext;
+}
+
+module.exports = {
+  createWorkerPool,
+  runThreadedTasks
+}