@parallel-park/run-jobs 0.3.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License Copyright (c) 2022-2026 Lily Skye
+
+Permission is hereby granted, free of
+charge, to any person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice
+(including the next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,67 @@
+# @parallel-park/run-jobs
+
+Parallel async work with an optional concurrency limit, like [Bluebird's Promise.map](http://bluebirdjs.com/docs/api/promise.map.html).
+
+## Usage
+
+`@parallel-park/run-jobs` exports one function: `runJobs`.
+
+### `runJobs`
+
+`runJobs` is kinda like `Promise.all`, but instead of running everything at once, it'll only run a few Promises at a time (you can choose how many to run at once). It's inspired by [Bluebird's Promise.map function](http://bluebirdjs.com/docs/api/promise.map.html).
+
+To use it, you pass in an iterable (array, set, generator function, etc) of inputs and a mapper function that transforms each input into a Promise. You can also optionally specify the maximum number of Promises to wait on at a time by passing an object with a `concurrency` property, which is a number. The concurrency defaults to 8.
+
+When using an iterable, if the iterable yields a Promise (ie. `iterable.next() returns { done: false, value: Promise }`), then the yielded Promise will be awaited before being passed into your mapper function. Additionally, async iterables are supported; if `iterable.next()` returns a Promise, it will be awaited.
+
+```ts
+import { runJobs } from "parallel-park";
+
+const inputs = ["alice", "bob", "carl", "dorsey", "edith"];
+const results = await runJobs(
+  inputs,
+  async (name, index, inputsCount) => {
+    // Do whatever async work you want inside this mapper function.
+    // In this case, we use a hypothetical "getUser" function to
+    // retrieve data about a user from some web API.
+    console.log(`Getting fullName for ${name}...`);
+    const user = await getUser(name);
+    return user.fullName;
+  },
+  // This options object with concurrency is an optional argument.
+  // If unspecified, it defaults to { concurrency: 8 }
+  {
+    // This number specifies how many times to call the mapper
+    // function before waiting for one of the returned Promises
+    // to resolve. Ie. "How many promises to have in-flight concurrently"
+    concurrency: 2,
+  }
+);
+// Logs these two immediately:
+//
+// Getting fullName for alice...
+// Getting fullName for bob...
+//
+// But, it doesn't log anything else yet, because we told it to only run two things at a time.
+// Then, after one of those Promises has finished, it logs:
+//
+// Getting fullName for carl...
+//
+// And so forth, until all of them are done.
+
+// `results` is an Array of the resolved value you returned from your mapper function.
+// The indices in the array correspond to the indices of your inputs.
+console.log(results);
+// Logs:
+// [
+//   "Alice Smith",
+//   "Bob Eriksson",
+//   "Carl Martinez",
+//   "Dorsey Toth",
+//   "Edith Skalla"
+// ]
+```
+
+## License
+
+MIT

package/dist/index.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setDebug = exports.runJobs = void 0;
+var run_jobs_1 = require("./run-jobs");
+Object.defineProperty(exports, "runJobs", { enumerable: true, get: function () { return run_jobs_1.runJobs; } });
+Object.defineProperty(exports, "setDebug", { enumerable: true, get: function () { return run_jobs_1.setDebug; } });

package/dist/run-jobs.js

@@ -0,0 +1,119 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setDebug = setDebug;
+exports.runJobs = runJobs;
+let debug = null;
+function setDebug(debugFunction) {
+    debug = debugFunction;
+}
+function isThenable(value) {
+    return (typeof value === "object" &&
+        value != null &&
+        // @ts-ignore accessing .then
+        typeof value.then === "function");
+}
+const NOTHING = Symbol("NOTHING");
+let runJobsCallId = 0;
+async function runJobs(inputs, mapper, { 
+/**
+ * How many jobs are allowed to run at once.
+ */
+concurrency = 8, } = {}) {
+    const callId = runJobsCallId;
+    runJobsCallId++;
+    if (debug) {
+        debug(`runJobs called (callId: ${callId})`, {
+            inputs,
+            mapper,
+            concurrency,
+        });
+    }
+    if (concurrency < 1) {
+        throw new Error("Concurrency can't be less than one; that doesn't make any sense.");
+    }
+    const inputsArray = [];
+    const inputIteratorFactory = inputs[Symbol.asyncIterator || NOTHING] || inputs[Symbol.iterator];
+    const inputIterator = inputIteratorFactory.call(inputs);
+    const maybeLength = Array.isArray(inputs) ? inputs.length : null;
+    let iteratorDone = false;
+    async function readInput() {
+        if (debug) {
+            debug(`reading next input (callId: ${callId})`);
+        }
+        let nextResult = inputIterator.next();
+        if (isThenable(nextResult)) {
+            nextResult = await nextResult;
+        }
+        if (nextResult.done) {
+            iteratorDone = true;
+            return false;
+        }
+        else {
+            let value = nextResult.value;
+            if (isThenable(value)) {
+                value = await value;
+            }
+            inputsArray.push(value);
+            return true;
+        }
+    }
+    let unstartedIndex = 0;
+    const results = new Array(maybeLength || 0);
+    const runningPromises = new Set();
+    let error = null;
+    async function takeInput() {
+        const read = await readInput();
+        if (!read)
+            return;
+        const inputIndex = unstartedIndex;
+        unstartedIndex++;
+        const input = inputsArray[inputIndex];
+        if (debug) {
+            debug(`mapping input into Promise (callId: ${callId})`);
+        }
+        const promise = mapper(input, inputIndex, maybeLength || Infinity);
+        if (!isThenable(promise)) {
+            throw new Error("Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function.");
+        }
+        const promiseWithMore = promise.then((result) => {
+            if (debug) {
+                debug(`child Promise resolved for input (callId: ${callId}):`, input);
+            }
+            results[inputIndex] = result;
+            runningPromises.delete(promiseWithMore);
+        }, (err) => {
+            if (debug) {
+                debug(`child Promise rejected for input (callId: ${callId}):`, input, "with error:", err);
+            }
+            runningPromises.delete(promiseWithMore);
+            error = err;
+        });
+        runningPromises.add(promiseWithMore);
+    }
+    async function proceed() {
+        while (!iteratorDone && runningPromises.size < concurrency) {
+            await takeInput();
+        }
+    }
+    await proceed();
+    while (runningPromises.size > 0 && !error) {
+        await Promise.race(runningPromises.values());
+        if (error) {
+            if (debug) {
+                debug(`throwing error (callId: ${callId})`);
+            }
+            throw error;
+        }
+        await proceed();
+    }
+    if (error) {
+        if (debug) {
+            debug(`throwing error (callId: ${callId})`);
+        }
+        throw error;
+    }
+    if (debug) {
+        debug(`all done (callId: ${callId})`);
+    }
+    return results;
+}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@parallel-park/run-jobs",
+  "version": "0.3.0",
+  "description": "Parallel async work with an optional concurrency limit, like Bluebird's Promise.map",
+  "main": "dist/index.js",
+  "types": "src/index.ts",
+  "scripts": {
+    "test": "vitest run",
+    "build": "rm -rf ./dist && tsc"
+  },
+  "keywords": [
+    "parallel",
+    "concurrent",
+    "concurrency",
+    "co-operative",
+    "promise",
+    "map",
+    "bluebird",
+    "bluebird.map"
+  ],
+  "author": "Lily Skye <me@suchipi.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/suchipi/parallel-park.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/index.ts

@@ -0,0 +1 @@
+export { runJobs, setDebug } from "./run-jobs";

package/src/run-jobs.ts

@@ -0,0 +1,159 @@
+let debug: null | ((...args: any) => void) = null;
+
+export function setDebug(debugFunction: typeof debug) {
+  debug = debugFunction;
+}
+
+function isThenable<T>(value: unknown): value is Promise<T> {
+  return (
+    typeof value === "object" &&
+    value != null &&
+    // @ts-ignore accessing .then
+    typeof value.then === "function"
+  );
+}
+
+const NOTHING = Symbol("NOTHING");
+
+let runJobsCallId = 0;
+
+export async function runJobs<T, U>(
+  inputs: Iterable<T | Promise<T>> | AsyncIterable<T | Promise<T>>,
+  mapper: (input: T, index: number, length: number) => Promise<U>,
+  {
+    /**
+     * How many jobs are allowed to run at once.
+     */
+    concurrency = 8,
+  }: {
+    /**
+     * How many jobs are allowed to run at once.
+     */
+    concurrency?: number;
+  } = {}
+): Promise<Array<U>> {
+  const callId = runJobsCallId;
+  runJobsCallId++;
+
+  if (debug) {
+    debug(`runJobs called (callId: ${callId})`, {
+      inputs,
+      mapper,
+      concurrency,
+    });
+  }
+
+  if (concurrency < 1) {
+    throw new Error(
+      "Concurrency can't be less than one; that doesn't make any sense."
+    );
+  }
+
+  const inputsArray: Array<T> = [];
+  const inputIteratorFactory =
+    inputs[Symbol.asyncIterator || NOTHING] || inputs[Symbol.iterator];
+  const inputIterator = inputIteratorFactory.call(inputs);
+  const maybeLength = Array.isArray(inputs) ? inputs.length : null;
+
+  let iteratorDone = false;
+
+  async function readInput(): Promise<boolean> {
+    if (debug) {
+      debug(`reading next input (callId: ${callId})`);
+    }
+    let nextResult = inputIterator.next();
+    if (isThenable(nextResult)) {
+      nextResult = await nextResult;
+    }
+    if (nextResult.done) {
+      iteratorDone = true;
+      return false;
+    } else {
+      let value = nextResult.value;
+      if (isThenable<T>(value)) {
+        value = await value;
+      }
+      inputsArray.push(value);
+      return true;
+    }
+  }
+
+  let unstartedIndex = 0;
+
+  const results = new Array(maybeLength || 0);
+  const runningPromises = new Set();
+  let error: Error | null = null;
+
+  async function takeInput() {
+    const read = await readInput();
+    if (!read) return;
+
+    const inputIndex = unstartedIndex;
+    unstartedIndex++;
+
+    const input = inputsArray[inputIndex];
+    if (debug) {
+      debug(`mapping input into Promise (callId: ${callId})`);
+    }
+    const promise = mapper(input, inputIndex, maybeLength || Infinity);
+
+    if (!isThenable(promise)) {
+      throw new Error(
+        "Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function."
+      );
+    }
+
+    const promiseWithMore = promise.then(
+      (result) => {
+        if (debug) {
+          debug(`child Promise resolved for input (callId: ${callId}):`, input);
+        }
+        results[inputIndex] = result;
+        runningPromises.delete(promiseWithMore);
+      },
+      (err) => {
+        if (debug) {
+          debug(
+            `child Promise rejected for input (callId: ${callId}):`,
+            input,
+            "with error:",
+            err
+          );
+        }
+        runningPromises.delete(promiseWithMore);
+        error = err;
+      }
+    );
+    runningPromises.add(promiseWithMore);
+  }
+
+  async function proceed() {
+    while (!iteratorDone && runningPromises.size < concurrency) {
+      await takeInput();
+    }
+  }
+
+  await proceed();
+  while (runningPromises.size > 0 && !error) {
+    await Promise.race(runningPromises.values());
+    if (error) {
+      if (debug) {
+        debug(`throwing error (callId: ${callId})`);
+      }
+      throw error;
+    }
+    await proceed();
+  }
+
+  if (error) {
+    if (debug) {
+      debug(`throwing error (callId: ${callId})`);
+    }
+    throw error;
+  }
+
+  if (debug) {
+    debug(`all done (callId: ${callId})`);
+  }
+  return results;
+}

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.json",
+
+  "include": ["./src/**/*.ts"],
+  "exclude": ["./**/*.test.ts"],
+
+  "compilerOptions": {
+    "outDir": "./dist"
+  }
+}