@actsecurity/job 0.1.63

package/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Cloud Copilot
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,9 @@
+# Job
+
+[![NPM Version](https://img.shields.io/npm/v/@cloud-copilot/job.svg?logo=nodedotjs)](https://www.npmjs.com/package/@cloud-copilot/job) [![MIT](https://img.shields.io/github/license/cloud-copilot/job)](LICENSE.txt) [![GuardDog](https://github.com/cloud-copilot/job/actions/workflows/guarddog.yml/badge.svg)](https://github.com/cloud-copilot/job/actions/workflows/guarddog.yml) [![Known Vulnerabilities](https://snyk.io/test/github/cloud-copilot/job/badge.svg?targetFile=package.json&style=flat-square)](https://snyk.io/test/github/cloud-copilot/job?targetFile=package.json)
+
+Functions and classes for running node tasks in parallel while controlling concurrency.
+
+## License
+
+MIT

package/dist/cjs/ConcurrentJobQueue.d.ts

@@ -0,0 +1,65 @@
+import { type Logger } from '@cloud-copilot/log';
+import { type Job, type JobResult } from './job.js';
+/**
+ * Creates a queue that runs jobs concurrently up to a specified limit.
+ * This will wait for jobs to be added to it and run them up the
+ * maximum concurrency.
+ *
+ * Results are available via `getResults()`.
+ */
+export declare class ConcurrentJobQueue<T = void, P = Record<string, unknown>> {
+    private concurrency;
+    private logger;
+    private queue;
+    private results;
+    private activeJobs;
+    private waitingResolvers;
+    private workers;
+    private isAcceptingWork;
+    private workAvailablePromise;
+    private resolveWorkAvailable;
+    /**
+     * Create a new runner with the specified concurrency.
+     *
+     * @param concurrency - The maximum number of jobs to run concurrently.
+     * @param logger - Optional logger instance for long-running job warnings. Defaults to the module-level logger from @cloud-copilot/log.
+     */
+    constructor(concurrency: number, logger?: Logger);
+    private worker;
+    private waitForWorkAvailable;
+    private ensureWorkers;
+    private notifyWorkersOfNewWork;
+    private checkIfIdle;
+    /**
+     * Add a job to the queue
+     */
+    enqueue(job: Job<T, P>): void;
+    /**
+     * Add multiple jobs to the queue
+     */
+    enqueueAll(jobs: Job<T, P>[]): void;
+    /**
+     * Returns a promise that resolves when all queued work is complete
+     */
+    waitForIdle(): Promise<void>;
+    /**
+     * Get all results accumulated so far
+     */
+    getResults(): JobResult<T, P>[];
+    /**
+     * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
+     *
+     * Returns when a promise that resolves when all jobs have been processed and
+     * are available in `getResults()`.
+     */
+    finishAllWork(): Promise<void>;
+    /**
+     * Get the current queue length
+     */
+    get queueLength(): number;
+    /**
+     * Get the number of currently active jobs
+     */
+    get activeJobCount(): number;
+}
+//# sourceMappingURL=ConcurrentJobQueue.d.ts.map

package/dist/cjs/ConcurrentJobQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConcurrentJobQueue.d.ts","sourceRoot":"","sources":["../../src/ConcurrentJobQueue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwB,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAA;AACtE,OAAO,EAAE,KAAK,GAAG,EAAmB,KAAK,SAAS,EAAE,MAAM,UAAU,CAAA;AAEpE;;;;;;GAMG;AAEH,qBAAa,kBAAkB,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAiBjE,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;IAjBhB,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAO;IAC9B,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,oBAAoB,CAA4B;IAExD;;;;;OAKG;gBAEO,WAAW,EAAE,MAAM,EACnB,MAAM,GAAE,MAAsB;YAG1B,MAAM;IAsCpB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI;IAS7B;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI;IAInC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;OAEG;IACH,UAAU,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;IAI/B;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IASpC;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;CACF"}

package/dist/cjs/ConcurrentJobQueue.js

@@ -0,0 +1,160 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConcurrentJobQueue = void 0;
+const log_1 = require("@cloud-copilot/log");
+/**
+ * Creates a queue that runs jobs concurrently up to a specified limit.
+ * This will wait for jobs to be added to it and run them up the
+ * maximum concurrency.
+ *
+ * Results are available via `getResults()`.
+ */
+class ConcurrentJobQueue {
+    concurrency;
+    logger;
+    queue = [];
+    results = [];
+    activeJobs = 0;
+    waitingResolvers = [];
+    workers = [];
+    isAcceptingWork = true;
+    workAvailablePromise = null;
+    resolveWorkAvailable = null;
+    /**
+     * Create a new runner with the specified concurrency.
+     *
+     * @param concurrency - The maximum number of jobs to run concurrently.
+     * @param logger - Optional logger instance for long-running job warnings. Defaults to the module-level logger from @cloud-copilot/log.
+     */
+    constructor(concurrency, logger = log_1.log) {
+        this.concurrency = concurrency;
+        this.logger = logger;
+    }
+    async worker(workerId) {
+        while (this.isAcceptingWork || this.queue.length > 0) {
+            const job = this.queue.shift();
+            if (!job) {
+                if (!this.isAcceptingWork) {
+                    // No longer accepting work and no jobs left, exit immediately
+                    return;
+                }
+                // No work available, wait for new work to be added
+                await this.waitForWorkAvailable();
+                continue;
+            }
+            this.activeJobs++;
+            const context = { workerId };
+            const startTime = Date.now();
+            const interval = setInterval(() => {
+                this.logger.warn(`Long-running job detected.`, { minutes: Math.floor((Date.now() - startTime) / 60000) }, { ...context, ...job.properties });
+            }, 60_000);
+            try {
+                const value = await job.execute({ ...context, properties: job.properties });
+                this.results.push({ status: 'fulfilled', value, properties: job.properties });
+            }
+            catch (reason) {
+                this.results.push({ status: 'rejected', reason, properties: job.properties });
+            }
+            finally {
+                clearInterval(interval);
+                this.activeJobs--;
+                this.checkIfIdle();
+            }
+        }
+    }
+    waitForWorkAvailable() {
+        if (!this.workAvailablePromise) {
+            this.workAvailablePromise = new Promise((resolve) => {
+                this.resolveWorkAvailable = resolve;
+            });
+        }
+        return this.workAvailablePromise;
+    }
+    ensureWorkers() {
+        if (this.workers.length === 0 && this.isAcceptingWork) {
+            for (let i = 0; i < this.concurrency; i++) {
+                this.workers.push(this.worker(i + 1));
+            }
+        }
+    }
+    notifyWorkersOfNewWork() {
+        // Wake up waiting workers
+        if (this.resolveWorkAvailable) {
+            this.resolveWorkAvailable();
+            this.workAvailablePromise = null;
+            this.resolveWorkAvailable = null;
+        }
+    }
+    checkIfIdle() {
+        if (this.activeJobs === 0 && this.queue.length === 0) {
+            // Notify all waiting resolvers
+            this.waitingResolvers.forEach((resolve) => resolve());
+            this.waitingResolvers = [];
+        }
+    }
+    /**
+     * Add a job to the queue
+     */
+    enqueue(job) {
+        if (!this.isAcceptingWork) {
+            throw new Error('Cannot enqueue jobs after shutdown');
+        }
+        this.queue.push(job);
+        this.ensureWorkers();
+        this.notifyWorkersOfNewWork();
+    }
+    /**
+     * Add multiple jobs to the queue
+     */
+    enqueueAll(jobs) {
+        jobs.forEach((job) => this.enqueue(job));
+    }
+    /**
+     * Returns a promise that resolves when all queued work is complete
+     */
+    waitForIdle() {
+        // log.debug('waitForIdle called', this.activeJobs, this.queue.length)
+        return new Promise((resolve) => {
+            if (this.activeJobs === 0 && this.queue.length === 0) {
+                resolve();
+            }
+            else {
+                this.waitingResolvers.push(resolve);
+            }
+        });
+    }
+    /**
+     * Get all results accumulated so far
+     */
+    getResults() {
+        return this.results;
+    }
+    /**
+     * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
+     *
+     * Returns when a promise that resolves when all jobs have been processed and
+     * are available in `getResults()`.
+     */
+    async finishAllWork() {
+        this.isAcceptingWork = false;
+        // Wake up any sleeping workers so they can process remaining jobs or exit
+        this.notifyWorkersOfNewWork();
+        // Check if we're already idle and notify any waiting resolvers
+        await Promise.all(this.workers);
+        this.workers = [];
+    }
+    /**
+     * Get the current queue length
+     */
+    get queueLength() {
+        return this.queue.length;
+    }
+    /**
+     * Get the number of currently active jobs
+     */
+    get activeJobCount() {
+        return this.activeJobs;
+    }
+}
+exports.ConcurrentJobQueue = ConcurrentJobQueue;
+//# sourceMappingURL=ConcurrentJobQueue.js.map

package/dist/cjs/ConcurrentJobQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConcurrentJobQueue.js","sourceRoot":"","sources":["../../src/ConcurrentJobQueue.ts"],"names":[],"mappings":";;;AAAA,4CAAsE;AAGtE;;;;;;GAMG;AAEH,MAAa,kBAAkB;IAiBnB;IACA;IAjBF,KAAK,GAAgB,EAAE,CAAA;IACvB,OAAO,GAAsB,EAAE,CAAA;IAC/B,UAAU,GAAG,CAAC,CAAA;IACd,gBAAgB,GAAmB,EAAE,CAAA;IACrC,OAAO,GAAoB,EAAE,CAAA;IAC7B,eAAe,GAAG,IAAI,CAAA;IACtB,oBAAoB,GAAyB,IAAI,CAAA;IACjD,oBAAoB,GAAwB,IAAI,CAAA;IAExD;;;;;OAKG;IACH,YACU,WAAmB,EACnB,SAAiB,SAAa;QAD9B,gBAAW,GAAX,WAAW,CAAQ;QACnB,WAAM,GAAN,MAAM,CAAwB;IACrC,CAAC;IAEI,KAAK,CAAC,MAAM,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC1B,8DAA8D;oBAC9D,OAAM;gBACR,CAAC;gBACD,mDAAmD;gBACnD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAA;gBACjC,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAA;YACjB,MAAM,OAAO,GAAe,EAAE,QAAQ,EAAE,CAAA;YAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,4BAA4B,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,EAAE,EACzD,EAAE,GAAG,OAAO,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAClC,CAAA;YACH,CAAC,EAAE,MAAM,CAAC,CAAA;YAEV,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;gBAC3E,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YAC/E,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,QAAQ,CAAC,CAAA;gBACvB,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,IAAI,CAAC,WAAW,EAAE,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACxD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAA;YACrC,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAA;IAClC,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC5B,0BAA0B;QAC1B,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,EAAE,CAAA;YAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;YAChC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;QAClC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;YACrD,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAc;QACpB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpB,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,sBAAsB,EAAE,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAiB;QAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,WAAW;QACT,sEAAsE;QACtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACX,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrC,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;QAC5B,0EAA0E;QAC1E,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,+DAA+D;QAC/D,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF;AAjKD,gDAiKC"}

package/dist/cjs/ConcurrentWorkerPool.d.ts

@@ -0,0 +1,64 @@
+import { type Logger } from '@cloud-copilot/log';
+import { type Job, type JobResult } from './job.js';
+/**
+ * A worker pool that will run jobs concurrently using promises up to a specified limit.
+ * If no work is being done, it will wait for jobs to be added and run them up to the
+ * maximum concurrency.
+ *
+ * This is designed to create a central queue for jobs of many types to be processed
+ * in a streaming fashion, allowing for efficient resource usage and managed network requests.
+ *
+ */
+export declare class ConcurrentWorkerPool<T = void, P = Record<string, unknown>> {
+    private concurrency;
+    private logger;
+    private jobCounter;
+    private resolveMap;
+    private queue;
+    private activeJobs;
+    private waitingResolvers;
+    private workers;
+    private isAcceptingWork;
+    private workAvailablePromise;
+    private resolveWorkAvailable;
+    /**
+     * Create a new runner with the specified concurrency.
+     *
+     * @param concurrency - The maximum number of jobs to run concurrently.
+     * @param logger - Optional logger instance for long-running job warnings. Defaults to the module-level logger from @cloud-copilot/log.
+     */
+    constructor(concurrency: number, logger?: Logger);
+    private worker;
+    private waitForWorkAvailable;
+    private ensureWorkers;
+    private notifyWorkersOfNewWork;
+    private checkIfIdle;
+    /**
+     * Add a job to the queue
+     */
+    enqueue(job: Job<T, P>): Promise<JobResult<T, P>>;
+    /**
+     * Add multiple jobs to the queue
+     */
+    enqueueAll(jobs: Job<T, P>[]): Promise<JobResult<T, P>>[];
+    /**
+     * Returns a promise that resolves when all queued work is complete
+     */
+    waitForIdle(): Promise<void>;
+    /**
+     * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
+     *
+     * Returns when a promise that resolves when all jobs have been processed and
+     * the onComplete callback has been called for each job.
+     */
+    finishAllWork(): Promise<void>;
+    /**
+     * Get the current queue length
+     */
+    get queueLength(): number;
+    /**
+     * Get the number of currently active jobs
+     */
+    get activeJobCount(): number;
+}
+//# sourceMappingURL=ConcurrentWorkerPool.d.ts.map

package/dist/cjs/ConcurrentWorkerPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConcurrentWorkerPool.d.ts","sourceRoot":"","sources":["../../src/ConcurrentWorkerPool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwB,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAA;AACtE,OAAO,EAAE,KAAK,GAAG,EAAmB,KAAK,SAAS,EAAE,MAAM,UAAU,CAAA;AAMpE;;;;;;;;GAQG;AACH,qBAAa,oBAAoB,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAkBnE,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;IAlBhB,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,UAAU,CAAuD;IACzE,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAO;IAC9B,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,oBAAoB,CAA4B;IAExD;;;;;OAKG;gBAEO,WAAW,EAAE,MAAM,EACnB,MAAM,GAAE,MAAsB;YAG1B,MAAM;IAwCpB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAmBjD;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;IAIzD;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAU5B;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IASpC;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;CACF"}

package/dist/cjs/ConcurrentWorkerPool.js

@@ -0,0 +1,166 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConcurrentWorkerPool = void 0;
+const log_1 = require("@cloud-copilot/log");
+/**
+ * A worker pool that will run jobs concurrently using promises up to a specified limit.
+ * If no work is being done, it will wait for jobs to be added and run them up to the
+ * maximum concurrency.
+ *
+ * This is designed to create a central queue for jobs of many types to be processed
+ * in a streaming fashion, allowing for efficient resource usage and managed network requests.
+ *
+ */
+class ConcurrentWorkerPool {
+    concurrency;
+    logger;
+    jobCounter = 0;
+    resolveMap = {};
+    queue = [];
+    activeJobs = 0;
+    waitingResolvers = [];
+    workers = [];
+    isAcceptingWork = true;
+    workAvailablePromise = null;
+    resolveWorkAvailable = null;
+    /**
+     * Create a new runner with the specified concurrency.
+     *
+     * @param concurrency - The maximum number of jobs to run concurrently.
+     * @param logger - Optional logger instance for long-running job warnings. Defaults to the module-level logger from @cloud-copilot/log.
+     */
+    constructor(concurrency, logger = log_1.log) {
+        this.concurrency = concurrency;
+        this.logger = logger;
+    }
+    async worker(workerId) {
+        while (this.isAcceptingWork || this.queue.length > 0) {
+            const job = this.queue.shift();
+            if (!job) {
+                if (!this.isAcceptingWork) {
+                    // No longer accepting work and no jobs left, exit immediately
+                    return;
+                }
+                // No work available, wait for new work to be added
+                await this.waitForWorkAvailable();
+                continue;
+            }
+            this.activeJobs++;
+            const context = { workerId };
+            const startTime = Date.now();
+            const interval = setInterval(() => {
+                this.logger.warn(`Long-running job detected.`, { minutes: Math.floor((Date.now() - startTime) / 60000) }, { ...context, ...job.properties });
+            }, 60_000);
+            const resolve = this.resolveMap[job.concurrentJobId];
+            try {
+                const value = await job.execute({ ...context, properties: job.properties });
+                resolve({ status: 'fulfilled', value, properties: job.properties });
+            }
+            catch (reason) {
+                resolve({ status: 'rejected', reason, properties: job.properties });
+            }
+            finally {
+                clearInterval(interval);
+                this.activeJobs--;
+                this.checkIfIdle();
+                delete this.resolveMap[job.concurrentJobId];
+            }
+        }
+    }
+    waitForWorkAvailable() {
+        if (!this.workAvailablePromise) {
+            this.workAvailablePromise = new Promise((resolve) => {
+                this.resolveWorkAvailable = resolve;
+            });
+        }
+        return this.workAvailablePromise;
+    }
+    ensureWorkers() {
+        if (this.workers.length === 0 && this.isAcceptingWork) {
+            for (let i = 0; i < this.concurrency; i++) {
+                this.workers.push(this.worker(i + 1));
+            }
+        }
+    }
+    notifyWorkersOfNewWork() {
+        // Wake up waiting workers
+        if (this.resolveWorkAvailable) {
+            this.resolveWorkAvailable();
+            this.workAvailablePromise = null;
+            this.resolveWorkAvailable = null;
+        }
+    }
+    checkIfIdle() {
+        if (this.activeJobs === 0 && this.queue.length === 0) {
+            // Notify all waiting resolvers
+            this.waitingResolvers.forEach((resolve) => resolve());
+            this.waitingResolvers = [];
+        }
+    }
+    /**
+     * Add a job to the queue
+     */
+    enqueue(job) {
+        if (!this.isAcceptingWork) {
+            throw new Error('Cannot enqueue jobs after shutdown');
+        }
+        return new Promise((resolve) => {
+            const jobId = this.jobCounter++;
+            const jobWithId = {
+                ...job,
+                concurrentJobId: jobId
+            };
+            this.resolveMap[jobId] = resolve;
+            this.queue.push(jobWithId);
+            this.ensureWorkers();
+            this.notifyWorkersOfNewWork();
+        });
+    }
+    /**
+     * Add multiple jobs to the queue
+     */
+    enqueueAll(jobs) {
+        return jobs.map((job) => this.enqueue(job));
+    }
+    /**
+     * Returns a promise that resolves when all queued work is complete
+     */
+    waitForIdle() {
+        return new Promise((resolve) => {
+            if (this.activeJobs === 0 && this.queue.length === 0) {
+                resolve();
+            }
+            else {
+                this.waitingResolvers.push(resolve);
+            }
+        });
+    }
+    /**
+     * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
+     *
+     * Returns when a promise that resolves when all jobs have been processed and
+     * the onComplete callback has been called for each job.
+     */
+    async finishAllWork() {
+        this.isAcceptingWork = false;
+        // Wake up any sleeping workers so they can process remaining jobs or exit
+        this.notifyWorkersOfNewWork();
+        // Check if we're already idle and notify any waiting resolvers
+        await Promise.all(this.workers);
+        this.workers = [];
+    }
+    /**
+     * Get the current queue length
+     */
+    get queueLength() {
+        return this.queue.length;
+    }
+    /**
+     * Get the number of currently active jobs
+     */
+    get activeJobCount() {
+        return this.activeJobs;
+    }
+}
+exports.ConcurrentWorkerPool = ConcurrentWorkerPool;
+//# sourceMappingURL=ConcurrentWorkerPool.js.map

package/dist/cjs/ConcurrentWorkerPool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConcurrentWorkerPool.js","sourceRoot":"","sources":["../../src/ConcurrentWorkerPool.ts"],"names":[],"mappings":";;;AAAA,4CAAsE;AAOtE;;;;;;;;GAQG;AACH,MAAa,oBAAoB;IAkBrB;IACA;IAlBF,UAAU,GAAG,CAAC,CAAA;IACd,UAAU,GAAqD,EAAE,CAAA;IACjE,KAAK,GAAsB,EAAE,CAAA;IAC7B,UAAU,GAAG,CAAC,CAAA;IACd,gBAAgB,GAAmB,EAAE,CAAA;IACrC,OAAO,GAAoB,EAAE,CAAA;IAC7B,eAAe,GAAG,IAAI,CAAA;IACtB,oBAAoB,GAAyB,IAAI,CAAA;IACjD,oBAAoB,GAAwB,IAAI,CAAA;IAExD;;;;;OAKG;IACH,YACU,WAAmB,EACnB,SAAiB,SAAa;QAD9B,gBAAW,GAAX,WAAW,CAAQ;QACnB,WAAM,GAAN,MAAM,CAAwB;IACrC,CAAC;IAEI,KAAK,CAAC,MAAM,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC1B,8DAA8D;oBAC9D,OAAM;gBACR,CAAC;gBACD,mDAAmD;gBACnD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAA;gBACjC,SAAQ;YACV,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAA;YACjB,MAAM,OAAO,GAAe,EAAE,QAAQ,EAAE,CAAA;YAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,4BAA4B,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,EAAE,EACzD,EAAE,GAAG,OAAO,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAClC,CAAA;YACH,CAAC,EAAE,MAAM,CAAC,CAAA;YAEV,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;YACpD,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;gBAC3E,OAAO,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YACrE,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,OAAO,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YACrE,CAAC;oBAAS,CAAC;gBACT,aAAa,CAAC,QAAQ,CAAC,CAAA;gBACvB,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,IAAI,CAAC,WAAW,EAAE,CAAA;gBAClB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBACxD,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAA;YACrC,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAA;IAClC,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,sBAAsB;QAC5B,0BAA0B;QAC1B,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,EAAE,CAAA;YAC3B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;YAChC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAA;QAClC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,+BAA+B;YAC/B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;YACrD,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,GAAc;QACpB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QAED,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,EAAE;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,EAAE,CAAA;YAC/B,MAAM,SAAS,GAAoB;gBACjC,GAAG,GAAG;gBACN,eAAe,EAAE,KAAK;aACvB,CAAA;YACD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,OAAO,CAAA;YAEhC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC1B,IAAI,CAAC,aAAa,EAAE,CAAA;YACpB,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC/B,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAiB;QAC1B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;IAC7C,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACX,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrC,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;QAC5B,0EAA0E;QAC1E,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,+DAA+D;QAC/D,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,UAAU,CAAA;IACxB,CAAC;CACF;AAtKD,oDAsKC"}

package/dist/cjs/StreamingJobQueue.d.ts

@@ -0,0 +1,63 @@
+import { type Logger } from '@cloud-copilot/log';
+import { type Job, type JobResult } from './job.js';
+/**
+ * A queue that will run jobs concurrently using promises up to a specified limit.
+ * If no work is being done, it will wait for jobs to be added and run them up to the
+ * maximum concurrency.
+ *
+ * Results are not stored in this queue, but are processed immediately via the `onComplete` callback.
+ *
+ */
+export declare class StreamingJobQueue<T = void, P = Record<string, unknown>> {
+    private concurrency;
+    private logger;
+    private onComplete;
+    private queue;
+    private activeJobs;
+    private waitingResolvers;
+    private workers;
+    private isAcceptingWork;
+    private workAvailablePromise;
+    private resolveWorkAvailable;
+    /**
+     * Create a new runner with the specified concurrency.
+     *
+     * @param concurrency - The maximum number of jobs to run concurrently.
+     * @param logger - Logger instance for long-running job warnings.
+     * @param onComplete - Callback to handle job completion, receives the job result.
+     */
+    constructor(concurrency: number, logger: Logger, onComplete: (response: JobResult<T, P>) => Promise<void>);
+    private worker;
+    private waitForWorkAvailable;
+    private ensureWorkers;
+    private notifyWorkersOfNewWork;
+    private checkIfIdle;
+    /**
+     * Add a job to the queue
+     */
+    enqueue(job: Job<T, P>): void;
+    /**
+     * Add multiple jobs to the queue
+     */
+    enqueueAll(jobs: Job<T, P>[]): void;
+    /**
+     * Returns a promise that resolves when all queued work is complete
+     */
+    waitForIdle(): Promise<void>;
+    /**
+     * Shutdown the queue - no new jobs will be accepted, but existing jobs will complete.
+     *
+     * Returns when a promise that resolves when all jobs have been processed and
+     * the onComplete callback has been called for each job.
+     */
+    finishAllWork(): Promise<void>;
+    /**
+     * Get the current queue length
+     */
+    get queueLength(): number;
+    /**
+     * Get the number of currently active jobs
+     */
+    get activeJobCount(): number;
+}
+//# sourceMappingURL=StreamingJobQueue.d.ts.map

package/dist/cjs/StreamingJobQueue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StreamingJobQueue.d.ts","sourceRoot":"","sources":["../../src/StreamingJobQueue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAA;AAChD,OAAO,EAAE,KAAK,GAAG,EAAmB,KAAK,SAAS,EAAE,MAAM,UAAU,CAAA;AAEpE;;;;;;;GAOG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAiBhE,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,UAAU;IAlBpB,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAO;IAC9B,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,oBAAoB,CAA4B;IAExD;;;;;;OAMG;gBAEO,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC;YAGpD,MAAM;IAwCpB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI;IAS7B;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI;IAInC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAU5B;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IASpC;;OAEG;IACH,IAAI,WAAW,IAAI,MAAM,CAExB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;CACF"}