@decaf-ts/core 0.8.50 → 0.8.52

package/lib/tasks/TaskEngine.cjs

@@ -11,6 +11,7 @@ const TaskStepResultModel_1 = require("./models/TaskStepResultModel.cjs");
 const TaskLogEntryModel_1 = require("./models/TaskLogEntryModel.cjs");
 const TaskBackoffModel_1 = require("./models/TaskBackoffModel.cjs");
 const TaskStepSpecModel_1 = require("./models/TaskStepSpecModel.cjs");
+const node_worker_threads_1 = require("node:worker_threads");
 const constants_1 = require("./constants.cjs");
 const logging_1 = require("@decaf-ts/logging");
 const ContextualLoggedClass_1 = require("./../utils/ContextualLoggedClass.cjs");
@@ -23,6 +24,7 @@ const TaskErrorModel_1 = require("./models/TaskErrorModel.cjs");
 const TaskTracker_1 = require("./TaskTracker.cjs");
 const transactional_decorators_1 = require("@decaf-ts/transactional-decorators");
 const index_1 = require("./../persistence/index.cjs");
+const WorkThreadEnvironment_1 = require("./workers/WorkThreadEnvironment.cjs");
 class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
     get Context() {
         return TaskContext_1.TaskContext;
@@ -52,15 +54,87 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
             this._events = this._events.override(this.config.overrides);
         return this._events;
     }
+    static createTaskContext(base, overrides) {
+        const ctx = new TaskContext_1.TaskContext(base);
+        if (overrides && Object.keys(overrides).length) {
+            return ctx.accumulate(overrides);
+        }
+        return ctx;
+    }
     constructor(config) {
         super();
         this.config = config;
+        this.workerThreads = [];
+        this.workerJobQueue = [];
+        this.workerJobs = new Map();
+        this.workerCounter = 0;
+        this.workerThreadCapacity = 1;
         this.lock = new transactional_decorators_1.Lock();
         this.running = false;
+        if (config.workerPool && !config.workerAdapter) {
+            throw new db_decorators_1.InternalError("Worker pool requires workerAdapter descriptor in TaskEngineConfig");
+        }
         this.config = Object.assign({}, constants_1.DefaultTaskEngineConfig, config, {
             bus: config.bus || new TaskEventBus_1.TaskEventBus(),
             registry: config.registry || new TaskHandlerRegistry_1.TaskHandlerRegistry(),
         });
+        this.workerThreadCapacity = Math.max(1, this.config.workerConcurrency ?? 1);
+        this.workerPoolConfig = this.normalizeWorkerPoolConfig(this.config.workerPool);
+    }
+    normalizeWorkerPoolConfig(pool) {
+        if (!pool)
+            return undefined;
+        if (!pool.entry) {
+            throw new db_decorators_1.InternalError("Worker pool configuration requires an explicit entry file path");
+        }
+        if (pool.size != null && pool.size !== this.config.concurrency) {
+            throw new db_decorators_1.InternalError("TaskEngine concurrency must match workerPool.size when worker pool is enabled");
+        }
+        return Object.assign({}, pool, {
+            size: this.config.concurrency,
+        });
+    }
+    hasWorkerPool() {
+        return !!this.workerPoolConfig && (this.workerPoolConfig.size ?? 0) > 0;
+    }
+    getWorkerCount() {
+        if (!this.workerPoolConfig)
+            return 0;
+        return this.workerPoolConfig.size ?? 0;
+    }
+    getWorkerExecutionSlots() {
+        return this.getWorkerCount() * this.workerThreadCapacity;
+    }
+    canDispatchToWorkers() {
+        return this.hasWorkerPool();
+    }
+    getExecutionConcurrency() {
+        if (this.hasWorkerPool()) {
+            return Math.max(1, this.getWorkerExecutionSlots());
+        }
+        return this.config.concurrency;
+    }
+    computeWorkerModules() {
+        const adapterDescriptor = this.config.workerAdapter ?? WorkThreadEnvironment_1.DefaultWorkThreadEnvironment.persistence;
+        if (!adapterDescriptor?.adapterModule) {
+            throw new db_decorators_1.InternalError("Worker adapter descriptor must include adapterModule");
+        }
+        const configuredImports = this.workerPoolConfig?.modules?.imports ??
+            WorkThreadEnvironment_1.DefaultWorkThreadEnvironment.modules.imports;
+        const imports = [];
+        const append = (specifier) => {
+            if (!specifier)
+                return;
+            if (!imports.includes(specifier))
+                imports.push(specifier);
+        };
+        append(adapterDescriptor.adapterModule);
+        for (const specifier of configuredImports) {
+            if (specifier === adapterDescriptor.adapterModule)
+                continue;
+            append(specifier);
+        }
+        return { imports };
     }
     async push(task, track = false, ...args) {
         const { ctx, log } = (await this.logCtx(args, db_decorators_1.OperationKeys.CREATE, true)).for(this.push);
@@ -142,10 +216,13 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
     async start(...args) {
         const { ctx } = (await this.logCtx(args, "run", true)).for(this.start);
         await this.lock.acquire();
-        if (this.running)
+        if (this.running) {
+            this.lock.release();
             return;
+        }
         this.running = true;
         this.lock.release();
+        await this.spawnWorkers();
         void this.loop(ctx);
     }
     async stop(...args) {
@@ -161,7 +238,7 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
             .execute(ctx);
         const timeout = ctx.getOrUndefined?.("gracefulShutdownMsTimeout") ??
             this.config.gracefulShutdownMsTimeout;
-        return new Promise((resolve, reject) => {
+        await new Promise((resolve, reject) => {
             const timer = setTimeout(() => {
                 log.error(`Graceful shutdown interrupted after ${timeout} ms...`);
                 resolve();
@@ -183,6 +260,249 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
                 reject(err);
             });
         });
+        await this.shutdownWorkers();
+    }
+    // -------------------------
+    // Worker pool orchestration
+    // -------------------------
+    async spawnWorkers() {
+        if (!this.hasWorkerPool())
+            return;
+        const target = this.getWorkerCount();
+        const creations = [];
+        while (this.workerThreads.length < target) {
+            const ready = this.createWorker();
+            if (ready)
+                creations.push(ready);
+        }
+        if (creations.length) {
+            await Promise.all(creations);
+        }
+    }
+    createWorker() {
+        if (!this.workerPoolConfig)
+            return undefined;
+        let resolveReady;
+        let rejectReady;
+        const readyPromise = new Promise((resolve, reject) => {
+            resolveReady = resolve;
+            rejectReady = reject;
+        });
+        const entry = this.workerPoolConfig.entry;
+        const workerId = `${this.config.workerId}-${this.workerCounter++}`;
+        const env = {
+            workerId,
+            mode: this.workerPoolConfig.mode ?? "node",
+            persistence: Object.assign({}, this.config.workerAdapter ?? WorkThreadEnvironment_1.DefaultWorkThreadEnvironment.persistence, {
+                alias: this.adapter.alias,
+                flavour: this.adapter.flavour,
+            }),
+            taskEngine: {
+                concurrency: this.workerThreadCapacity,
+                leaseMs: this.config.leaseMs,
+                pollMsBusy: this.config.pollMsBusy,
+                pollMsIdle: this.config.pollMsIdle,
+                logTailMax: this.config.logTailMax,
+                streamBufferSize: this.config.streamBufferSize,
+                maxLoggingBuffer: this.config.maxLoggingBuffer,
+                loggingBufferTruncation: this.config.loggingBufferTruncation,
+                gracefulShutdownMsTimeout: this.config.gracefulShutdownMsTimeout,
+            },
+            modules: this.computeWorkerModules(),
+        };
+        const worker = new node_worker_threads_1.Worker(entry, {
+            workerData: { environment: env },
+        });
+        const state = {
+            id: workerId,
+            worker,
+            ready: false,
+            activeJobs: 0,
+            capacity: this.workerThreadCapacity,
+            readyPromise,
+            resolveReady,
+            rejectReady,
+        };
+        this.workerThreads.push(state);
+        worker.on("message", (msg) => this.handleWorkerMessage(state, msg));
+        worker.on("error", (err) => this.handleWorkerError(state, err));
+        worker.on("exit", (code) => this.handleWorkerExit(state, code));
+        return readyPromise;
+    }
+    async shutdownWorkers() {
+        for (const state of this.workerThreads.splice(0)) {
+            const message = {
+                type: "control",
+                command: "shutdown",
+            };
+            try {
+                state.worker.postMessage(message);
+            }
+            catch {
+                // ignore
+            }
+            await state.worker.terminate();
+        }
+        for (const job of this.workerJobQueue.splice(0)) {
+            job.reject(new db_decorators_1.InternalError(`Worker pool shutting down before job ${job.id} could start`));
+        }
+        for (const job of this.workerJobs.values()) {
+            job.reject(new db_decorators_1.InternalError(`Worker terminated before finishing job ${job.id}`));
+        }
+        this.workerJobs.clear();
+    }
+    handleWorkerError(state, err) {
+        this.log.error(`worker ${state.id} error: ${err.message}`, err);
+        if (state.rejectReady) {
+            state.rejectReady(err);
+            state.resolveReady = undefined;
+            state.rejectReady = undefined;
+        }
+    }
+    handleWorkerExit(state, code) {
+        this.log.info(`worker ${state.id} exited with code ${code}`);
+        if (state.rejectReady) {
+            state.rejectReady(new Error(`worker ${state.id} exited before reporting ready`));
+            state.resolveReady = undefined;
+            state.rejectReady = undefined;
+        }
+        const idx = this.workerThreads.indexOf(state);
+        if (idx >= 0)
+            this.workerThreads.splice(idx, 1);
+        for (const [jobId, job] of this.workerJobs.entries()) {
+            if (job.worker === state) {
+                job.worker = undefined;
+                this.workerJobs.delete(jobId);
+                this.workerJobQueue.unshift(job);
+            }
+        }
+        if (this.running) {
+            void this.spawnWorkers().catch((err) => this.log.error(`failed to respawn worker`, err));
+            this.processWorkerQueue();
+        }
+    }
+    handleWorkerMessage(state, msg) {
+        if (msg.type === "ready") {
+            state.ready = true;
+            if (state.resolveReady) {
+                state.resolveReady();
+                state.resolveReady = undefined;
+                state.rejectReady = undefined;
+            }
+            this.log.info(`worker ${state.id} ready`);
+            this.processWorkerQueue();
+            return;
+        }
+        if (msg.type === "error") {
+            this.log.error(`worker ${state.id} reported error: ${msg.error}`);
+            return;
+        }
+        if (msg.type === "log") {
+            const job = this.workerJobs.get(msg.jobId);
+            if (!job)
+                return;
+            void this.appendLog(job.ctx, job.task, msg.entries)
+                .then(([updated, entries]) => {
+                job.task = updated;
+                return this.emitLog(job.ctx, job.task.id, entries);
+            })
+                .catch((err) => this.log.error(`Failed to append worker log`, err));
+            return;
+        }
+        if (msg.type === "progress") {
+            const job = this.workerJobs.get(msg.jobId);
+            if (!job)
+                return;
+            void this.emitProgress(job.ctx, job.task.id, msg.payload);
+            return;
+        }
+        if (msg.type === "heartbeat") {
+            const job = this.workerJobs.get(msg.jobId);
+            if (!job)
+                return;
+            job.task.leaseExpiry = new Date(Date.now() + this.config.leaseMs);
+            void this.tasks.update(job.task).catch(() => null);
+            return;
+        }
+        if (msg.type === "result") {
+            const job = this.workerJobs.get(msg.jobId);
+            if (!job)
+                return;
+            this.workerJobs.delete(job.id);
+            state.activeJobs = Math.max(0, state.activeJobs - 1);
+            this.applyWorkerCache(job.ctx, msg.cache);
+            switch (msg.status) {
+                case "success":
+                    job.resolve(msg.output);
+                    break;
+                case "error": {
+                    const err = new Error(msg.error.message);
+                    if (msg.error.name)
+                        err.name = msg.error.name;
+                    if (msg.error.stack)
+                        err.stack = msg.error.stack;
+                    job.reject(err);
+                    break;
+                }
+                case "state-change":
+                    job.reject(new TaskStateChangeError_1.TaskStateChangeError(msg.request));
+                    break;
+            }
+            this.processWorkerQueue();
+            return;
+        }
+    }
+    applyWorkerCache(ctx, cache) {
+        if (!cache)
+            return;
+        Object.entries(cache).forEach(([key, value]) => {
+            ctx.cacheResult(key, value);
+        });
+    }
+    processWorkerQueue() {
+        if (!this.hasWorkerPool())
+            return;
+        const available = this.workerThreads
+            .filter((state) => state.ready && state.activeJobs < state.capacity)
+            .sort((a, b) => a.activeJobs - b.activeJobs);
+        for (const state of available) {
+            while (state.activeJobs < state.capacity &&
+                this.workerJobQueue.length > 0) {
+                const job = this.workerJobQueue.shift();
+                if (!job)
+                    break;
+                this.assignWorker(state, job);
+            }
+            if (!this.workerJobQueue.length)
+                break;
+        }
+    }
+    assignWorker(state, job) {
+        if (!this.workerPoolConfig)
+            return;
+        const payload = {
+            jobId: job.id,
+            taskId: job.task.id,
+            classification: job.classification,
+            input: job.input,
+            attempt: job.task.attempt ?? 0,
+            resultCache: job.ctx.resultCache ?? {},
+            streamBufferSize: this.config.streamBufferSize,
+            maxLoggingBuffer: this.config.maxLoggingBuffer,
+            loggingBufferTruncation: this.config.loggingBufferTruncation,
+        };
+        job.worker = state;
+        this.workerJobs.set(job.id, job);
+        state.activeJobs += 1;
+        const message = {
+            type: "execute",
+            job: payload,
+        };
+        state.worker.postMessage(message);
+    }
+    enqueueWorkerJob(job) {
+        this.workerJobQueue.push(job);
+        this.processWorkerQueue();
     }
     // -------------------------
     // Worker loop
@@ -217,10 +537,11 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
             .or(condLeaseExpired)
             .or(condScheduled);
         // Fetch more than concurrency because some will fail to claim due to conflicts
+        const concurrency = this.getExecutionConcurrency();
         const candidates = await this.tasks
             .select()
             .where(runnable)
-            .limit(Math.max(this.config.concurrency * 4, 20))
+            .limit(Math.max(concurrency * 4, 20))
             .execute();
         log.verbose(`claimBatch candidates:${candidates.length}`);
         const out = [];
@@ -228,7 +549,7 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
             const claimed = await this.tryClaim(c, ctx);
             if (claimed)
                 out.push(claimed);
-            if (out.length >= this.config.concurrency)
+            if (out.length >= concurrency)
                 break;
         }
         log.verbose(`claimBatch claimed:${out.length}`);
@@ -264,9 +585,39 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
     // -------------------------
     // Execution
     // -------------------------
+    async runHandlerInline(classification, input, ctx) {
+        const handler = this.registry.get(classification);
+        if (!handler)
+            throw new db_decorators_1.InternalError(`No task handler registered for type: ${classification}`);
+        return handler.run(input, ctx);
+    }
+    async dispatchToWorker(classification, input, task, ctx) {
+        if (!this.canDispatchToWorkers()) {
+            return this.runHandlerInline(classification, input, ctx);
+        }
+        const uuid = await index_1.UUID.instance.generate();
+        return new Promise((resolve, reject) => {
+            const job = {
+                id: uuid,
+                classification,
+                input,
+                task,
+                ctx,
+                resolve,
+                reject,
+            };
+            this.enqueueWorkerJob(job);
+        });
+    }
+    async invokeHandler(classification, input, task, ctx) {
+        if (!this.hasWorkerPool()) {
+            return this.runHandlerInline(classification, input, ctx);
+        }
+        return this.dispatchToWorker(classification, input, task, ctx);
+    }
     async executeClaimed(task) {
         const { ctx, log } = (await this.logCtx([], task.classification, true)).for(this.executeClaimed);
-        const taskCtx = new TaskContext_1.TaskContext(ctx).accumulate({
+        const taskCtx = TaskEngine.createTaskContext(ctx, {
             taskId: task.id,
             logger: new logging_2.TaskLogger(log, this.config.streamBufferSize, this.config.maxLoggingBuffer),
             attempt: task.attempt,
@@ -276,7 +627,7 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
                 await this.emitLog(taskCtx, task.id, logs);
             },
             flush: async () => {
-                return taskCtx.logger.flush(taskCtx.pipe);
+                await taskCtx.logger.flush(taskCtx.pipe);
             },
             progress: async (data) => {
                 await this.emitProgress(taskCtx, task.id, data);
@@ -311,11 +662,8 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
                 }
             }
             else {
-                const handler = this.registry.get(task.classification);
-                log.debug(`handler type for ${task.id} is ${handler?.constructor?.name ?? "none"}`);
-                if (!handler)
-                    throw new db_decorators_1.InternalError(`No task handler registered for type: ${task.classification}`);
-                output = await handler.run(task.input, taskCtx);
+                log.debug(`dispatching handler for ${task.id}`);
+                output = await this.invokeHandler(task.classification, task.input, task, taskCtx);
                 log.verbose(`handler finished for ${task.id}`);
             }
             task.status = constants_1.TaskStatus.SUCCEEDED;
@@ -473,15 +821,12 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
         }
         while (idx < steps.length) {
             const step = steps[idx];
-            const handler = this.registry.get(step.classification);
-            if (!handler)
-                throw new Error(`No task handler registered for composite step: ${step.classification}`);
             await context.pipe([
                 logging_1.LogLevel.info,
                 `Composite step ${idx + 1}/${steps.length}: ${step.classification}`,
             ]);
             try {
-                const out = await handler.run(step.input, context);
+                const out = await this.invokeHandler(step.classification, step.input, task, context);
                 const stepIndex = idx;
                 const now = new Date();
                 results[stepIndex] = new TaskStepResultModel_1.TaskStepResultModel({

package/lib/tasks/TaskEngine.d.ts

@@ -4,17 +4,25 @@ import { TaskEventModel } from "./models/TaskEventModel";
 import { TaskHandlerRegistry } from "./TaskHandlerRegistry";
 import { TaskEventBus } from "./TaskEventBus";
 import { Adapter } from "../persistence/Adapter";
+import { Context } from "../persistence/Context";
 import { ContextOf, FlagsOf } from "../persistence/types";
 import { AbsContextual, MaybeContextualArg } from "../utils/ContextualLoggedClass";
 import { OperationKeys } from "@decaf-ts/db-decorators";
+import { TaskContext } from "./TaskContext";
 import { DateTarget } from "@decaf-ts/decorator-validation";
 import { Constructor } from "@decaf-ts/decoration";
-import { TaskEngineConfig } from "./types";
+import { TaskEngineConfig, TaskFlags } from "./types";
 import { TaskTracker } from "./TaskTracker";
 export declare class TaskEngine<A extends Adapter<any, any, any, any>> extends AbsContextual<ContextOf<A>> {
     private config;
     private _tasks?;
     private _events?;
+    private workerPoolConfig?;
+    private workerThreads;
+    private workerJobQueue;
+    private workerJobs;
+    private workerCounter;
+    private workerThreadCapacity;
     private lock;
     protected get Context(): Constructor<ContextOf<A>>;
     protected get adapter(): A;
@@ -23,7 +31,15 @@ export declare class TaskEngine<A extends Adapter<any, any, any, any>> extends A
     protected get tasks(): Repo<TaskModel>;
     protected get events(): Repo<TaskEventModel>;
     protected running: boolean;
+    static createTaskContext(base?: Context<any>, overrides?: Partial<TaskFlags>): TaskContext;
     constructor(config: TaskEngineConfig<A>);
+    private normalizeWorkerPoolConfig;
+    private hasWorkerPool;
+    private getWorkerCount;
+    private getWorkerExecutionSlots;
+    private canDispatchToWorkers;
+    private getExecutionConcurrency;
+    private computeWorkerModules;
     push<I, O>(task: TaskModel<I, O>, ...args: MaybeContextualArg<any>): Promise<TaskModel<I, O>>;
     push<I, O>(task: TaskModel<I, O>, track: false, ...args: MaybeContextualArg<any>): Promise<TaskModel<I, O>>;
     push<I, O>(task: TaskModel<I, O>, track: true, ...args: MaybeContextualArg<any>): Promise<{
@@ -51,9 +67,22 @@ export declare class TaskEngine<A extends Adapter<any, any, any, any>> extends A
     isRunning(): Promise<boolean>;
     start(...args: MaybeContextualArg<any>): Promise<void>;
     stop(...args: MaybeContextualArg<any>): Promise<void>;
+    private spawnWorkers;
+    private createWorker;
+    private shutdownWorkers;
+    private handleWorkerError;
+    private handleWorkerExit;
+    private handleWorkerMessage;
+    private applyWorkerCache;
+    private processWorkerQueue;
+    private assignWorker;
+    private enqueueWorkerJob;
     private loop;
     private claimBatch;
     private tryClaim;
+    private runHandlerInline;
+    private dispatchToWorker;
+    private invokeHandler;
     private executeClaimed;
     private handleTaskStateChange;
     private runComposite;