@supergrowthai/tq 1.0.13 → 1.1.1

package/dist/index.mjs

@@ -48,7 +48,7 @@ import { getEnvironmentQueueName } from "@supergrowthai/mq";
 import { L as Logger, a as LogLevel } from "./client-dvHNt8qU.js";
 import { tId } from "./utils/task-id-gen.mjs";
 import { Actions } from "./core/Actions.mjs";
-import { b as buildProjection, s as syncProjections, A as AsyncActions } from "./AsyncActions-CZYO8ShR.js";
+import { b as buildProjection, s as syncProjections, A as AsyncActions } from "./AsyncActions-B8ImDgTo.js";
 import { AsyncLocalStorage } from "node:async_hooks";
 import moment from "moment";
 import * as os from "os";
@@ -663,7 +663,22 @@ function getLogContext() {
   return als.getStore();
 }
 class TaskRunner {
-  constructor(messageQueue, taskQueue, taskStore, cacheProvider, generateId, lifecycleProvider, lifecycleConfig, entityProjection, entityProjectionConfig, flowMiddleware) {
+  constructor(opts) {
+    this.taskStartTimes = /* @__PURE__ */ new Map();
+    const {
+      messageQueue,
+      taskQueue,
+      taskStore,
+      cacheProvider,
+      generateId,
+      lifecycleProvider,
+      lifecycleConfig,
+      entityProjection,
+      entityProjectionConfig,
+      flowMiddleware,
+      flowLifecycleProvider,
+      workerId = ""
+    } = opts;
     this.messageQueue = messageQueue;
     this.taskQueue = taskQueue;
     this.taskStore = taskStore;
@@ -673,7 +688,8 @@ class TaskRunner {
     this.entityProjection = entityProjection;
     this.entityProjectionConfig = entityProjectionConfig;
     this.flowMiddleware = flowMiddleware;
-    this.taskStartTimes = /* @__PURE__ */ new Map();
+    this.flowLifecycleProvider = flowLifecycleProvider;
+    this.workerId = workerId;
     this.logger = new Logger("TaskRunner", LogLevel.INFO);
     this.lockManager = new LockManager(cacheProvider, {
       prefix: "task_lock_",
@@ -740,7 +756,7 @@ class TaskRunner {
           this.logger.error(`[TQ] Entity projection failed (non-fatal): ${err}`);
         }
       }
-      const actions = new Actions(taskRunnerId);
+      const actions = new Actions(taskRunnerId, this.flowLifecycleProvider, this.workerId);
       const asyncTasks = [];
       const processedTaskIds = /* @__PURE__ */ new Set();
       for (let i = 0; i < groupedTasksArray.length; i++) {
@@ -776,7 +792,20 @@ class TaskRunner {
         taskGroup.tasks.forEach((task) => processedTaskIds.add(tId(task)));
         this.logger.info(`[${taskRunnerId}] Processing ${taskGroup.tasks.length} tasks of type: ${taskGroup.type}`);
         if (executor.multiple) {
-          const batchStore = this.buildBatchLogStore(taskGroup.tasks, taskRunnerId);
+          const batchStore = this.buildBatchLogStore(taskGroup.tasks, this.workerId);
+          const batchTaskContexts = taskGroup.tasks.map((t) => this.buildTaskContext(t, this.workerId, taskRunnerId));
+          const batchStartedAt = Date.now();
+          this.emitLifecycleEvent(
+            this.lifecycleProvider?.onTaskBatchStarted,
+            {
+              task_type: taskGroup.type,
+              queue_id: firstTask.queue_id,
+              tasks: batchTaskContexts,
+              worker_id: this.workerId,
+              consumer_id: taskRunnerId,
+              started_at: new Date(batchStartedAt)
+            }
+          );
           await runWithLogContext(
             batchStore,
             () => executor.onTasks(taskGroup.tasks, actions).catch((err) => {
@@ -788,19 +817,40 @@ class TaskRunner {
               }
             })
           );
+          const succeeded = [];
+          const failed = [];
+          for (const task of taskGroup.tasks) {
+            const status = actions.getTaskResultStatus(tId(task));
+            if (status === "success") succeeded.push(tId(task));
+            else if (status === "fail") failed.push(tId(task));
+            else failed.push(tId(task));
+          }
+          this.emitLifecycleEvent(
+            this.lifecycleProvider?.onTaskBatchCompleted,
+            {
+              task_type: taskGroup.type,
+              queue_id: firstTask.queue_id,
+              tasks: batchTaskContexts,
+              worker_id: this.workerId,
+              consumer_id: taskRunnerId,
+              succeeded,
+              failed,
+              duration_ms: Date.now() - batchStartedAt
+            }
+          );
         } else {
           if (executor.parallel) {
             const chunks = chunk(taskGroup.tasks, executor.chunkSize);
             this.logger.info(`[${taskRunnerId}] Processing in parallel chunks of ${executor.chunkSize}`);
             for (const taskChunk of chunks) {
               for (const task of taskChunk) {
-                this.emitTaskStarted(task, taskRunnerId);
+                this.emitTaskStarted(task, this.workerId, taskRunnerId);
               }
               const chunkPromises = [];
               for (let j = 0; j < taskChunk.length; j++) {
                 const task = taskChunk[j];
                 const taskActions = actions.forkForTask(task);
-                const logStore = this.buildLogStore(task, taskRunnerId);
+                const logStore = this.buildLogStore(task, this.workerId);
                 chunkPromises.push(runWithLogContext(
                   logStore,
                   () => executor.onTask(task, taskActions).catch((err) => {
@@ -815,12 +865,12 @@ class TaskRunner {
               for (const task of taskChunk) {
                 const resultStatus = actions.getTaskResultStatus(tId(task));
                 if (resultStatus === "success") {
-                  this.emitTaskCompleted(task, taskRunnerId, actions.getTaskResult(tId(task)));
+                  this.emitTaskCompleted(task, this.workerId, actions.getTaskResult(tId(task)), taskRunnerId);
                 } else if (resultStatus === "fail") {
                   const retryCount = task.execution_stats?.retry_count || 0;
                   const maxRetries = task.retries ?? executor.default_retries ?? 0;
                   const willRetry = retryCount < maxRetries;
-                  this.emitTaskFailed(task, taskRunnerId, new Error("Task failed"), willRetry);
+                  this.emitTaskFailed(task, this.workerId, new Error("Task failed"), willRetry, void 0, taskRunnerId);
                 }
               }
             }
@@ -829,9 +879,9 @@ class TaskRunner {
             for (let j = 0; j < taskGroup.tasks.length; j++) {
               const task = taskGroup.tasks[j];
               if (!timeoutMs) {
-                this.emitTaskStarted(task, taskRunnerId);
+                this.emitTaskStarted(task, this.workerId, taskRunnerId);
                 const taskActions = actions.forkForTask(task);
-                const logStore = this.buildLogStore(task, taskRunnerId);
+                const logStore = this.buildLogStore(task, this.workerId);
                 await runWithLogContext(
                   logStore,
                   () => executor.onTask(task, taskActions).catch((err) => {
@@ -843,18 +893,18 @@ class TaskRunner {
                 );
                 const resultStatus = actions.getTaskResultStatus(tId(task));
                 if (resultStatus === "success") {
-                  this.emitTaskCompleted(task, taskRunnerId, actions.getTaskResult(tId(task)));
+                  this.emitTaskCompleted(task, this.workerId, actions.getTaskResult(tId(task)), taskRunnerId);
                 } else if (resultStatus === "fail") {
                   const retryCount = task.execution_stats?.retry_count || 0;
                   const maxRetries = task.retries ?? executor.default_retries ?? 0;
                   const willRetry = retryCount < maxRetries;
-                  this.emitTaskFailed(task, taskRunnerId, new Error("Task failed"), willRetry);
+                  this.emitTaskFailed(task, this.workerId, new Error("Task failed"), willRetry, void 0, taskRunnerId);
                 }
               } else {
-                this.emitTaskStarted(task, taskRunnerId);
+                this.emitTaskStarted(task, this.workerId, taskRunnerId);
                 const startTime = Date.now();
                 const taskActions = actions.forkForTask(task);
-                const logStore = this.buildLogStore(task, taskRunnerId);
+                const logStore = this.buildLogStore(task, this.workerId);
                 const taskPromise = runWithLogContext(
                   logStore,
                   () => executor.onTask(task, taskActions).catch((err) => {
@@ -877,12 +927,12 @@ class TaskRunner {
                 if (result !== "~~~timeout") {
                   const resultStatus = actions.getTaskResultStatus(tId(task));
                   if (resultStatus === "success") {
-                    this.emitTaskCompleted(task, taskRunnerId, actions.getTaskResult(tId(task)));
+                    this.emitTaskCompleted(task, this.workerId, actions.getTaskResult(tId(task)), taskRunnerId);
                   } else if (resultStatus === "fail") {
                     const retryCount = task.execution_stats?.retry_count || 0;
                     const maxRetries = task.retries ?? executor.default_retries ?? 0;
                     const willRetry = retryCount < maxRetries;
-                    this.emitTaskFailed(task, taskRunnerId, new Error("Task failed"), willRetry);
+                    this.emitTaskFailed(task, this.workerId, new Error("Task failed"), willRetry, void 0, taskRunnerId);
                   }
                 }
                 if (result === "~~~timeout") {
@@ -895,10 +945,18 @@ class TaskRunner {
                   } else {
                     const asyncLifecycleEmitter = this.lifecycleProvider ? {
                       onCompleted: (t, result2) => {
-                        this.emitTaskCompleted(t, taskRunnerId, result2);
+                        this.emitTaskCompleted(t, this.workerId, result2, taskRunnerId);
                       },
                       onFailed: (t, error, willRetry) => {
-                        this.emitTaskFailed(t, taskRunnerId, error, willRetry);
+                        this.emitTaskFailed(t, this.workerId, error, willRetry, void 0, taskRunnerId);
+                      },
+                      onScheduled: (t) => {
+                        if (this.lifecycleProvider?.onTaskScheduled) {
+                          this.emitLifecycleEvent(
+                            this.lifecycleProvider.onTaskScheduled,
+                            this.buildTaskContext(t, this.workerId, taskRunnerId)
+                          );
+                        }
                       }
                     } : void 0;
                     const asyncActions = new AsyncActions(this.messageQueue, this.taskStore, this.taskQueue, actions, task, this.generateId, asyncLifecycleEmitter, this.entityProjection, this.entityProjectionConfig, this.flowMiddleware);
@@ -946,7 +1004,7 @@ class TaskRunner {
       this.logger.error(`[TQ] Lifecycle callback error: ${err}`);
     }
   }
-  buildTaskContext(task, workerId) {
+  buildTaskContext(task, workerId, consumerId) {
     const retryCount = task.execution_stats && typeof task.execution_stats.retry_count === "number" ? task.execution_stats.retry_count : 0;
     const executor = this.taskQueue.getExecutor(task.queue_id, task.type);
     const maxRetries = task.retries ?? executor?.default_retries ?? 0;
@@ -961,14 +1019,15 @@ class TaskRunner {
       max_retries: maxRetries,
       scheduled_at: task.created_at || /* @__PURE__ */ new Date(),
       worker_id: workerId,
+      consumer_id: consumerId,
       log_context: task.metadata?.log_context
     };
   }
-  emitTaskStarted(task, workerId) {
+  emitTaskStarted(task, workerId, consumerId) {
     const startedAt = Date.now();
     this.taskStartTimes.set(tId(task), startedAt);
     if (this.lifecycleProvider?.onTaskStarted) {
-      const ctx = this.buildTaskContext(task, workerId);
+      const ctx = this.buildTaskContext(task, workerId, consumerId);
       const queuedDuration = startedAt - (task.created_at?.getTime() || startedAt);
       this.emitLifecycleEvent(
         this.lifecycleProvider.onTaskStarted,
@@ -980,12 +1039,12 @@ class TaskRunner {
       );
     }
   }
-  emitTaskCompleted(task, workerId, result) {
+  emitTaskCompleted(task, workerId, result, consumerId) {
     const completedAt = Date.now();
     const startedAt = this.taskStartTimes.get(tId(task)) || completedAt;
     this.taskStartTimes.delete(tId(task));
     if (this.lifecycleProvider?.onTaskCompleted) {
-      const ctx = this.buildTaskContext(task, workerId);
+      const ctx = this.buildTaskContext(task, workerId, consumerId);
       const timing = {
         queued_duration_ms: startedAt - (task.created_at?.getTime() || startedAt),
         processing_duration_ms: completedAt - startedAt,
@@ -997,12 +1056,12 @@ class TaskRunner {
       );
     }
   }
-  emitTaskFailed(task, workerId, error, willRetry, nextAttemptAt) {
+  emitTaskFailed(task, workerId, error, willRetry, nextAttemptAt, consumerId) {
     const completedAt = Date.now();
     const startedAt = this.taskStartTimes.get(tId(task)) || completedAt;
     this.taskStartTimes.delete(tId(task));
     if (this.lifecycleProvider?.onTaskFailed) {
-      const ctx = this.buildTaskContext(task, workerId);
+      const ctx = this.buildTaskContext(task, workerId, consumerId);
       const timing = {
         queued_duration_ms: startedAt - (task.created_at?.getTime() || startedAt),
         processing_duration_ms: completedAt - startedAt,
@@ -1020,6 +1079,247 @@ function getEnabledQueues() {
   if (enabledQueues.length === 0) throw new Error("No queues enabled");
   return enabledQueues.map(getEnvironmentQueueName);
 }
+function getFlowMeta(task) {
+  return task.metadata?.flow_meta;
+}
+function getFlowMetaRequired(task) {
+  return task.metadata.flow_meta;
+}
+class FlowMiddleware {
+  constructor(barrierProvider, generateId, flowLifecycleProvider, workerId = "") {
+    this.barrierProvider = barrierProvider;
+    this.generateId = generateId;
+    this.flowLifecycleProvider = flowLifecycleProvider;
+    this.workerId = workerId;
+  }
+  emitFlowEvent(callback, ctx) {
+    if (!callback) return;
+    try {
+      const result = callback(ctx);
+      if (result instanceof Promise) {
+        result.catch(() => {
+        });
+      }
+    } catch {
+    }
+  }
+  buildFlowContext(flowMeta) {
+    return {
+      flow_id: flowMeta.flow_id,
+      total_steps: flowMeta.total_steps,
+      join: flowMeta.join,
+      failure_policy: flowMeta.failure_policy,
+      entity: flowMeta.entity,
+      worker_id: this.workerId
+    };
+  }
+  /**
+   * Process completed tasks for flow orchestration.
+   * Called from TaskHandler.postProcessTasks after markFailed/markSuccess.
+   *
+   * @param input Categorized terminal tasks — success and final-failed (no retries left)
+   * @returns Join tasks to dispatch and entity projections to sync
+   */
+  async onPostProcess(input) {
+    const joinTasks = [];
+    const projections = [];
+    const successTaskSet = new Set(input.successTasks);
+    const allTasks = [...input.successTasks, ...input.failedTasks];
+    const joinCompletions = [];
+    const timeoutTasks = [];
+    const stepTasks = [];
+    for (const task of allTasks) {
+      const flowMeta = getFlowMeta(task);
+      if (!flowMeta) continue;
+      const isSuccess = successTaskSet.has(task);
+      if (flowMeta.is_join) {
+        joinCompletions.push(task);
+      } else if (flowMeta.is_timeout) {
+        timeoutTasks.push(task);
+      } else {
+        stepTasks.push({ task, isSuccess });
+      }
+    }
+    for (const task of joinCompletions) {
+      const flowMeta = getFlowMetaRequired(task);
+      const isSuccess = successTaskSet.has(task);
+      if (flowMeta.entity) {
+        try {
+          const status = isSuccess ? "executed" : "failed";
+          const error = !isSuccess ? task.execution_stats?.last_error || "Join task failed" : void 0;
+          const p = buildProjection(
+            {
+              ...task,
+              id: flowMeta.flow_id,
+              entity: flowMeta.entity
+            },
+            status,
+            { result: task.execution_result, error }
+          );
+          if (p) projections.push(p);
+        } catch {
+        }
+      }
+    }
+    for (const task of timeoutTasks) {
+      const flowMeta = getFlowMetaRequired(task);
+      const flowId = flowMeta.flow_id;
+      await this.barrierProvider.initBarrier(flowId, flowMeta.total_steps);
+      const isComplete = await this.barrierProvider.isComplete(flowId);
+      if (isComplete) continue;
+      const isFirstAbort = await this.barrierProvider.markAborted(flowId);
+      if (!isFirstAbort) continue;
+      const partialResults = await this.barrierProvider.getStepResults(flowId);
+      const flowResults = {
+        flow_id: flowId,
+        steps: partialResults,
+        timed_out: true
+      };
+      const joinTask = this.buildJoinTask(flowMeta, flowResults);
+      joinTasks.push(joinTask);
+      if (this.flowLifecycleProvider?.onFlowTimedOut) {
+        const startedAt = await this.barrierProvider.getStartedAt(flowId);
+        const durationMs = startedAt ? Date.now() - startedAt.getTime() : 0;
+        this.emitFlowEvent(this.flowLifecycleProvider.onFlowTimedOut, {
+          ...this.buildFlowContext(flowMeta),
+          duration_ms: durationMs,
+          steps_completed: partialResults.length
+        });
+      }
+      if (flowMeta.entity) {
+        try {
+          const p = buildProjection(
+            {
+              ...joinTask,
+              id: flowId,
+              entity: flowMeta.entity
+            },
+            "failed",
+            { error: "flow_timeout" }
+          );
+          if (p) projections.push(p);
+        } catch {
+        }
+      }
+    }
+    const stepsByFlow = /* @__PURE__ */ new Map();
+    for (const entry of stepTasks) {
+      const flowMeta = getFlowMetaRequired(entry.task);
+      const group = stepsByFlow.get(flowMeta.flow_id) || [];
+      group.push(entry);
+      stepsByFlow.set(flowMeta.flow_id, group);
+    }
+    for (const [flowId, entries] of stepsByFlow) {
+      const firstFlowMeta = getFlowMetaRequired(entries[0].task);
+      await this.barrierProvider.initBarrier(flowId, firstFlowMeta.total_steps);
+      if (firstFlowMeta.failure_policy === "abort") {
+        const hasFailure = entries.some((e) => !e.isSuccess);
+        if (hasFailure) {
+          const isFirstAbort = await this.barrierProvider.markAborted(flowId);
+          if (isFirstAbort) {
+            const stepResults2 = this.buildStepResults(entries);
+            await this.barrierProvider.batchDecrementAndCheck(flowId, stepResults2);
+            const allResults = await this.barrierProvider.getStepResults(flowId);
+            const flowResults = {
+              flow_id: flowId,
+              steps: allResults,
+              aborted: true
+            };
+            const joinTask = this.buildJoinTask(firstFlowMeta, flowResults);
+            joinTasks.push(joinTask);
+            if (this.flowLifecycleProvider?.onFlowAborted) {
+              const startedAt = await this.barrierProvider.getStartedAt(flowId);
+              const durationMs = startedAt ? Date.now() - startedAt.getTime() : 0;
+              const failedEntry = entries.find((e) => !e.isSuccess);
+              const triggerIndex = failedEntry ? failedEntry.task.metadata?.flow_meta?.step_index ?? -1 : -1;
+              this.emitFlowEvent(this.flowLifecycleProvider.onFlowAborted, {
+                ...this.buildFlowContext(firstFlowMeta),
+                duration_ms: durationMs,
+                steps_completed: allResults.length,
+                trigger_step_index: triggerIndex
+              });
+            }
+            if (firstFlowMeta.entity) {
+              try {
+                const p = buildProjection(
+                  {
+                    ...joinTask,
+                    id: flowId,
+                    entity: firstFlowMeta.entity
+                  },
+                  "failed",
+                  { error: "flow_aborted" }
+                );
+                if (p) projections.push(p);
+              } catch {
+              }
+            }
+          }
+          continue;
+        }
+      }
+      const stepResults = this.buildStepResults(entries);
+      const { remaining } = await this.barrierProvider.batchDecrementAndCheck(flowId, stepResults);
+      if (remaining === 0) {
+        const allResults = await this.barrierProvider.getStepResults(flowId);
+        const flowResults = {
+          flow_id: flowId,
+          steps: allResults
+        };
+        const joinTask = this.buildJoinTask(firstFlowMeta, flowResults);
+        joinTasks.push(joinTask);
+        if (this.flowLifecycleProvider?.onFlowCompleted) {
+          const startedAt = await this.barrierProvider.getStartedAt(flowId);
+          const durationMs = startedAt ? Date.now() - startedAt.getTime() : 0;
+          const stepsSucceeded = allResults.filter((r) => r.status === "success").length;
+          const stepsFailed = allResults.filter((r) => r.status === "fail").length;
+          this.emitFlowEvent(this.flowLifecycleProvider.onFlowCompleted, {
+            ...this.buildFlowContext(firstFlowMeta),
+            duration_ms: durationMs,
+            steps_succeeded: stepsSucceeded,
+            steps_failed: stepsFailed
+          });
+        }
+      }
+    }
+    return { joinTasks, projections };
+  }
+  buildStepResults(entries) {
+    return entries.map(({ task, isSuccess }) => {
+      const flowMeta = getFlowMetaRequired(task);
+      return {
+        step_index: flowMeta.step_index,
+        status: isSuccess ? "success" : "fail",
+        result: task.execution_result,
+        error: !isSuccess ? task.execution_stats?.last_error || "Step failed" : void 0
+      };
+    });
+  }
+  buildJoinTask(flowMeta, flowResults) {
+    const now = /* @__PURE__ */ new Date();
+    const joinFlowMeta = {
+      ...flowMeta,
+      is_join: true,
+      is_timeout: void 0,
+      step_index: -1
+    };
+    return {
+      id: this.generateId(),
+      type: flowMeta.join.type,
+      queue_id: flowMeta.join.queue_id,
+      payload: { flow_results: flowResults },
+      execute_at: now,
+      status: "scheduled",
+      created_at: now,
+      updated_at: now,
+      force_store: true,
+      metadata: {
+        flow_meta: joinFlowMeta
+      },
+      ...flowMeta.entity ? { entity: flowMeta.entity } : {}
+    };
+  }
+}
 const METRICS_KEY_PREFIX = "task_metrics:";
 const DISCARDED_TASKS_KEY = `${METRICS_KEY_PREFIX}discarded_tasks`;
 const STATS_THRESHOLD = parseInt(process.env.TQ_STATS_THRESHOLD || "1000");
@@ -1046,23 +1346,35 @@ class TaskHandler {
     };
     this.totalProcessingMs = 0;
     this.queueStats = /* @__PURE__ */ new Map();
+    this.consumerStatsMap = /* @__PURE__ */ new Map();
     this.logger = new Logger("TaskHandler", LogLevel.INFO);
     this.config = config || {};
     this.workerId = `${os.hostname()}-${process.pid}-${Date.now()}`;
     this.workerStartedAt = /* @__PURE__ */ new Date();
     this.taskStore = new TaskStore(databaseAdapter);
-    this.taskRunner = new TaskRunner(
-      messageQueue,
-      taskQueuesManager,
-      this.taskStore,
-      this.cacheAdapter,
+    if (this.config.flowLifecycleProvider && !this.config.flowBarrierProvider) {
+      throw new Error("[TQ] flowLifecycleProvider requires flowBarrierProvider — flow lifecycle events need flow orchestration enabled");
+    }
+    this.flowMiddleware = this.config.flowBarrierProvider ? new FlowMiddleware(
+      this.config.flowBarrierProvider,
       databaseAdapter.generateId.bind(databaseAdapter),
-      this.config.lifecycleProvider,
-      this.config.lifecycle,
-      this.config.entityProjection,
-      this.config.entityProjectionConfig,
-      this.config.flowMiddleware
-    );
+      this.config.flowLifecycleProvider,
+      this.workerId
+    ) : void 0;
+    this.taskRunner = new TaskRunner({
+      messageQueue,
+      taskQueue: taskQueuesManager,
+      taskStore: this.taskStore,
+      cacheProvider: this.cacheAdapter,
+      generateId: databaseAdapter.generateId.bind(databaseAdapter),
+      lifecycleProvider: this.config.lifecycleProvider,
+      lifecycleConfig: this.config.lifecycle,
+      entityProjection: this.config.entityProjection,
+      entityProjectionConfig: this.config.entityProjectionConfig,
+      flowMiddleware: this.flowMiddleware,
+      flowLifecycleProvider: this.config.flowLifecycleProvider,
+      workerId: this.workerId
+    });
   }
   // ============ Lifecycle Event Helpers ============
   get lifecycleProvider() {
@@ -1367,9 +1679,9 @@ class TaskHandler {
         this.logger.error(`[TQ] Entity projection failed (non-fatal): ${err}`);
       }
     }
-    if (this.config.flowMiddleware) {
+    if (this.flowMiddleware) {
       try {
-        const flowResult = await this.config.flowMiddleware.onPostProcess({ successTasks, failedTasks: finalFailedTasks });
+        const flowResult = await this.flowMiddleware.onPostProcess({ successTasks, failedTasks: finalFailedTasks });
         if (flowResult.projections.length > 0 && this.entityProjectionProvider) {
           await syncProjections(flowResult.projections, this.entityProjectionProvider, this.logger);
         }
@@ -1392,6 +1704,7 @@ class TaskHandler {
       }
       const batchStartTime = Date.now();
       const taskTypes = [...new Set(tasks.map((t) => t.type))];
+      this.registerConsumerIfNew(id, streamName);
       if (this.workerProvider?.onBatchStarted) {
         this.emitLifecycleEvent(
           this.workerProvider.onBatchStarted,
@@ -1468,6 +1781,7 @@ class TaskHandler {
       await this.reportQueueStats(streamName);
       const batchDuration = Date.now() - batchStartTime;
       this.updateWorkerStats(successTasks.length, failedTasks.length, batchDuration);
+      this.updateConsumerStats(id, successTasks.length, failedTasks.length);
       if (this.workerProvider?.onBatchCompleted) {
         this.emitLifecycleEvent(
           this.workerProvider.onBatchCompleted,
@@ -1500,6 +1814,7 @@ class TaskHandler {
     this.processMatureTasks(abortSignal);
     abortSignal?.addEventListener("abort", () => {
       this.stopHeartbeat();
+      this.emitAllConsumersStopped("shutdown");
       this.emitWorkerStopped("shutdown");
     });
   }
@@ -1535,7 +1850,8 @@ class TaskHandler {
       {
         ...this.buildWorkerInfo(),
         stats: { ...this.workerStats },
-        memory_usage_mb: memUsage.heapUsed / 1024 / 1024
+        memory_usage_mb: memUsage.heapUsed / 1024 / 1024,
+        active_consumers: this.getActiveConsumerStats()
       }
     );
   }
@@ -1561,6 +1877,58 @@ class TaskHandler {
       this.workerStats.avg_processing_ms = this.totalProcessingMs / this.workerStats.tasks_processed;
     }
   }
+  // ============ Consumer Tracking ============
+  registerConsumerIfNew(consumerId, queueId) {
+    if (this.consumerStatsMap.has(consumerId)) return;
+    const now = /* @__PURE__ */ new Date();
+    this.consumerStatsMap.set(consumerId, {
+      consumer_id: consumerId,
+      queue_id: queueId,
+      tasks_processed: 0,
+      tasks_succeeded: 0,
+      tasks_failed: 0,
+      started_at: now
+    });
+    if (this.workerProvider?.onConsumerStarted) {
+      this.emitLifecycleEvent(
+        this.workerProvider.onConsumerStarted,
+        {
+          consumer_id: consumerId,
+          queue_id: queueId,
+          worker_id: this.workerId,
+          started_at: now
+        }
+      );
+    }
+  }
+  updateConsumerStats(consumerId, succeeded, failed) {
+    const stats = this.consumerStatsMap.get(consumerId);
+    if (!stats) return;
+    stats.tasks_processed += succeeded + failed;
+    stats.tasks_succeeded += succeeded;
+    stats.tasks_failed += failed;
+    stats.last_task_at = /* @__PURE__ */ new Date();
+  }
+  getActiveConsumerStats() {
+    return Array.from(this.consumerStatsMap.values()).map(({ started_at, ...stats }) => stats);
+  }
+  emitAllConsumersStopped(reason) {
+    if (!this.workerProvider?.onConsumerStopped) return;
+    for (const entry of this.consumerStatsMap.values()) {
+      const { started_at, ...stats } = entry;
+      this.emitLifecycleEvent(
+        this.workerProvider.onConsumerStopped,
+        {
+          consumer_id: entry.consumer_id,
+          queue_id: entry.queue_id,
+          worker_id: this.workerId,
+          started_at,
+          reason,
+          stats
+        }
+      );
+    }
+  }
   emitLifecycleEvent(callback, ctx) {
     if (!callback) return;
     try {
@@ -1656,6 +2024,7 @@ class TaskHandler {
       attempt: retryCount + 1,
       max_retries: maxRetries,
       scheduled_at: task.created_at || /* @__PURE__ */ new Date(),
+      worker_id: this.workerId,
       log_context: task.metadata?.log_context
     };
   }
@@ -1835,7 +2204,8 @@ class InMemoryFlowBarrierProvider {
     this.barriers.set(flowId, {
       remaining: totalSteps,
       status: "active",
-      results: /* @__PURE__ */ new Map()
+      results: /* @__PURE__ */ new Map(),
+      started_at: /* @__PURE__ */ new Date()
     });
   }
   async batchDecrementAndCheck(flowId, results) {
@@ -1879,190 +2249,10 @@ class InMemoryFlowBarrierProvider {
     if (!state) return false;
     return state.status === "complete";
   }
-}
-function getFlowMeta(task) {
-  return task.metadata?.flow_meta;
-}
-function getFlowMetaRequired(task) {
-  return task.metadata.flow_meta;
-}
-class FlowMiddleware {
-  constructor(barrierProvider, generateId) {
-    this.barrierProvider = barrierProvider;
-    this.generateId = generateId;
-  }
-  /**
-   * Process completed tasks for flow orchestration.
-   * Called from TaskHandler.postProcessTasks after markFailed/markSuccess.
-   *
-   * @param input Categorized terminal tasks — success and final-failed (no retries left)
-   * @returns Join tasks to dispatch and entity projections to sync
-   */
-  async onPostProcess(input) {
-    const joinTasks = [];
-    const projections = [];
-    const successTaskSet = new Set(input.successTasks);
-    const allTasks = [...input.successTasks, ...input.failedTasks];
-    const joinCompletions = [];
-    const timeoutTasks = [];
-    const stepTasks = [];
-    for (const task of allTasks) {
-      const flowMeta = getFlowMeta(task);
-      if (!flowMeta) continue;
-      const isSuccess = successTaskSet.has(task);
-      if (flowMeta.is_join) {
-        joinCompletions.push(task);
-      } else if (flowMeta.is_timeout) {
-        timeoutTasks.push(task);
-      } else {
-        stepTasks.push({ task, isSuccess });
-      }
-    }
-    for (const task of joinCompletions) {
-      const flowMeta = getFlowMetaRequired(task);
-      const isSuccess = successTaskSet.has(task);
-      if (flowMeta.entity) {
-        try {
-          const status = isSuccess ? "executed" : "failed";
-          const error = !isSuccess ? task.execution_stats?.last_error || "Join task failed" : void 0;
-          const p = buildProjection(
-            {
-              ...task,
-              id: flowMeta.flow_id,
-              entity: flowMeta.entity
-            },
-            status,
-            { result: task.execution_result, error }
-          );
-          if (p) projections.push(p);
-        } catch {
-        }
-      }
-    }
-    for (const task of timeoutTasks) {
-      const flowMeta = getFlowMetaRequired(task);
-      const flowId = flowMeta.flow_id;
-      await this.barrierProvider.initBarrier(flowId, flowMeta.total_steps);
-      const isComplete = await this.barrierProvider.isComplete(flowId);
-      if (isComplete) continue;
-      const isFirstAbort = await this.barrierProvider.markAborted(flowId);
-      if (!isFirstAbort) continue;
-      const partialResults = await this.barrierProvider.getStepResults(flowId);
-      const flowResults = {
-        flow_id: flowId,
-        steps: partialResults,
-        timed_out: true
-      };
-      const joinTask = this.buildJoinTask(flowMeta, flowResults);
-      joinTasks.push(joinTask);
-      if (flowMeta.entity) {
-        try {
-          const p = buildProjection(
-            {
-              ...joinTask,
-              id: flowId,
-              entity: flowMeta.entity
-            },
-            "failed",
-            { error: "flow_timeout" }
-          );
-          if (p) projections.push(p);
-        } catch {
-        }
-      }
-    }
-    const stepsByFlow = /* @__PURE__ */ new Map();
-    for (const entry of stepTasks) {
-      const flowMeta = getFlowMetaRequired(entry.task);
-      const group = stepsByFlow.get(flowMeta.flow_id) || [];
-      group.push(entry);
-      stepsByFlow.set(flowMeta.flow_id, group);
-    }
-    for (const [flowId, entries] of stepsByFlow) {
-      const firstFlowMeta = getFlowMetaRequired(entries[0].task);
-      await this.barrierProvider.initBarrier(flowId, firstFlowMeta.total_steps);
-      if (firstFlowMeta.failure_policy === "abort") {
-        const hasFailure = entries.some((e) => !e.isSuccess);
-        if (hasFailure) {
-          const isFirstAbort = await this.barrierProvider.markAborted(flowId);
-          if (isFirstAbort) {
-            const stepResults2 = this.buildStepResults(entries);
-            await this.barrierProvider.batchDecrementAndCheck(flowId, stepResults2);
-            const allResults = await this.barrierProvider.getStepResults(flowId);
-            const flowResults = {
-              flow_id: flowId,
-              steps: allResults,
-              aborted: true
-            };
-            const joinTask = this.buildJoinTask(firstFlowMeta, flowResults);
-            joinTasks.push(joinTask);
-            if (firstFlowMeta.entity) {
-              try {
-                const p = buildProjection(
-                  {
-                    ...joinTask,
-                    id: flowId,
-                    entity: firstFlowMeta.entity
-                  },
-                  "failed",
-                  { error: "flow_aborted" }
-                );
-                if (p) projections.push(p);
-              } catch {
-              }
-            }
-          }
-          continue;
-        }
-      }
-      const stepResults = this.buildStepResults(entries);
-      const { remaining } = await this.barrierProvider.batchDecrementAndCheck(flowId, stepResults);
-      if (remaining === 0) {
-        const allResults = await this.barrierProvider.getStepResults(flowId);
-        const flowResults = {
-          flow_id: flowId,
-          steps: allResults
-        };
-        const joinTask = this.buildJoinTask(firstFlowMeta, flowResults);
-        joinTasks.push(joinTask);
-      }
-    }
-    return { joinTasks, projections };
-  }
-  buildStepResults(entries) {
-    return entries.map(({ task, isSuccess }) => {
-      const flowMeta = getFlowMetaRequired(task);
-      return {
-        step_index: flowMeta.step_index,
-        status: isSuccess ? "success" : "fail",
-        result: task.execution_result,
-        error: !isSuccess ? task.execution_stats?.last_error || "Step failed" : void 0
-      };
-    });
-  }
-  buildJoinTask(flowMeta, flowResults) {
-    const now = /* @__PURE__ */ new Date();
-    const joinFlowMeta = {
-      ...flowMeta,
-      is_join: true,
-      is_timeout: void 0,
-      step_index: -1
-    };
-    return {
-      id: this.generateId(),
-      type: flowMeta.join.type,
-      queue_id: flowMeta.join.queue_id,
-      payload: { flow_results: flowResults },
-      execute_at: now,
-      status: "scheduled",
-      created_at: now,
-      updated_at: now,
-      force_store: true,
-      metadata: {
-        flow_meta: joinFlowMeta
-      },
-      ...flowMeta.entity ? { entity: flowMeta.entity } : {}
-    };
+  async getStartedAt(flowId) {
+    const state = this.barriers.get(flowId);
+    if (!state) return null;
+    return state.started_at;
   }
 }
 class ConsoleHealthProvider {
@@ -2102,7 +2292,7 @@ class ConsoleHealthProvider {
   }
   onWorkerHeartbeat(info) {
     console.log(
-      `${this.prefix} Worker heartbeat: ${info.worker_id} - processed: ${info.stats.tasks_processed}, success: ${info.stats.tasks_succeeded}, failed: ${info.stats.tasks_failed}, avg: ${info.stats.avg_processing_ms.toFixed(2)}ms, memory: ${info.memory_usage_mb.toFixed(2)}MB`
+      `${this.prefix} Worker heartbeat: ${info.worker_id} - processed: ${info.stats.tasks_processed}, success: ${info.stats.tasks_succeeded}, failed: ${info.stats.tasks_failed}, avg: ${info.stats.avg_processing_ms.toFixed(2)}ms, memory: ${info.memory_usage_mb.toFixed(2)}MB, consumers: ${info.active_consumers.length}`
     );
   }
   onWorkerStopped(info) {
@@ -2120,6 +2310,49 @@ class ConsoleHealthProvider {
       `${this.prefix} Batch completed: ${info.batch_size} tasks in ${info.duration_ms}ms - succeeded: ${info.succeeded}, failed: ${info.failed}`
     );
   }
+  // ============ Consumer Lifecycle ============
+  onConsumerStarted(info) {
+    console.log(
+      `${this.prefix} Consumer started: ${info.consumer_id} on worker ${info.worker_id} - queue: ${info.queue_id}`
+    );
+  }
+  onConsumerStopped(info) {
+    console.log(
+      `${this.prefix} Consumer stopped: ${info.consumer_id} on worker ${info.worker_id} - reason: ${info.reason}, processed: ${info.stats.tasks_processed}, succeeded: ${info.stats.tasks_succeeded}, failed: ${info.stats.tasks_failed}`
+    );
+  }
+  // ============ Task Batch Lifecycle ============
+  onTaskBatchStarted(ctx) {
+    console.log(
+      `${this.prefix} Task batch started: ${ctx.task_type} - ${ctx.tasks.length} tasks in queue ${ctx.queue_id}`
+    );
+  }
+  onTaskBatchCompleted(ctx) {
+    console.log(
+      `${this.prefix} Task batch completed: ${ctx.task_type} in ${ctx.duration_ms}ms - succeeded: ${ctx.succeeded.length}, failed: ${ctx.failed.length}`
+    );
+  }
+  // ============ Flow Lifecycle ============
+  onFlowStarted(ctx) {
+    console.log(
+      `${this.prefix} Flow started: ${ctx.flow_id} - ${ctx.total_steps} steps [${ctx.step_types.join(", ")}], join: ${ctx.join.type}, policy: ${ctx.failure_policy}`
+    );
+  }
+  onFlowCompleted(ctx) {
+    console.log(
+      `${this.prefix} Flow completed: ${ctx.flow_id} in ${ctx.duration_ms}ms - succeeded: ${ctx.steps_succeeded}, failed: ${ctx.steps_failed}`
+    );
+  }
+  onFlowAborted(ctx) {
+    console.log(
+      `${this.prefix} Flow aborted: ${ctx.flow_id} in ${ctx.duration_ms}ms - steps completed: ${ctx.steps_completed}, trigger: step ${ctx.trigger_step_index}`
+    );
+  }
+  onFlowTimedOut(ctx) {
+    console.log(
+      `${this.prefix} Flow timed out: ${ctx.flow_id} in ${ctx.duration_ms}ms - steps completed: ${ctx.steps_completed}`
+    );
+  }
 }
 export {
   Actions,