notifkit 0.1.7 → 0.1.9

package/dist/{main-CjbVdUqa.mjs.map → main-D19Wxs0x.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"main-CjbVdUqa.mjs","names":["sql"],"sources":["../src/services/workflow/main.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { loadEnv, readBaseConfig } from \"@/index.js\";\nimport { createLogger } from \"@/index.js\";\nimport { RedisClient, type Redis } from \"@/index.js\";\nimport {\n  StreamConsumer,\n  PendingMessageScanner,\n  StreamProducer,\n  type StreamMessage,\n} from \"@/index.js\";\nimport { BaseWorker } from \"@/index.js\";\nimport { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from \"@/index.js\";\nimport { type StreamName } from \"@/contracts/streams.js\";\nimport { createDatabase } from \"@/db/index.js\";\nimport {\n  workflowInstances,\n  workflowSteps,\n  workflowWaiters,\n  workflowDefinitions,\n} from \"@/db/schema.js\";\nimport { eq, and } from \"drizzle-orm\";\nimport {\n  workflowRegistry,\n  SuspendExecutionError,\n  buildStepNotifyPayload,\n  type WorkflowContext,\n  type WorkflowStepContext,\n} from \"@/workflows/index.js\";\nimport {\n  type WorkerOptions,\n  LUA_SCHEDULER_POLL,\n  LUA_RELEASE_LOCK,\n  LUA_RENEW_LOCK,\n} from \"@/shared/index.js\";\nimport { startHealthReporter } from \"@/workers/index.js\";\n\n/** How long a workflow instance lock is held before it self-expires. */\nconst WORKFLOW_LOCK_TTL_SECONDS = 60;\n/** Renew the lock well inside its TTL so long-running handlers keep it. */\nconst WORKFLOW_LOCK_RENEW_MS = (WORKFLOW_LOCK_TTL_SECONDS / 3) * 1000;\n/** How long a claimed workflow timer stays invisible to other pollers. */\nconst WORKFLOW_TIMER_VISIBILITY_MS = 60_000;\n\nloadEnv();\nconst config = readBaseConfig();\nlet logger: ReturnType<typeof createLogger>;\nlet redis: RedisClient;\nlet sql: any;\nlet db: any;\n\nlet consumer: StreamConsumer;\nlet pendingScanner: PendingMessageScanner;\nlet worker: BaseWorker;\nlet healthInterval: NodeJS.Timeout | null = null;\nlet pollInterval: NodeJS.Timeout | null = null;\nlet reaperInterval: NodeJS.Timeout | null = null;\n\nlet notificationProducer: StreamProducer;\nlet workflowProducer: StreamProducer;\n\nexport interface WorkflowWorkerOptions extends WorkerOptions {\n  redis: Redis;\n  db: any;\n  workflowProducer: any;\n  notificationProducer: any;\n}\n\nexport class WorkflowWorker extends BaseWorker {\n  private readonly redisCli: Redis;\n  private readonly dbConn: any;\n  private readonly workflowProducer: any;\n  private readonly notificationProducer: any;\n\n  private eventBuffer: {\n    producer: any;\n    event: any;\n    resolve: (result?: { messageId: string; notificationId: string }) => void;\n    reject: (err: any) => void;\n  }[] = [];\n  private flushTimer: NodeJS.Timeout | null = null;\n\n  constructor(options: WorkflowWorkerOptions) {\n    super(options);\n    this.redisCli = options.redis;\n    this.dbConn = options.db;\n    this.workflowProducer = options.workflowProducer;\n    this.notificationProducer = options.notificationProducer;\n\n    this.flushTimer = setInterval(() => void this.flushWorkerBuffers(), 100);\n  }\n\n  override async stop(): Promise<void> {\n    if (this.flushTimer) {\n      clearInterval(this.flushTimer);\n      this.flushTimer = null;\n    }\n    await this.flushWorkerBuffers();\n    await super.stop();\n  }\n\n  private async flushWorkerBuffers(): Promise<void> {\n    if (this.eventBuffer.length === 0) return;\n\n    const events = this.eventBuffer;\n    this.eventBuffer = [];\n\n    try {\n      const byProducer = new Map<any, typeof events>();\n      for (const item of events) {\n        if (!byProducer.has(item.producer)) byProducer.set(item.producer, []);\n        byProducer.get(item.producer)!.push(item);\n      }\n\n      for (const [producer, batch] of byProducer) {\n        const { messageIds, eventIds } = await producer.publishBatch(batch.map((b) => b.event));\n        for (let i = 0; i < batch.length; i++) {\n          const mId = messageIds[i];\n          const eId = eventIds[i];\n          if (mId && eId) batch[i]!.resolve({ messageId: mId, notificationId: eId });\n        }\n      }\n    } catch (err: any) {\n      this.logger.error({ err }, \"failed to flush workflow worker buffer\");\n      for (const e of events) e.reject(err);\n    }\n  }\n\n  async process(message: StreamMessage): Promise<void> {\n    const { event } = message;\n    const publishPromises: Promise<void>[] = [];\n\n    if (event.type !== \"workflow.triggered\" && event.type !== \"workflow.resumed\") {\n      return;\n    }\n\n    const payload = event.payload as any;\n    const name = payload.name;\n    const instanceId = payload.instanceId;\n\n    if (!name || !instanceId || !payload.projectId) {\n      this.logger.warn(\"Missing name, instanceId, or projectId in workflow event\");\n      return;\n    }\n\n    let handler = workflowRegistry.get(name);\n    if (!handler) {\n      // Fallback: check dynamic JSON workflows\n      const defRows = await this.dbConn\n        .select()\n        .from(workflowDefinitions)\n        .where(\n          and(\n            eq(workflowDefinitions.projectId, payload.projectId),\n            eq(workflowDefinitions.name, name),\n          ),\n        )\n        .limit(1);\n\n      if (defRows.length === 0) {\n        this.logger.warn(\n          { name, projectId: payload.projectId },\n          \"No handler or dynamic definition found for workflow\",\n        );\n        return;\n      }\n\n      const def = defRows[0];\n      handler = async ({ step }) => {\n        const steps = def.steps as any[];\n        for (const stepDef of steps) {\n          if (stepDef.action === \"notify\") {\n            // A payload target wins; naming none inherits the instance user.\n            await step.notify(stepDef.payload);\n          } else if (stepDef.action === \"wait\") {\n            await step.wait(stepDef.duration);\n          } else if (stepDef.action === \"waitForEvent\") {\n            await step.waitForEvent(stepDef.event, stepDef.options);\n          } else {\n            this.logger.warn(\n              { action: (stepDef as any)?.action, name },\n              \"unknown workflow step action\",\n            );\n          }\n        }\n      };\n    }\n\n    const lockKey = `lock:workflow:${instanceId}`;\n    const lockToken = randomUUID();\n    const acquired = await this.redisCli.set(\n      lockKey,\n      lockToken,\n      \"EX\",\n      WORKFLOW_LOCK_TTL_SECONDS,\n      \"NX\",\n    );\n    if (!acquired) {\n      this.logger.info({ instanceId }, \"Workflow is locked by another process, skipping\");\n      return;\n    }\n\n    // Keep the lock alive while the handler runs; without this a handler that\n    // outlives the TTL lets a second resume execute the same steps in parallel.\n    const renewTimer = setInterval(() => {\n      void this.redisCli\n        .eval(LUA_RENEW_LOCK, 1, lockKey, lockToken, String(WORKFLOW_LOCK_TTL_SECONDS))\n        .catch((err: unknown) => {\n          this.logger.warn({ err, instanceId }, \"failed to renew workflow lock\");\n        });\n    }, WORKFLOW_LOCK_RENEW_MS);\n\n    try {\n      let instance = (\n        await this.dbConn\n          .select()\n          .from(workflowInstances)\n          .where(eq(workflowInstances.id, instanceId))\n          .limit(1)\n      )[0];\n      if (!instance) {\n        const rows = await this.dbConn\n          .insert(workflowInstances)\n          .values({\n            id: instanceId,\n            projectId: payload.projectId,\n            name: name,\n            status: \"pending\",\n            input: payload.input || {},\n          })\n          .returning();\n        instance = rows[0]!;\n      }\n\n      if (instance.status !== \"pending\") {\n        this.logger.info({ instanceId }, \"Workflow is not pending, skipping\");\n        return;\n      }\n\n      await this.dbConn\n        .update(workflowInstances)\n        .set({ status: \"running\" })\n        .where(eq(workflowInstances.id, instanceId));\n\n      // Load existing steps\n      const existingSteps = await this.dbConn\n        .select()\n        .from(workflowSteps)\n        .where(eq(workflowSteps.instanceId, instanceId));\n      const stepOutputMap = new Map<string, any>();\n      for (const s of existingSteps) {\n        stepOutputMap.set(s.stepIndex, s.output);\n      }\n\n      let currentStepIndex = 0;\n\n      const stepProxy: WorkflowStepContext = {\n        notify: async (args) => {\n          const stepId = String(currentStepIndex++);\n          if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);\n\n          // The step payload is the same shape as notify(), but the wire event\n          // is not — translate rather than spread.\n          const requested = buildStepNotifyPayload(\n            args,\n            instance!.input,\n            payload.projectId,\n            `wf-${instanceId}-${stepId}`,\n          );\n\n          const result = await new Promise<{ messageId: string; notificationId: string }>(\n            (resolve, reject) => {\n              this.eventBuffer.push({\n                producer: this.notificationProducer,\n                event: buildStreamEvent(\n                  \"notification.requested\",\n                  requested as unknown as Record<string, unknown>,\n                  \"workflow\",\n                  event.metadata.traceId,\n                ),\n                resolve: resolve as any,\n                reject,\n              });\n            },\n          );\n\n          const output = {\n            success: true,\n            messageId: result.messageId,\n            notificationId: result.notificationId,\n          };\n\n          await this.dbConn.insert(workflowSteps).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            stepIndex: stepId,\n            action: \"notify\",\n            output,\n          });\n\n          return output;\n        },\n        wait: async (duration) => {\n          const stepId = String(currentStepIndex++);\n          if (stepOutputMap.has(stepId)) return;\n\n          // Simple duration parse (e.g. '2h' -> ms)\n          let ms = 0;\n          if (duration.endsWith(\"d\")) ms = parseInt(duration) * 24 * 60 * 60 * 1000;\n          else if (duration.endsWith(\"h\")) ms = parseInt(duration) * 60 * 60 * 1000;\n          else if (duration.endsWith(\"m\")) ms = parseInt(duration) * 60 * 1000;\n          else if (duration.endsWith(\"s\")) ms = parseInt(duration) * 1000;\n          else throw new Error(`Invalid wait duration: ${duration}`);\n\n          const resumeAt = Date.now() + ms;\n\n          // Persist the wake-up signal before committing the suspended state.\n          // If the process dies after this point the source event is retried;\n          // if it dies after the database write, the timer is already durable.\n          await this.redisCli.zadd(\n            \"notif:workflow:timers\",\n            resumeAt,\n            JSON.stringify({\n              instanceId: instance!.id,\n              name: instance!.name,\n              projectId: payload.projectId,\n              input: instance!.input,\n            }),\n          );\n\n          await this.dbConn.insert(workflowSteps).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            stepIndex: stepId,\n            action: \"wait\",\n            output: { scheduledAt: resumeAt },\n          });\n\n          throw new SuspendExecutionError(\"wait\", { duration });\n        },\n        waitForEvent: async (eventName, options) => {\n          const stepId = String(currentStepIndex++);\n          if (stepOutputMap.has(stepId)) {\n            const out = stepOutputMap.get(stepId);\n            if (out && typeof out === \"object\" && (out as any).timedOut === true) {\n              return null;\n            }\n            return out;\n          }\n\n          options = options || {};\n          options.timeout = options.timeout || \"24h\";\n          options.match = options.match || {};\n\n          let ms = 0;\n          if (options.timeout.endsWith(\"d\")) ms = parseInt(options.timeout) * 24 * 60 * 60 * 1000;\n          else if (options.timeout.endsWith(\"h\")) ms = parseInt(options.timeout) * 60 * 60 * 1000;\n          else if (options.timeout.endsWith(\"m\")) ms = parseInt(options.timeout) * 60 * 1000;\n          else if (options.timeout.endsWith(\"s\")) ms = parseInt(options.timeout) * 1000;\n          else throw new Error(`Invalid waitForEvent timeout: ${options.timeout}`);\n\n          const resumeAt = Date.now() + ms;\n\n          await this.redisCli.zadd(\n            \"notif:workflow:timers\",\n            resumeAt,\n            JSON.stringify({\n              instanceId: instance!.id,\n              name: instance!.name,\n              projectId: payload.projectId,\n              input: instance!.input,\n              isEventTimeout: true,\n              eventName,\n              stepId,\n            }),\n          );\n\n          // Register waiter\n          await this.dbConn.insert(workflowWaiters).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            eventName: eventName,\n            matchCriteria: options.match,\n            expiresAt: new Date(resumeAt),\n          });\n\n          // Register step as pending event\n          await this.dbConn.insert(workflowSteps).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            stepIndex: stepId,\n            action: \"waitForEvent\",\n            output: null, // this will be updated by event worker or timeout\n          });\n\n          throw new SuspendExecutionError(\"waitForEvent\", { eventName });\n        },\n        run: async (stepName, fn) => {\n          const stepId = String(currentStepIndex++);\n          if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);\n\n          const result = await fn();\n          await this.dbConn.insert(workflowSteps).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            stepIndex: stepId,\n            action: \"run\",\n            output: result,\n          });\n          return result;\n        },\n      };\n\n      const ctx: WorkflowContext = {\n        step: stepProxy,\n        event: (instance!.input as any) || { user: { id: \"unknown\" } },\n      };\n\n      try {\n        await handler(ctx);\n        // If we reach here, workflow completed\n        await this.dbConn\n          .update(workflowInstances)\n          .set({ status: \"completed\" })\n          .where(eq(workflowInstances.id, instance!.id));\n        this.logger.info({ instanceId: instance!.id }, \"Workflow completed successfully\");\n      } catch (err: any) {\n        if (err instanceof SuspendExecutionError || err.name === \"SuspendExecutionError\") {\n          await this.dbConn\n            .update(workflowInstances)\n            .set({ status: \"pending\" })\n            .where(eq(workflowInstances.id, instance!.id));\n          this.logger.info({ instanceId: instance!.id, reason: err.reason }, \"Workflow suspended\");\n        } else {\n          this.logger.error({ err, instanceId: instance!.id }, \"Workflow failed\");\n          await this.dbConn\n            .update(workflowInstances)\n            .set({ status: \"failed\" })\n            .where(eq(workflowInstances.id, instance!.id));\n        }\n      }\n    } finally {\n      clearInterval(renewTimer);\n      // Compare-and-delete: never release a lock a later process re-acquired.\n      await this.redisCli.eval(LUA_RELEASE_LOCK, 1, lockKey, lockToken);\n    }\n\n    await Promise.all(publishPromises);\n  }\n}\n\nexport function __injectForTests(r: any, d: any, wp: any, np: any) {\n  redis = r;\n  db = d;\n  workflowProducer = wp;\n  notificationProducer = np;\n}\n\nexport async function startWorkflowWorker() {\n  logger = createLogger({ name: \"workflow-worker\", level: config.LOG_LEVEL });\n  redis = new RedisClient({ url: config.REDIS_URL, name: \"workflow\", logger });\n  const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: \"workflow\", logger });\n  sql = dbData.sql;\n  db = dbData.db;\n  notificationProducer = new StreamProducer({\n    redis: redis.native,\n    stream: STREAMS.INBOUND_NORMAL,\n    logger,\n  });\n  workflowProducer = new StreamProducer({\n    redis: redis.native,\n    stream: STREAMS.WORKFLOW_INBOUND,\n    logger,\n  });\n  consumer = new StreamConsumer({\n    redis: redis.native,\n    stream: STREAMS.WORKFLOW_INBOUND as StreamName,\n    group: CONSUMER_GROUPS.WORKFLOW as any,\n    consumer: `workflow-${process.pid}`,\n    dlqStream: STREAMS.DEAD_LETTER,\n    batchSize: config.WORKER_CONCURRENCY,\n    logger,\n  });\n\n  pendingScanner = new PendingMessageScanner({\n    redis: redis.native,\n    stream: STREAMS.WORKFLOW_INBOUND as StreamName,\n    group: CONSUMER_GROUPS.WORKFLOW as any,\n    consumer: `workflow-${process.pid}`,\n    logger,\n  });\n\n  worker = new WorkflowWorker({\n    consumer,\n    pendingScanner,\n    logger,\n    concurrency: config.WORKER_CONCURRENCY,\n    redis: redis.native,\n    db,\n    workflowProducer,\n    notificationProducer,\n  });\n\n  // Polling loop for timers\n  pollInterval = setInterval(() => {\n    void (async () => {\n      try {\n        const now = Date.now();\n        const tasks = (await redis.native.eval(\n          LUA_SCHEDULER_POLL,\n          1,\n          \"notif:workflow:timers\",\n          now,\n          100,\n          WORKFLOW_TIMER_VISIBILITY_MS,\n        )) as string[];\n\n        for (const taskStr of tasks) {\n          const task = JSON.parse(taskStr);\n\n          // Check if workflow is already completed/failed\n          const inst = (\n            await db\n              .select()\n              .from(workflowInstances)\n              .where(eq(workflowInstances.id, task.instanceId))\n              .limit(1)\n          )[0];\n          if (!inst || inst.status === \"completed\" || inst.status === \"failed\") {\n            await redis.native.zrem(\"notif:workflow:timers\", taskStr);\n            continue;\n          }\n          // A timer can be observed while the worker is still persisting the\n          // corresponding suspension. Leave the claimed member in place; its\n          // visibility timeout will make it eligible once the instance becomes\n          // pending instead of losing the wake-up signal.\n          if (inst.status === \"running\") continue;\n\n          if (task.isEventTimeout) {\n            // It's a timeout for waitForEvent. Clean up only this step's waiter\n            const deleted = await db\n              .delete(workflowWaiters)\n              .where(\n                and(\n                  eq(workflowWaiters.instanceId, task.instanceId),\n                  eq(workflowWaiters.eventName, task.eventName),\n                ),\n              )\n              .returning();\n\n            // If the waiter was already deleted (by EventWorker when event arrived before timeout),\n            // this timer is stale. Clean up from Redis and skip re-resuming the workflow.\n            if (deleted.length === 0) {\n              await redis.native.zrem(\"notif:workflow:timers\", taskStr);\n              continue;\n            }\n\n            // Record timedOut state on the pending waitForEvent step\n            const steps = await db\n              .select()\n              .from(workflowSteps)\n              .where(\n                and(\n                  eq(workflowSteps.instanceId, task.instanceId),\n                  eq(workflowSteps.action, \"waitForEvent\"),\n                ),\n              );\n            const pendingStep = steps.find((s: any) => s.output === null);\n            if (pendingStep) {\n              await db\n                .update(workflowSteps)\n                .set({ output: { timedOut: true } })\n                .where(eq(workflowSteps.id, pendingStep.id));\n            }\n          }\n\n          await workflowProducer.publish(\n            buildStreamEvent(\"workflow.resumed\", task, \"scheduler\", undefined),\n          );\n          await redis.native.zrem(\"notif:workflow:timers\", taskStr);\n          logger.info({ instanceId: task.instanceId }, \"Workflow resumed from timer\");\n        }\n      } catch (err) {\n        logger.error({ err }, \"error in workflow polling loop\");\n      }\n    })();\n  }, 5000);\n\n  reaperInterval = setInterval(() => {\n    void (async () => {\n      try {\n        const runningInstances = await db\n          .select({ id: workflowInstances.id })\n          .from(workflowInstances)\n          .where(eq(workflowInstances.status, \"running\"));\n        for (const inst of runningInstances) {\n          const lockKey = `lock:workflow:${inst.id}`;\n          const hasLock = await redis.native.exists(lockKey);\n          if (!hasLock) {\n            await db\n              .update(workflowInstances)\n              .set({ status: \"pending\" })\n              .where(eq(workflowInstances.id, inst.id));\n            logger.info({ instanceId: inst.id }, \"Reaped stuck workflow instance (lock expired)\");\n          }\n        }\n      } catch (err) {\n        logger.error({ err }, \"error in stuck workflow reaper\");\n      }\n    })();\n  }, 60000);\n\n  healthInterval = startHealthReporter(\"workflow\", worker, redis, logger);\n\n  logger.info(\"workflow worker starting\");\n  await worker.start();\n}\n\nexport async function stopWorkflowWorker(): Promise<void> {\n  logger?.info(\"shutdown initiated\");\n  if (healthInterval) clearInterval(healthInterval);\n  if (pollInterval) clearInterval(pollInterval);\n  if (reaperInterval) clearInterval(reaperInterval);\n  if (worker) await worker.stop();\n  if (sql) await sql.end();\n  if (redis) await redis.disconnect();\n  logger?.info(\"workflow worker stopped\");\n}\n"],"mappings":";;;;;AAqCA,MAAM,4BAA4B;;AAElC,MAAM,yBAA0B,4BAA4B,IAAK;;AAEjE,MAAM,+BAA+B;AAErC,QAAQ;AACR,MAAM,SAAS,eAAe;AAC9B,IAAI;AACJ,IAAI;AACJ,IAAIA;AACJ,IAAI;AAEJ,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI,iBAAwC;AAC5C,IAAI,eAAsC;AAC1C,IAAI,iBAAwC;AAE5C,IAAI;AACJ,IAAI;AASJ,IAAa,iBAAb,cAAoC,WAAW;CAC7C;CACA;CACA;CACA;CAEA,cAKM,CAAC;CACP,aAA4C;CAE5C,YAAY,SAAgC;EAC1C,MAAM,OAAO;EACb,KAAK,WAAW,QAAQ;EACxB,KAAK,SAAS,QAAQ;EACtB,KAAK,mBAAmB,QAAQ;EAChC,KAAK,uBAAuB,QAAQ;EAEpC,KAAK,aAAa,kBAAkB,KAAK,KAAK,mBAAmB,GAAG,GAAG;CACzE;CAEA,MAAe,OAAsB;EACnC,IAAI,KAAK,YAAY;GACnB,cAAc,KAAK,UAAU;GAC7B,KAAK,aAAa;EACpB;EACA,MAAM,KAAK,mBAAmB;EAC9B,MAAM,MAAM,KAAK;CACnB;CAEA,MAAc,qBAAoC;EAChD,IAAI,KAAK,YAAY,WAAW,GAAG;EAEnC,MAAM,SAAS,KAAK;EACpB,KAAK,cAAc,CAAC;EAEpB,IAAI;GACF,MAAM,6BAAa,IAAI,IAAwB;GAC/C,KAAK,MAAM,QAAQ,QAAQ;IACzB,IAAI,CAAC,WAAW,IAAI,KAAK,QAAQ,GAAG,WAAW,IAAI,KAAK,UAAU,CAAC,CAAC;IACpE,WAAW,IAAI,KAAK,QAAQ,CAAC,CAAE,KAAK,IAAI;GAC1C;GAEA,KAAK,MAAM,CAAC,UAAU,UAAU,YAAY;IAC1C,MAAM,EAAE,YAAY,aAAa,MAAM,SAAS,aAAa,MAAM,KAAK,MAAM,EAAE,KAAK,CAAC;IACtF,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;KACrC,MAAM,MAAM,WAAW;KACvB,MAAM,MAAM,SAAS;KACrB,IAAI,OAAO,KAAK,MAAM,EAAE,CAAE,QAAQ;MAAE,WAAW;MAAK,gBAAgB;KAAI,CAAC;IAC3E;GACF;EACF,SAAS,KAAU;GACjB,KAAK,OAAO,MAAM,EAAE,IAAI,GAAG,wCAAwC;GACnE,KAAK,MAAM,KAAK,QAAQ,EAAE,OAAO,GAAG;EACtC;CACF;CAEA,MAAM,QAAQ,SAAuC;EACnD,MAAM,EAAE,UAAU;EAClB,MAAM,kBAAmC,CAAC;EAE1C,IAAI,MAAM,SAAS,wBAAwB,MAAM,SAAS,oBACxD;EAGF,MAAM,UAAU,MAAM;EACtB,MAAM,OAAO,QAAQ;EACrB,MAAM,aAAa,QAAQ;EAE3B,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,WAAW;GAC9C,KAAK,OAAO,KAAK,0DAA0D;GAC3E;EACF;EAEA,IAAI,UAAU,iBAAiB,IAAI,IAAI;EACvC,IAAI,CAAC,SAAS;GAEZ,MAAM,UAAU,MAAM,KAAK,OACxB,OAAO,CAAC,CACR,KAAK,mBAAmB,CAAC,CACzB,MACC,IACE,GAAG,oBAAoB,WAAW,QAAQ,SAAS,GACnD,GAAG,oBAAoB,MAAM,IAAI,CACnC,CACF,CAAC,CACA,MAAM,CAAC;GAEV,IAAI,QAAQ,WAAW,GAAG;IACxB,KAAK,OAAO,KACV;KAAE;KAAM,WAAW,QAAQ;IAAU,GACrC,qDACF;IACA;GACF;GAEA,MAAM,MAAM,QAAQ;GACpB,UAAU,OAAO,EAAE,WAAW;IAC5B,MAAM,QAAQ,IAAI;IAClB,KAAK,MAAM,WAAW,OACpB,IAAI,QAAQ,WAAW,UAErB,MAAM,KAAK,OAAO,QAAQ,OAAO;SAC5B,IAAI,QAAQ,WAAW,QAC5B,MAAM,KAAK,KAAK,QAAQ,QAAQ;SAC3B,IAAI,QAAQ,WAAW,gBAC5B,MAAM,KAAK,aAAa,QAAQ,OAAO,QAAQ,OAAO;SAEtD,KAAK,OAAO,KACV;KAAE,QAAS,SAAiB;KAAQ;IAAK,GACzC,8BACF;GAGN;EACF;EAEA,MAAM,UAAU,iBAAiB;EACjC,MAAM,YAAY,WAAW;EAQ7B,IAAI,CAAC,MAPkB,KAAK,SAAS,IACnC,SACA,WACA,MACA,2BACA,IACF,GACe;GACb,KAAK,OAAO,KAAK,EAAE,WAAW,GAAG,iDAAiD;GAClF;EACF;EAIA,MAAM,aAAa,kBAAkB;GACnC,KAAU,SACP,KAAK,gBAAgB,GAAG,SAAS,WAAW,OAAO,yBAAyB,CAAC,CAAC,CAC9E,OAAO,QAAiB;IACvB,KAAK,OAAO,KAAK;KAAE;KAAK;IAAW,GAAG,+BAA+B;GACvE,CAAC;EACL,GAAG,sBAAsB;EAEzB,IAAI;GACF,IAAI,YACF,MAAM,KAAK,OACR,OAAO,CAAC,CACR,KAAK,iBAAiB,CAAC,CACvB,MAAM,GAAG,kBAAkB,IAAI,UAAU,CAAC,CAAC,CAC3C,MAAM,CAAC,EAAA,CACV;GACF,IAAI,CAAC,UAWH,YAAW,MAVQ,KAAK,OACrB,OAAO,iBAAiB,CAAC,CACzB,OAAO;IACN,IAAI;IACJ,WAAW,QAAQ;IACb;IACN,QAAQ;IACR,OAAO,QAAQ,SAAS,CAAC;GAC3B,CAAC,CAAC,CACD,UAAU,EAAA,CACG;GAGlB,IAAI,SAAS,WAAW,WAAW;IACjC,KAAK,OAAO,KAAK,EAAE,WAAW,GAAG,mCAAmC;IACpE;GACF;GAEA,MAAM,KAAK,OACR,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,UAAU,CAAC,CAAC,CAC1B,MAAM,GAAG,kBAAkB,IAAI,UAAU,CAAC;GAG7C,MAAM,gBAAgB,MAAM,KAAK,OAC9B,OAAO,CAAC,CACR,KAAK,aAAa,CAAC,CACnB,MAAM,GAAG,cAAc,YAAY,UAAU,CAAC;GACjD,MAAM,gCAAgB,IAAI,IAAiB;GAC3C,KAAK,MAAM,KAAK,eACd,cAAc,IAAI,EAAE,WAAW,EAAE,MAAM;GAGzC,IAAI,mBAAmB;GA+JvB,MAAM,MAAuB;IAC3B,MAAM;KA7JN,QAAQ,OAAO,SAAS;MACtB,MAAM,SAAS,OAAO,kBAAkB;MACxC,IAAI,cAAc,IAAI,MAAM,GAAG,OAAO,cAAc,IAAI,MAAM;MAI9D,MAAM,YAAY,uBAChB,MACA,SAAU,OACV,QAAQ,WACR,MAAM,WAAW,GAAG,QACtB;MAEA,MAAM,SAAS,MAAM,IAAI,SACtB,SAAS,WAAW;OACnB,KAAK,YAAY,KAAK;QACpB,UAAU,KAAK;QACf,OAAO,iBACL,0BACA,WACA,YACA,MAAM,SAAS,OACjB;QACS;QACT;OACF,CAAC;MACH,CACF;MAEA,MAAM,SAAS;OACb,SAAS;OACT,WAAW,OAAO;OAClB,gBAAgB,OAAO;MACzB;MAEA,MAAM,KAAK,OAAO,OAAO,aAAa,CAAC,CAAC,OAAO;OAC7C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACnB,WAAW;OACX,QAAQ;OACR;MACF,CAAC;MAED,OAAO;KACT;KACA,MAAM,OAAO,aAAa;MACxB,MAAM,SAAS,OAAO,kBAAkB;MACxC,IAAI,cAAc,IAAI,MAAM,GAAG;MAG/B,IAAI,KAAK;MACT,IAAI,SAAS,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,IAAI,KAAK,KAAK,KAAK;WAChE,IAAI,SAAS,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,IAAI,KAAK,KAAK;WAChE,IAAI,SAAS,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,IAAI,KAAK;WAC3D,IAAI,SAAS,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,IAAI;WACtD,MAAM,IAAI,MAAM,0BAA0B,UAAU;MAEzD,MAAM,WAAW,KAAK,IAAI,IAAI;MAK9B,MAAM,KAAK,SAAS,KAClB,yBACA,UACA,KAAK,UAAU;OACb,YAAY,SAAU;OACtB,MAAM,SAAU;OAChB,WAAW,QAAQ;OACnB,OAAO,SAAU;MACnB,CAAC,CACH;MAEA,MAAM,KAAK,OAAO,OAAO,aAAa,CAAC,CAAC,OAAO;OAC7C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACnB,WAAW;OACX,QAAQ;OACR,QAAQ,EAAE,aAAa,SAAS;MAClC,CAAC;MAED,MAAM,IAAI,sBAAsB,QAAQ,EAAE,SAAS,CAAC;KACtD;KACA,cAAc,OAAO,WAAW,YAAY;MAC1C,MAAM,SAAS,OAAO,kBAAkB;MACxC,IAAI,cAAc,IAAI,MAAM,GAAG;OAC7B,MAAM,MAAM,cAAc,IAAI,MAAM;OACpC,IAAI,OAAO,OAAO,QAAQ,YAAa,IAAY,aAAa,MAC9D,OAAO;OAET,OAAO;MACT;MAEA,UAAU,WAAW,CAAC;MACtB,QAAQ,UAAU,QAAQ,WAAW;MACrC,QAAQ,QAAQ,QAAQ,SAAS,CAAC;MAElC,IAAI,KAAK;MACT,IAAI,QAAQ,QAAQ,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK;WAC9E,IAAI,QAAQ,QAAQ,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,OAAO,IAAI,KAAK,KAAK;WAC9E,IAAI,QAAQ,QAAQ,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,OAAO,IAAI,KAAK;WACzE,IAAI,QAAQ,QAAQ,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,OAAO,IAAI;WACpE,MAAM,IAAI,MAAM,iCAAiC,QAAQ,SAAS;MAEvE,MAAM,WAAW,KAAK,IAAI,IAAI;MAE9B,MAAM,KAAK,SAAS,KAClB,yBACA,UACA,KAAK,UAAU;OACb,YAAY,SAAU;OACtB,MAAM,SAAU;OAChB,WAAW,QAAQ;OACnB,OAAO,SAAU;OACjB,gBAAgB;OAChB;OACA;MACF,CAAC,CACH;MAGA,MAAM,KAAK,OAAO,OAAO,eAAe,CAAC,CAAC,OAAO;OAC/C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACR;OACX,eAAe,QAAQ;OACvB,WAAW,IAAI,KAAK,QAAQ;MAC9B,CAAC;MAGD,MAAM,KAAK,OAAO,OAAO,aAAa,CAAC,CAAC,OAAO;OAC7C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACnB,WAAW;OACX,QAAQ;OACR,QAAQ;MACV,CAAC;MAED,MAAM,IAAI,sBAAsB,gBAAgB,EAAE,UAAU,CAAC;KAC/D;KACA,KAAK,OAAO,UAAU,OAAO;MAC3B,MAAM,SAAS,OAAO,kBAAkB;MACxC,IAAI,cAAc,IAAI,MAAM,GAAG,OAAO,cAAc,IAAI,MAAM;MAE9D,MAAM,SAAS,MAAM,GAAG;MACxB,MAAM,KAAK,OAAO,OAAO,aAAa,CAAC,CAAC,OAAO;OAC7C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACnB,WAAW;OACX,QAAQ;OACR,QAAQ;MACV,CAAC;MACD,OAAO;KACT;IAIc;IACd,OAAQ,SAAU,SAAiB,EAAE,MAAM,EAAE,IAAI,UAAU,EAAE;GAC/D;GAEA,IAAI;IACF,MAAM,QAAQ,GAAG;IAEjB,MAAM,KAAK,OACR,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,YAAY,CAAC,CAAC,CAC5B,MAAM,GAAG,kBAAkB,IAAI,SAAU,EAAE,CAAC;IAC/C,KAAK,OAAO,KAAK,EAAE,YAAY,SAAU,GAAG,GAAG,iCAAiC;GAClF,SAAS,KAAU;IACjB,IAAI,eAAe,yBAAyB,IAAI,SAAS,yBAAyB;KAChF,MAAM,KAAK,OACR,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,UAAU,CAAC,CAAC,CAC1B,MAAM,GAAG,kBAAkB,IAAI,SAAU,EAAE,CAAC;KAC/C,KAAK,OAAO,KAAK;MAAE,YAAY,SAAU;MAAI,QAAQ,IAAI;KAAO,GAAG,oBAAoB;IACzF,OAAO;KACL,KAAK,OAAO,MAAM;MAAE;MAAK,YAAY,SAAU;KAAG,GAAG,iBAAiB;KACtE,MAAM,KAAK,OACR,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,SAAS,CAAC,CAAC,CACzB,MAAM,GAAG,kBAAkB,IAAI,SAAU,EAAE,CAAC;IACjD;GACF;EACF,UAAU;GACR,cAAc,UAAU;GAExB,MAAM,KAAK,SAAS,KAAK,kBAAkB,GAAG,SAAS,SAAS;EAClE;EAEA,MAAM,QAAQ,IAAI,eAAe;CACnC;AACF;AASA,eAAsB,sBAAsB;CAC1C,SAAS,aAAa;EAAE,MAAM;EAAmB,OAAO,OAAO;CAAU,CAAC;CAC1E,QAAQ,IAAI,YAAY;EAAE,KAAK,OAAO;EAAW,MAAM;EAAY;CAAO,CAAC;CAC3E,MAAM,SAAS,eAAe;EAAE,KAAK,OAAO;EAAc,iBAAiB;EAAY;CAAO,CAAC;CAC/F,QAAM,OAAO;CACb,KAAK,OAAO;CACZ,uBAAuB,IAAI,eAAe;EACxC,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB;CACF,CAAC;CACD,mBAAmB,IAAI,eAAe;EACpC,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB;CACF,CAAC;CACD,WAAW,IAAI,eAAe;EAC5B,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB,OAAO,gBAAgB;EACvB,UAAU,YAAY,QAAQ;EAC9B,WAAW,QAAQ;EACnB,WAAW,OAAO;EAClB;CACF,CAAC;CAED,iBAAiB,IAAI,sBAAsB;EACzC,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB,OAAO,gBAAgB;EACvB,UAAU,YAAY,QAAQ;EAC9B;CACF,CAAC;CAED,SAAS,IAAI,eAAe;EAC1B;EACA;EACA;EACA,aAAa,OAAO;EACpB,OAAO,MAAM;EACb;EACA;EACA;CACF,CAAC;CAGD,eAAe,kBAAkB;EAC/B,CAAM,YAAY;GAChB,IAAI;IACF,MAAM,MAAM,KAAK,IAAI;IACrB,MAAM,QAAS,MAAM,MAAM,OAAO,KAChC,oBACA,GACA,yBACA,KACA,KACA,4BACF;IAEA,KAAK,MAAM,WAAW,OAAO;KAC3B,MAAM,OAAO,KAAK,MAAM,OAAO;KAG/B,MAAM,QACJ,MAAM,GACH,OAAO,CAAC,CACR,KAAK,iBAAiB,CAAC,CACvB,MAAM,GAAG,kBAAkB,IAAI,KAAK,UAAU,CAAC,CAAC,CAChD,MAAM,CAAC,EAAA,CACV;KACF,IAAI,CAAC,QAAQ,KAAK,WAAW,eAAe,KAAK,WAAW,UAAU;MACpE,MAAM,MAAM,OAAO,KAAK,yBAAyB,OAAO;MACxD;KACF;KAKA,IAAI,KAAK,WAAW,WAAW;KAE/B,IAAI,KAAK,gBAAgB;MAcvB,KAAI,MAZkB,GACnB,OAAO,eAAe,CAAC,CACvB,MACC,IACE,GAAG,gBAAgB,YAAY,KAAK,UAAU,GAC9C,GAAG,gBAAgB,WAAW,KAAK,SAAS,CAC9C,CACF,CAAC,CACA,UAAU,EAAA,CAID,WAAW,GAAG;OACxB,MAAM,MAAM,OAAO,KAAK,yBAAyB,OAAO;OACxD;MACF;MAYA,MAAM,eAAc,MATA,GACjB,OAAO,CAAC,CACR,KAAK,aAAa,CAAC,CACnB,MACC,IACE,GAAG,cAAc,YAAY,KAAK,UAAU,GAC5C,GAAG,cAAc,QAAQ,cAAc,CACzC,CACF,EAAA,CACwB,MAAM,MAAW,EAAE,WAAW,IAAI;MAC5D,IAAI,aACF,MAAM,GACH,OAAO,aAAa,CAAC,CACrB,IAAI,EAAE,QAAQ,EAAE,UAAU,KAAK,EAAE,CAAC,CAAC,CACnC,MAAM,GAAG,cAAc,IAAI,YAAY,EAAE,CAAC;KAEjD;KAEA,MAAM,iBAAiB,QACrB,iBAAiB,oBAAoB,MAAM,aAAa,KAAA,CAAS,CACnE;KACA,MAAM,MAAM,OAAO,KAAK,yBAAyB,OAAO;KACxD,OAAO,KAAK,EAAE,YAAY,KAAK,WAAW,GAAG,6BAA6B;IAC5E;GACF,SAAS,KAAK;IACZ,OAAO,MAAM,EAAE,IAAI,GAAG,gCAAgC;GACxD;EACF,EAAA,CAAG;CACL,GAAG,GAAI;CAEP,iBAAiB,kBAAkB;EACjC,CAAM,YAAY;GAChB,IAAI;IACF,MAAM,mBAAmB,MAAM,GAC5B,OAAO,EAAE,IAAI,kBAAkB,GAAG,CAAC,CAAC,CACpC,KAAK,iBAAiB,CAAC,CACvB,MAAM,GAAG,kBAAkB,QAAQ,SAAS,CAAC;IAChD,KAAK,MAAM,QAAQ,kBAAkB;KACnC,MAAM,UAAU,iBAAiB,KAAK;KAEtC,IAAI,CAAC,MADiB,MAAM,OAAO,OAAO,OAAO,GACnC;MACZ,MAAM,GACH,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,UAAU,CAAC,CAAC,CAC1B,MAAM,GAAG,kBAAkB,IAAI,KAAK,EAAE,CAAC;MAC1C,OAAO,KAAK,EAAE,YAAY,KAAK,GAAG,GAAG,+CAA+C;KACtF;IACF;GACF,SAAS,KAAK;IACZ,OAAO,MAAM,EAAE,IAAI,GAAG,gCAAgC;GACxD;EACF,EAAA,CAAG;CACL,GAAG,GAAK;CAER,iBAAiB,oBAAoB,YAAY,QAAQ,OAAO,MAAM;CAEtE,OAAO,KAAK,0BAA0B;CACtC,MAAM,OAAO,MAAM;AACrB;AAEA,eAAsB,qBAAoC;CACxD,QAAQ,KAAK,oBAAoB;CACjC,IAAI,gBAAgB,cAAc,cAAc;CAChD,IAAI,cAAc,cAAc,YAAY;CAC5C,IAAI,gBAAgB,cAAc,cAAc;CAChD,IAAI,QAAQ,MAAM,OAAO,KAAK;CAC9B,IAAIA,OAAK,MAAMA,MAAI,IAAI;CACvB,IAAI,OAAO,MAAM,MAAM,WAAW;CAClC,QAAQ,KAAK,yBAAyB;AACxC"}
+{"version":3,"file":"main-D19Wxs0x.mjs","names":["sql"],"sources":["../src/services/workflow/main.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { loadEnv, readBaseConfig } from \"@/index.js\";\nimport { createLogger } from \"@/index.js\";\nimport { RedisClient, type Redis } from \"@/index.js\";\nimport {\n  StreamConsumer,\n  PendingMessageScanner,\n  StreamProducer,\n  type StreamMessage,\n} from \"@/index.js\";\nimport { BaseWorker } from \"@/index.js\";\nimport { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from \"@/index.js\";\nimport { type StreamName } from \"@/contracts/streams.js\";\nimport { createDatabase } from \"@/db/index.js\";\nimport {\n  workflowInstances,\n  workflowSteps,\n  workflowWaiters,\n  workflowDefinitions,\n} from \"@/db/schema.js\";\nimport { eq, and } from \"drizzle-orm\";\nimport {\n  workflowRegistry,\n  SuspendExecutionError,\n  buildStepNotifyPayload,\n  type WorkflowContext,\n  type WorkflowStepContext,\n} from \"@/workflows/index.js\";\nimport {\n  type WorkerOptions,\n  LUA_SCHEDULER_POLL,\n  LUA_RELEASE_LOCK,\n  LUA_RENEW_LOCK,\n} from \"@/shared/index.js\";\nimport { startHealthReporter } from \"@/workers/index.js\";\n\n/** How long a workflow instance lock is held before it self-expires. */\nconst WORKFLOW_LOCK_TTL_SECONDS = 60;\n/** Renew the lock well inside its TTL so long-running handlers keep it. */\nconst WORKFLOW_LOCK_RENEW_MS = (WORKFLOW_LOCK_TTL_SECONDS / 3) * 1000;\n/** How long a claimed workflow timer stays invisible to other pollers. */\nconst WORKFLOW_TIMER_VISIBILITY_MS = 60_000;\n\nloadEnv();\nconst config = readBaseConfig();\nlet logger: ReturnType<typeof createLogger>;\nlet redis: RedisClient;\nlet sql: any;\nlet db: any;\n\nlet consumer: StreamConsumer;\nlet pendingScanner: PendingMessageScanner;\nlet worker: BaseWorker;\nlet healthInterval: NodeJS.Timeout | null = null;\nlet pollInterval: NodeJS.Timeout | null = null;\nlet reaperInterval: NodeJS.Timeout | null = null;\n\nlet notificationProducer: StreamProducer;\nlet workflowProducer: StreamProducer;\n\nexport interface WorkflowWorkerOptions extends WorkerOptions {\n  redis: Redis;\n  db: any;\n  workflowProducer: any;\n  notificationProducer: any;\n}\n\nexport class WorkflowWorker extends BaseWorker {\n  private readonly redisCli: Redis;\n  private readonly dbConn: any;\n  private readonly workflowProducer: any;\n  private readonly notificationProducer: any;\n\n  private eventBuffer: {\n    producer: any;\n    event: any;\n    resolve: (result?: { messageId: string; notificationId: string }) => void;\n    reject: (err: any) => void;\n  }[] = [];\n  private flushTimer: NodeJS.Timeout | null = null;\n\n  constructor(options: WorkflowWorkerOptions) {\n    super(options);\n    this.redisCli = options.redis;\n    this.dbConn = options.db;\n    this.workflowProducer = options.workflowProducer;\n    this.notificationProducer = options.notificationProducer;\n\n    this.flushTimer = setInterval(() => void this.flushWorkerBuffers(), 100);\n  }\n\n  override async stop(): Promise<void> {\n    if (this.flushTimer) {\n      clearInterval(this.flushTimer);\n      this.flushTimer = null;\n    }\n    await this.flushWorkerBuffers();\n    await super.stop();\n  }\n\n  private async flushWorkerBuffers(): Promise<void> {\n    if (this.eventBuffer.length === 0) return;\n\n    const events = this.eventBuffer;\n    this.eventBuffer = [];\n\n    try {\n      const byProducer = new Map<any, typeof events>();\n      for (const item of events) {\n        if (!byProducer.has(item.producer)) byProducer.set(item.producer, []);\n        byProducer.get(item.producer)!.push(item);\n      }\n\n      for (const [producer, batch] of byProducer) {\n        const { messageIds, eventIds } = await producer.publishBatch(batch.map((b) => b.event));\n        for (let i = 0; i < batch.length; i++) {\n          const mId = messageIds[i];\n          const eId = eventIds[i];\n          if (mId && eId) batch[i]!.resolve({ messageId: mId, notificationId: eId });\n        }\n      }\n    } catch (err: any) {\n      this.logger.error({ err }, \"failed to flush workflow worker buffer\");\n      for (const e of events) e.reject(err);\n    }\n  }\n\n  async process(message: StreamMessage): Promise<void> {\n    const { event } = message;\n    const publishPromises: Promise<void>[] = [];\n\n    if (event.type !== \"workflow.triggered\" && event.type !== \"workflow.resumed\") {\n      return;\n    }\n\n    const payload = event.payload as any;\n    const name = payload.name;\n    const instanceId = payload.instanceId;\n\n    if (!name || !instanceId || !payload.projectId) {\n      this.logger.warn(\"Missing name, instanceId, or projectId in workflow event\");\n      return;\n    }\n\n    let handler = workflowRegistry.get(name);\n    if (!handler) {\n      // Fallback: check dynamic JSON workflows\n      const defRows = await this.dbConn\n        .select()\n        .from(workflowDefinitions)\n        .where(\n          and(\n            eq(workflowDefinitions.projectId, payload.projectId),\n            eq(workflowDefinitions.name, name),\n          ),\n        )\n        .limit(1);\n\n      if (defRows.length === 0) {\n        this.logger.warn(\n          { name, projectId: payload.projectId },\n          \"No handler or dynamic definition found for workflow\",\n        );\n        return;\n      }\n\n      const def = defRows[0];\n      handler = async ({ step }) => {\n        const steps = def.steps as any[];\n        for (const stepDef of steps) {\n          if (stepDef.action === \"notify\") {\n            // A payload target wins; naming none inherits the instance user.\n            await step.notify(stepDef.payload);\n          } else if (stepDef.action === \"wait\") {\n            await step.wait(stepDef.duration);\n          } else if (stepDef.action === \"waitForEvent\") {\n            await step.waitForEvent(stepDef.event, stepDef.options);\n          } else {\n            this.logger.warn(\n              { action: (stepDef as any)?.action, name },\n              \"unknown workflow step action\",\n            );\n          }\n        }\n      };\n    }\n\n    const lockKey = `lock:workflow:${instanceId}`;\n    const lockToken = randomUUID();\n    const acquired = await this.redisCli.set(\n      lockKey,\n      lockToken,\n      \"EX\",\n      WORKFLOW_LOCK_TTL_SECONDS,\n      \"NX\",\n    );\n    if (!acquired) {\n      this.logger.info({ instanceId }, \"Workflow is locked by another process, skipping\");\n      return;\n    }\n\n    // Keep the lock alive while the handler runs; without this a handler that\n    // outlives the TTL lets a second resume execute the same steps in parallel.\n    const renewTimer = setInterval(() => {\n      void this.redisCli\n        .eval(LUA_RENEW_LOCK, 1, lockKey, lockToken, String(WORKFLOW_LOCK_TTL_SECONDS))\n        .catch((err: unknown) => {\n          this.logger.warn({ err, instanceId }, \"failed to renew workflow lock\");\n        });\n    }, WORKFLOW_LOCK_RENEW_MS);\n\n    try {\n      let instance = (\n        await this.dbConn\n          .select()\n          .from(workflowInstances)\n          .where(eq(workflowInstances.id, instanceId))\n          .limit(1)\n      )[0];\n      if (!instance) {\n        const rows = await this.dbConn\n          .insert(workflowInstances)\n          .values({\n            id: instanceId,\n            projectId: payload.projectId,\n            name: name,\n            status: \"pending\",\n            input: payload.input || {},\n          })\n          .returning();\n        instance = rows[0]!;\n      }\n\n      if (instance.status !== \"pending\") {\n        this.logger.info({ instanceId }, \"Workflow is not pending, skipping\");\n        return;\n      }\n\n      await this.dbConn\n        .update(workflowInstances)\n        .set({ status: \"running\" })\n        .where(eq(workflowInstances.id, instanceId));\n\n      // Load existing steps\n      const existingSteps = await this.dbConn\n        .select()\n        .from(workflowSteps)\n        .where(eq(workflowSteps.instanceId, instanceId));\n      const stepOutputMap = new Map<string, any>();\n      for (const s of existingSteps) {\n        stepOutputMap.set(s.stepIndex, s.output);\n      }\n\n      let currentStepIndex = 0;\n\n      const stepProxy: WorkflowStepContext = {\n        notify: async (args) => {\n          const stepId = String(currentStepIndex++);\n          if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);\n\n          // The step payload is the same shape as notify(), but the wire event\n          // is not — translate rather than spread.\n          const requested = buildStepNotifyPayload(\n            args,\n            instance!.input,\n            payload.projectId,\n            `wf-${instanceId}-${stepId}`,\n          );\n\n          const result = await new Promise<{ messageId: string; notificationId: string }>(\n            (resolve, reject) => {\n              this.eventBuffer.push({\n                producer: this.notificationProducer,\n                event: buildStreamEvent(\n                  \"notification.requested\",\n                  requested as unknown as Record<string, unknown>,\n                  \"workflow\",\n                  event.metadata.traceId,\n                ),\n                resolve: resolve as any,\n                reject,\n              });\n            },\n          );\n\n          const output = {\n            success: true,\n            messageId: result.messageId,\n            notificationId: result.notificationId,\n          };\n\n          await this.dbConn.insert(workflowSteps).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            stepIndex: stepId,\n            action: \"notify\",\n            output,\n          });\n\n          return output;\n        },\n        wait: async (duration) => {\n          const stepId = String(currentStepIndex++);\n          if (stepOutputMap.has(stepId)) return;\n\n          // Simple duration parse (e.g. '2h' -> ms)\n          let ms = 0;\n          if (duration.endsWith(\"d\")) ms = parseInt(duration) * 24 * 60 * 60 * 1000;\n          else if (duration.endsWith(\"h\")) ms = parseInt(duration) * 60 * 60 * 1000;\n          else if (duration.endsWith(\"m\")) ms = parseInt(duration) * 60 * 1000;\n          else if (duration.endsWith(\"s\")) ms = parseInt(duration) * 1000;\n          else throw new Error(`Invalid wait duration: ${duration}`);\n\n          const resumeAt = Date.now() + ms;\n\n          // Persist the wake-up signal before committing the suspended state.\n          // If the process dies after this point the source event is retried;\n          // if it dies after the database write, the timer is already durable.\n          await this.redisCli.zadd(\n            \"notif:workflow:timers\",\n            resumeAt,\n            JSON.stringify({\n              instanceId: instance!.id,\n              name: instance!.name,\n              projectId: payload.projectId,\n              input: instance!.input,\n            }),\n          );\n\n          await this.dbConn.insert(workflowSteps).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            stepIndex: stepId,\n            action: \"wait\",\n            output: { scheduledAt: resumeAt },\n          });\n\n          throw new SuspendExecutionError(\"wait\", { duration });\n        },\n        waitForEvent: async (eventName, options) => {\n          const stepId = String(currentStepIndex++);\n          if (stepOutputMap.has(stepId)) {\n            const out = stepOutputMap.get(stepId);\n            if (out && typeof out === \"object\" && (out as any).timedOut === true) {\n              return null;\n            }\n            return out;\n          }\n\n          options = options || {};\n          options.timeout = options.timeout || \"24h\";\n          options.match = options.match || {};\n\n          let ms = 0;\n          if (options.timeout.endsWith(\"d\")) ms = parseInt(options.timeout) * 24 * 60 * 60 * 1000;\n          else if (options.timeout.endsWith(\"h\")) ms = parseInt(options.timeout) * 60 * 60 * 1000;\n          else if (options.timeout.endsWith(\"m\")) ms = parseInt(options.timeout) * 60 * 1000;\n          else if (options.timeout.endsWith(\"s\")) ms = parseInt(options.timeout) * 1000;\n          else throw new Error(`Invalid waitForEvent timeout: ${options.timeout}`);\n\n          const resumeAt = Date.now() + ms;\n\n          await this.redisCli.zadd(\n            \"notif:workflow:timers\",\n            resumeAt,\n            JSON.stringify({\n              instanceId: instance!.id,\n              name: instance!.name,\n              projectId: payload.projectId,\n              input: instance!.input,\n              isEventTimeout: true,\n              eventName,\n              stepId,\n            }),\n          );\n\n          // Register waiter\n          await this.dbConn.insert(workflowWaiters).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            eventName: eventName,\n            matchCriteria: options.match,\n            expiresAt: new Date(resumeAt),\n          });\n\n          // Register step as pending event\n          await this.dbConn.insert(workflowSteps).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            stepIndex: stepId,\n            action: \"waitForEvent\",\n            output: null, // this will be updated by event worker or timeout\n          });\n\n          throw new SuspendExecutionError(\"waitForEvent\", { eventName });\n        },\n        run: async (stepName, fn) => {\n          const stepId = String(currentStepIndex++);\n          if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);\n\n          const result = await fn();\n          await this.dbConn.insert(workflowSteps).values({\n            instanceId: instance!.id,\n            projectId: payload.projectId,\n            stepIndex: stepId,\n            action: \"run\",\n            output: result,\n          });\n          return result;\n        },\n      };\n\n      const ctx: WorkflowContext = {\n        step: stepProxy,\n        event: (instance!.input as any) || { user: { id: \"unknown\" } },\n      };\n\n      try {\n        await handler(ctx);\n        // If we reach here, workflow completed\n        await this.dbConn\n          .update(workflowInstances)\n          .set({ status: \"completed\" })\n          .where(eq(workflowInstances.id, instance!.id));\n        this.logger.info({ instanceId: instance!.id }, \"Workflow completed successfully\");\n      } catch (err: any) {\n        if (err instanceof SuspendExecutionError || err.name === \"SuspendExecutionError\") {\n          await this.dbConn\n            .update(workflowInstances)\n            .set({ status: \"pending\" })\n            .where(eq(workflowInstances.id, instance!.id));\n          this.logger.info({ instanceId: instance!.id, reason: err.reason }, \"Workflow suspended\");\n        } else {\n          this.logger.error({ err, instanceId: instance!.id }, \"Workflow failed\");\n          await this.dbConn\n            .update(workflowInstances)\n            .set({ status: \"failed\" })\n            .where(eq(workflowInstances.id, instance!.id));\n        }\n      }\n    } finally {\n      clearInterval(renewTimer);\n      // Compare-and-delete: never release a lock a later process re-acquired.\n      await this.redisCli.eval(LUA_RELEASE_LOCK, 1, lockKey, lockToken);\n    }\n\n    await Promise.all(publishPromises);\n  }\n}\n\nexport function __injectForTests(r: any, d: any, wp: any, np: any) {\n  redis = r;\n  db = d;\n  workflowProducer = wp;\n  notificationProducer = np;\n}\n\nexport async function startWorkflowWorker() {\n  logger = createLogger({ name: \"workflow-worker\", level: config.LOG_LEVEL });\n  redis = new RedisClient({ url: config.REDIS_URL, name: \"workflow\", logger });\n  const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: \"workflow\", logger });\n  sql = dbData.sql;\n  db = dbData.db;\n  notificationProducer = new StreamProducer({\n    redis: redis.native,\n    stream: STREAMS.INBOUND_NORMAL,\n    logger,\n  });\n  workflowProducer = new StreamProducer({\n    redis: redis.native,\n    stream: STREAMS.WORKFLOW_INBOUND,\n    logger,\n  });\n  consumer = new StreamConsumer({\n    redis: redis.native,\n    stream: STREAMS.WORKFLOW_INBOUND as StreamName,\n    group: CONSUMER_GROUPS.WORKFLOW as any,\n    consumer: `workflow-${process.pid}`,\n    dlqStream: STREAMS.DEAD_LETTER,\n    batchSize: config.WORKER_CONCURRENCY,\n    logger,\n  });\n\n  pendingScanner = new PendingMessageScanner({\n    redis: redis.native,\n    stream: STREAMS.WORKFLOW_INBOUND as StreamName,\n    group: CONSUMER_GROUPS.WORKFLOW as any,\n    consumer: `workflow-${process.pid}`,\n    logger,\n  });\n\n  worker = new WorkflowWorker({\n    consumer,\n    pendingScanner,\n    logger,\n    concurrency: config.WORKER_CONCURRENCY,\n    redis: redis.native,\n    db,\n    workflowProducer,\n    notificationProducer,\n  });\n\n  // Polling loop for timers\n  pollInterval = setInterval(() => {\n    void (async () => {\n      try {\n        const now = Date.now();\n        const tasks = (await redis.native.eval(\n          LUA_SCHEDULER_POLL,\n          1,\n          \"notif:workflow:timers\",\n          now,\n          100,\n          WORKFLOW_TIMER_VISIBILITY_MS,\n        )) as string[];\n\n        for (const taskStr of tasks) {\n          const task = JSON.parse(taskStr);\n\n          // Check if workflow is already completed/failed\n          const inst = (\n            await db\n              .select()\n              .from(workflowInstances)\n              .where(eq(workflowInstances.id, task.instanceId))\n              .limit(1)\n          )[0];\n          if (!inst || inst.status === \"completed\" || inst.status === \"failed\") {\n            await redis.native.zrem(\"notif:workflow:timers\", taskStr);\n            continue;\n          }\n          // A timer can be observed while the worker is still persisting the\n          // corresponding suspension. Leave the claimed member in place; its\n          // visibility timeout will make it eligible once the instance becomes\n          // pending instead of losing the wake-up signal.\n          if (inst.status === \"running\") continue;\n\n          if (task.isEventTimeout) {\n            // It's a timeout for waitForEvent. Clean up only this step's waiter\n            const deleted = await db\n              .delete(workflowWaiters)\n              .where(\n                and(\n                  eq(workflowWaiters.instanceId, task.instanceId),\n                  eq(workflowWaiters.eventName, task.eventName),\n                ),\n              )\n              .returning();\n\n            // If the waiter was already deleted (by EventWorker when event arrived before timeout),\n            // this timer is stale. Clean up from Redis and skip re-resuming the workflow.\n            if (deleted.length === 0) {\n              await redis.native.zrem(\"notif:workflow:timers\", taskStr);\n              continue;\n            }\n\n            // Record timedOut state on the pending waitForEvent step\n            const steps = await db\n              .select()\n              .from(workflowSteps)\n              .where(\n                and(\n                  eq(workflowSteps.instanceId, task.instanceId),\n                  eq(workflowSteps.action, \"waitForEvent\"),\n                ),\n              );\n            const pendingStep = steps.find((s: any) => s.output === null);\n            if (pendingStep) {\n              await db\n                .update(workflowSteps)\n                .set({ output: { timedOut: true } })\n                .where(eq(workflowSteps.id, pendingStep.id));\n            }\n          }\n\n          await workflowProducer.publish(\n            buildStreamEvent(\"workflow.resumed\", task, \"scheduler\", undefined),\n          );\n          await redis.native.zrem(\"notif:workflow:timers\", taskStr);\n          logger.info({ instanceId: task.instanceId }, \"Workflow resumed from timer\");\n        }\n      } catch (err) {\n        logger.error({ err }, \"error in workflow polling loop\");\n      }\n    })();\n  }, 5000);\n\n  reaperInterval = setInterval(() => {\n    void (async () => {\n      try {\n        const runningInstances = await db\n          .select({ id: workflowInstances.id })\n          .from(workflowInstances)\n          .where(eq(workflowInstances.status, \"running\"));\n        for (const inst of runningInstances) {\n          const lockKey = `lock:workflow:${inst.id}`;\n          const hasLock = await redis.native.exists(lockKey);\n          if (!hasLock) {\n            await db\n              .update(workflowInstances)\n              .set({ status: \"pending\" })\n              .where(eq(workflowInstances.id, inst.id));\n            logger.info({ instanceId: inst.id }, \"Reaped stuck workflow instance (lock expired)\");\n          }\n        }\n      } catch (err) {\n        logger.error({ err }, \"error in stuck workflow reaper\");\n      }\n    })();\n  }, 60000);\n\n  healthInterval = startHealthReporter(\"workflow\", worker, redis, logger);\n\n  logger.info(\"workflow worker starting\");\n  await worker.start();\n}\n\nexport async function stopWorkflowWorker(): Promise<void> {\n  logger?.info(\"shutdown initiated\");\n  if (healthInterval) clearInterval(healthInterval);\n  if (pollInterval) clearInterval(pollInterval);\n  if (reaperInterval) clearInterval(reaperInterval);\n  if (worker) await worker.stop();\n  if (sql) await sql.end();\n  if (redis) await redis.disconnect();\n  logger?.info(\"workflow worker stopped\");\n}\n"],"mappings":";;;;;AAqCA,MAAM,4BAA4B;;AAElC,MAAM,yBAA0B,4BAA4B,IAAK;;AAEjE,MAAM,+BAA+B;AAErC,QAAQ;AACR,MAAM,SAAS,eAAe;AAC9B,IAAI;AACJ,IAAI;AACJ,IAAIA;AACJ,IAAI;AAEJ,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI,iBAAwC;AAC5C,IAAI,eAAsC;AAC1C,IAAI,iBAAwC;AAE5C,IAAI;AACJ,IAAI;AASJ,IAAa,iBAAb,cAAoC,WAAW;CAC7C;CACA;CACA;CACA;CAEA,cAKM,CAAC;CACP,aAA4C;CAE5C,YAAY,SAAgC;EAC1C,MAAM,OAAO;EACb,KAAK,WAAW,QAAQ;EACxB,KAAK,SAAS,QAAQ;EACtB,KAAK,mBAAmB,QAAQ;EAChC,KAAK,uBAAuB,QAAQ;EAEpC,KAAK,aAAa,kBAAkB,KAAK,KAAK,mBAAmB,GAAG,GAAG;CACzE;CAEA,MAAe,OAAsB;EACnC,IAAI,KAAK,YAAY;GACnB,cAAc,KAAK,UAAU;GAC7B,KAAK,aAAa;EACpB;EACA,MAAM,KAAK,mBAAmB;EAC9B,MAAM,MAAM,KAAK;CACnB;CAEA,MAAc,qBAAoC;EAChD,IAAI,KAAK,YAAY,WAAW,GAAG;EAEnC,MAAM,SAAS,KAAK;EACpB,KAAK,cAAc,CAAC;EAEpB,IAAI;GACF,MAAM,6BAAa,IAAI,IAAwB;GAC/C,KAAK,MAAM,QAAQ,QAAQ;IACzB,IAAI,CAAC,WAAW,IAAI,KAAK,QAAQ,GAAG,WAAW,IAAI,KAAK,UAAU,CAAC,CAAC;IACpE,WAAW,IAAI,KAAK,QAAQ,CAAC,CAAE,KAAK,IAAI;GAC1C;GAEA,KAAK,MAAM,CAAC,UAAU,UAAU,YAAY;IAC1C,MAAM,EAAE,YAAY,aAAa,MAAM,SAAS,aAAa,MAAM,KAAK,MAAM,EAAE,KAAK,CAAC;IACtF,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;KACrC,MAAM,MAAM,WAAW;KACvB,MAAM,MAAM,SAAS;KACrB,IAAI,OAAO,KAAK,MAAM,EAAE,CAAE,QAAQ;MAAE,WAAW;MAAK,gBAAgB;KAAI,CAAC;IAC3E;GACF;EACF,SAAS,KAAU;GACjB,KAAK,OAAO,MAAM,EAAE,IAAI,GAAG,wCAAwC;GACnE,KAAK,MAAM,KAAK,QAAQ,EAAE,OAAO,GAAG;EACtC;CACF;CAEA,MAAM,QAAQ,SAAuC;EACnD,MAAM,EAAE,UAAU;EAClB,MAAM,kBAAmC,CAAC;EAE1C,IAAI,MAAM,SAAS,wBAAwB,MAAM,SAAS,oBACxD;EAGF,MAAM,UAAU,MAAM;EACtB,MAAM,OAAO,QAAQ;EACrB,MAAM,aAAa,QAAQ;EAE3B,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,WAAW;GAC9C,KAAK,OAAO,KAAK,0DAA0D;GAC3E;EACF;EAEA,IAAI,UAAU,iBAAiB,IAAI,IAAI;EACvC,IAAI,CAAC,SAAS;GAEZ,MAAM,UAAU,MAAM,KAAK,OACxB,OAAO,CAAC,CACR,KAAK,mBAAmB,CAAC,CACzB,MACC,IACE,GAAG,oBAAoB,WAAW,QAAQ,SAAS,GACnD,GAAG,oBAAoB,MAAM,IAAI,CACnC,CACF,CAAC,CACA,MAAM,CAAC;GAEV,IAAI,QAAQ,WAAW,GAAG;IACxB,KAAK,OAAO,KACV;KAAE;KAAM,WAAW,QAAQ;IAAU,GACrC,qDACF;IACA;GACF;GAEA,MAAM,MAAM,QAAQ;GACpB,UAAU,OAAO,EAAE,WAAW;IAC5B,MAAM,QAAQ,IAAI;IAClB,KAAK,MAAM,WAAW,OACpB,IAAI,QAAQ,WAAW,UAErB,MAAM,KAAK,OAAO,QAAQ,OAAO;SAC5B,IAAI,QAAQ,WAAW,QAC5B,MAAM,KAAK,KAAK,QAAQ,QAAQ;SAC3B,IAAI,QAAQ,WAAW,gBAC5B,MAAM,KAAK,aAAa,QAAQ,OAAO,QAAQ,OAAO;SAEtD,KAAK,OAAO,KACV;KAAE,QAAS,SAAiB;KAAQ;IAAK,GACzC,8BACF;GAGN;EACF;EAEA,MAAM,UAAU,iBAAiB;EACjC,MAAM,YAAY,WAAW;EAQ7B,IAAI,CAAC,MAPkB,KAAK,SAAS,IACnC,SACA,WACA,MACA,2BACA,IACF,GACe;GACb,KAAK,OAAO,KAAK,EAAE,WAAW,GAAG,iDAAiD;GAClF;EACF;EAIA,MAAM,aAAa,kBAAkB;GACnC,KAAU,SACP,KAAK,gBAAgB,GAAG,SAAS,WAAW,OAAO,yBAAyB,CAAC,CAAC,CAC9E,OAAO,QAAiB;IACvB,KAAK,OAAO,KAAK;KAAE;KAAK;IAAW,GAAG,+BAA+B;GACvE,CAAC;EACL,GAAG,sBAAsB;EAEzB,IAAI;GACF,IAAI,YACF,MAAM,KAAK,OACR,OAAO,CAAC,CACR,KAAK,iBAAiB,CAAC,CACvB,MAAM,GAAG,kBAAkB,IAAI,UAAU,CAAC,CAAC,CAC3C,MAAM,CAAC,EAAA,CACV;GACF,IAAI,CAAC,UAWH,YAAW,MAVQ,KAAK,OACrB,OAAO,iBAAiB,CAAC,CACzB,OAAO;IACN,IAAI;IACJ,WAAW,QAAQ;IACb;IACN,QAAQ;IACR,OAAO,QAAQ,SAAS,CAAC;GAC3B,CAAC,CAAC,CACD,UAAU,EAAA,CACG;GAGlB,IAAI,SAAS,WAAW,WAAW;IACjC,KAAK,OAAO,KAAK,EAAE,WAAW,GAAG,mCAAmC;IACpE;GACF;GAEA,MAAM,KAAK,OACR,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,UAAU,CAAC,CAAC,CAC1B,MAAM,GAAG,kBAAkB,IAAI,UAAU,CAAC;GAG7C,MAAM,gBAAgB,MAAM,KAAK,OAC9B,OAAO,CAAC,CACR,KAAK,aAAa,CAAC,CACnB,MAAM,GAAG,cAAc,YAAY,UAAU,CAAC;GACjD,MAAM,gCAAgB,IAAI,IAAiB;GAC3C,KAAK,MAAM,KAAK,eACd,cAAc,IAAI,EAAE,WAAW,EAAE,MAAM;GAGzC,IAAI,mBAAmB;GA+JvB,MAAM,MAAuB;IAC3B,MAAM;KA7JN,QAAQ,OAAO,SAAS;MACtB,MAAM,SAAS,OAAO,kBAAkB;MACxC,IAAI,cAAc,IAAI,MAAM,GAAG,OAAO,cAAc,IAAI,MAAM;MAI9D,MAAM,YAAY,uBAChB,MACA,SAAU,OACV,QAAQ,WACR,MAAM,WAAW,GAAG,QACtB;MAEA,MAAM,SAAS,MAAM,IAAI,SACtB,SAAS,WAAW;OACnB,KAAK,YAAY,KAAK;QACpB,UAAU,KAAK;QACf,OAAO,iBACL,0BACA,WACA,YACA,MAAM,SAAS,OACjB;QACS;QACT;OACF,CAAC;MACH,CACF;MAEA,MAAM,SAAS;OACb,SAAS;OACT,WAAW,OAAO;OAClB,gBAAgB,OAAO;MACzB;MAEA,MAAM,KAAK,OAAO,OAAO,aAAa,CAAC,CAAC,OAAO;OAC7C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACnB,WAAW;OACX,QAAQ;OACR;MACF,CAAC;MAED,OAAO;KACT;KACA,MAAM,OAAO,aAAa;MACxB,MAAM,SAAS,OAAO,kBAAkB;MACxC,IAAI,cAAc,IAAI,MAAM,GAAG;MAG/B,IAAI,KAAK;MACT,IAAI,SAAS,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,IAAI,KAAK,KAAK,KAAK;WAChE,IAAI,SAAS,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,IAAI,KAAK,KAAK;WAChE,IAAI,SAAS,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,IAAI,KAAK;WAC3D,IAAI,SAAS,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,IAAI;WACtD,MAAM,IAAI,MAAM,0BAA0B,UAAU;MAEzD,MAAM,WAAW,KAAK,IAAI,IAAI;MAK9B,MAAM,KAAK,SAAS,KAClB,yBACA,UACA,KAAK,UAAU;OACb,YAAY,SAAU;OACtB,MAAM,SAAU;OAChB,WAAW,QAAQ;OACnB,OAAO,SAAU;MACnB,CAAC,CACH;MAEA,MAAM,KAAK,OAAO,OAAO,aAAa,CAAC,CAAC,OAAO;OAC7C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACnB,WAAW;OACX,QAAQ;OACR,QAAQ,EAAE,aAAa,SAAS;MAClC,CAAC;MAED,MAAM,IAAI,sBAAsB,QAAQ,EAAE,SAAS,CAAC;KACtD;KACA,cAAc,OAAO,WAAW,YAAY;MAC1C,MAAM,SAAS,OAAO,kBAAkB;MACxC,IAAI,cAAc,IAAI,MAAM,GAAG;OAC7B,MAAM,MAAM,cAAc,IAAI,MAAM;OACpC,IAAI,OAAO,OAAO,QAAQ,YAAa,IAAY,aAAa,MAC9D,OAAO;OAET,OAAO;MACT;MAEA,UAAU,WAAW,CAAC;MACtB,QAAQ,UAAU,QAAQ,WAAW;MACrC,QAAQ,QAAQ,QAAQ,SAAS,CAAC;MAElC,IAAI,KAAK;MACT,IAAI,QAAQ,QAAQ,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK;WAC9E,IAAI,QAAQ,QAAQ,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,OAAO,IAAI,KAAK,KAAK;WAC9E,IAAI,QAAQ,QAAQ,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,OAAO,IAAI,KAAK;WACzE,IAAI,QAAQ,QAAQ,SAAS,GAAG,GAAG,KAAK,SAAS,QAAQ,OAAO,IAAI;WACpE,MAAM,IAAI,MAAM,iCAAiC,QAAQ,SAAS;MAEvE,MAAM,WAAW,KAAK,IAAI,IAAI;MAE9B,MAAM,KAAK,SAAS,KAClB,yBACA,UACA,KAAK,UAAU;OACb,YAAY,SAAU;OACtB,MAAM,SAAU;OAChB,WAAW,QAAQ;OACnB,OAAO,SAAU;OACjB,gBAAgB;OAChB;OACA;MACF,CAAC,CACH;MAGA,MAAM,KAAK,OAAO,OAAO,eAAe,CAAC,CAAC,OAAO;OAC/C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACR;OACX,eAAe,QAAQ;OACvB,WAAW,IAAI,KAAK,QAAQ;MAC9B,CAAC;MAGD,MAAM,KAAK,OAAO,OAAO,aAAa,CAAC,CAAC,OAAO;OAC7C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACnB,WAAW;OACX,QAAQ;OACR,QAAQ;MACV,CAAC;MAED,MAAM,IAAI,sBAAsB,gBAAgB,EAAE,UAAU,CAAC;KAC/D;KACA,KAAK,OAAO,UAAU,OAAO;MAC3B,MAAM,SAAS,OAAO,kBAAkB;MACxC,IAAI,cAAc,IAAI,MAAM,GAAG,OAAO,cAAc,IAAI,MAAM;MAE9D,MAAM,SAAS,MAAM,GAAG;MACxB,MAAM,KAAK,OAAO,OAAO,aAAa,CAAC,CAAC,OAAO;OAC7C,YAAY,SAAU;OACtB,WAAW,QAAQ;OACnB,WAAW;OACX,QAAQ;OACR,QAAQ;MACV,CAAC;MACD,OAAO;KACT;IAIc;IACd,OAAQ,SAAU,SAAiB,EAAE,MAAM,EAAE,IAAI,UAAU,EAAE;GAC/D;GAEA,IAAI;IACF,MAAM,QAAQ,GAAG;IAEjB,MAAM,KAAK,OACR,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,YAAY,CAAC,CAAC,CAC5B,MAAM,GAAG,kBAAkB,IAAI,SAAU,EAAE,CAAC;IAC/C,KAAK,OAAO,KAAK,EAAE,YAAY,SAAU,GAAG,GAAG,iCAAiC;GAClF,SAAS,KAAU;IACjB,IAAI,eAAe,yBAAyB,IAAI,SAAS,yBAAyB;KAChF,MAAM,KAAK,OACR,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,UAAU,CAAC,CAAC,CAC1B,MAAM,GAAG,kBAAkB,IAAI,SAAU,EAAE,CAAC;KAC/C,KAAK,OAAO,KAAK;MAAE,YAAY,SAAU;MAAI,QAAQ,IAAI;KAAO,GAAG,oBAAoB;IACzF,OAAO;KACL,KAAK,OAAO,MAAM;MAAE;MAAK,YAAY,SAAU;KAAG,GAAG,iBAAiB;KACtE,MAAM,KAAK,OACR,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,SAAS,CAAC,CAAC,CACzB,MAAM,GAAG,kBAAkB,IAAI,SAAU,EAAE,CAAC;IACjD;GACF;EACF,UAAU;GACR,cAAc,UAAU;GAExB,MAAM,KAAK,SAAS,KAAK,kBAAkB,GAAG,SAAS,SAAS;EAClE;EAEA,MAAM,QAAQ,IAAI,eAAe;CACnC;AACF;AASA,eAAsB,sBAAsB;CAC1C,SAAS,aAAa;EAAE,MAAM;EAAmB,OAAO,OAAO;CAAU,CAAC;CAC1E,QAAQ,IAAI,YAAY;EAAE,KAAK,OAAO;EAAW,MAAM;EAAY;CAAO,CAAC;CAC3E,MAAM,SAAS,eAAe;EAAE,KAAK,OAAO;EAAc,iBAAiB;EAAY;CAAO,CAAC;CAC/F,QAAM,OAAO;CACb,KAAK,OAAO;CACZ,uBAAuB,IAAI,eAAe;EACxC,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB;CACF,CAAC;CACD,mBAAmB,IAAI,eAAe;EACpC,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB;CACF,CAAC;CACD,WAAW,IAAI,eAAe;EAC5B,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB,OAAO,gBAAgB;EACvB,UAAU,YAAY,QAAQ;EAC9B,WAAW,QAAQ;EACnB,WAAW,OAAO;EAClB;CACF,CAAC;CAED,iBAAiB,IAAI,sBAAsB;EACzC,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB,OAAO,gBAAgB;EACvB,UAAU,YAAY,QAAQ;EAC9B;CACF,CAAC;CAED,SAAS,IAAI,eAAe;EAC1B;EACA;EACA;EACA,aAAa,OAAO;EACpB,OAAO,MAAM;EACb;EACA;EACA;CACF,CAAC;CAGD,eAAe,kBAAkB;EAC/B,CAAM,YAAY;GAChB,IAAI;IACF,MAAM,MAAM,KAAK,IAAI;IACrB,MAAM,QAAS,MAAM,MAAM,OAAO,KAChC,oBACA,GACA,yBACA,KACA,KACA,4BACF;IAEA,KAAK,MAAM,WAAW,OAAO;KAC3B,MAAM,OAAO,KAAK,MAAM,OAAO;KAG/B,MAAM,QACJ,MAAM,GACH,OAAO,CAAC,CACR,KAAK,iBAAiB,CAAC,CACvB,MAAM,GAAG,kBAAkB,IAAI,KAAK,UAAU,CAAC,CAAC,CAChD,MAAM,CAAC,EAAA,CACV;KACF,IAAI,CAAC,QAAQ,KAAK,WAAW,eAAe,KAAK,WAAW,UAAU;MACpE,MAAM,MAAM,OAAO,KAAK,yBAAyB,OAAO;MACxD;KACF;KAKA,IAAI,KAAK,WAAW,WAAW;KAE/B,IAAI,KAAK,gBAAgB;MAcvB,KAAI,MAZkB,GACnB,OAAO,eAAe,CAAC,CACvB,MACC,IACE,GAAG,gBAAgB,YAAY,KAAK,UAAU,GAC9C,GAAG,gBAAgB,WAAW,KAAK,SAAS,CAC9C,CACF,CAAC,CACA,UAAU,EAAA,CAID,WAAW,GAAG;OACxB,MAAM,MAAM,OAAO,KAAK,yBAAyB,OAAO;OACxD;MACF;MAYA,MAAM,eAAc,MATA,GACjB,OAAO,CAAC,CACR,KAAK,aAAa,CAAC,CACnB,MACC,IACE,GAAG,cAAc,YAAY,KAAK,UAAU,GAC5C,GAAG,cAAc,QAAQ,cAAc,CACzC,CACF,EAAA,CACwB,MAAM,MAAW,EAAE,WAAW,IAAI;MAC5D,IAAI,aACF,MAAM,GACH,OAAO,aAAa,CAAC,CACrB,IAAI,EAAE,QAAQ,EAAE,UAAU,KAAK,EAAE,CAAC,CAAC,CACnC,MAAM,GAAG,cAAc,IAAI,YAAY,EAAE,CAAC;KAEjD;KAEA,MAAM,iBAAiB,QACrB,iBAAiB,oBAAoB,MAAM,aAAa,KAAA,CAAS,CACnE;KACA,MAAM,MAAM,OAAO,KAAK,yBAAyB,OAAO;KACxD,OAAO,KAAK,EAAE,YAAY,KAAK,WAAW,GAAG,6BAA6B;IAC5E;GACF,SAAS,KAAK;IACZ,OAAO,MAAM,EAAE,IAAI,GAAG,gCAAgC;GACxD;EACF,EAAA,CAAG;CACL,GAAG,GAAI;CAEP,iBAAiB,kBAAkB;EACjC,CAAM,YAAY;GAChB,IAAI;IACF,MAAM,mBAAmB,MAAM,GAC5B,OAAO,EAAE,IAAI,kBAAkB,GAAG,CAAC,CAAC,CACpC,KAAK,iBAAiB,CAAC,CACvB,MAAM,GAAG,kBAAkB,QAAQ,SAAS,CAAC;IAChD,KAAK,MAAM,QAAQ,kBAAkB;KACnC,MAAM,UAAU,iBAAiB,KAAK;KAEtC,IAAI,CAAC,MADiB,MAAM,OAAO,OAAO,OAAO,GACnC;MACZ,MAAM,GACH,OAAO,iBAAiB,CAAC,CACzB,IAAI,EAAE,QAAQ,UAAU,CAAC,CAAC,CAC1B,MAAM,GAAG,kBAAkB,IAAI,KAAK,EAAE,CAAC;MAC1C,OAAO,KAAK,EAAE,YAAY,KAAK,GAAG,GAAG,+CAA+C;KACtF;IACF;GACF,SAAS,KAAK;IACZ,OAAO,MAAM,EAAE,IAAI,GAAG,gCAAgC;GACxD;EACF,EAAA,CAAG;CACL,GAAG,GAAK;CAER,iBAAiB,oBAAoB,YAAY,QAAQ,OAAO,MAAM;CAEtE,OAAO,KAAK,0BAA0B;CACtC,MAAM,OAAO,MAAM;AACrB;AAEA,eAAsB,qBAAoC;CACxD,QAAQ,KAAK,oBAAoB;CACjC,IAAI,gBAAgB,cAAc,cAAc;CAChD,IAAI,cAAc,cAAc,YAAY;CAC5C,IAAI,gBAAgB,cAAc,cAAc;CAChD,IAAI,QAAQ,MAAM,OAAO,KAAK;CAC9B,IAAIA,OAAK,MAAMA,MAAI,IAAI;CACvB,IAAI,OAAO,MAAM,MAAM,WAAW;CAClC,QAAQ,KAAK,yBAAyB;AACxC"}

package/dist/{main-D6SG3Isk.mjs → main-DSmh1e-V.mjs}

@@ -1,4 +1,4 @@
-import { J as PendingMessageScanner, T as TemplateRepository, W as getPriorityBucket, X as StreamProducer, Xt as buildStreamEvent, Y as StreamConsumer, Zt as CONSUMER_GROUPS, _n as readBaseConfig, b as renderWithTemplate, c as BaseWorker, et as createLogger, fn as AI_DEFAULTS, g as TemplateCache, hn as loadEnv, it as createDatabase, k as RedisClient, mn as getAiConfig, nn as STREAMS, q as globalEmitter, rt as IdempotencyGuard, sn as registry, u as startHealthReporter, ut as scheduledPayloads } from "./src-CPMwsUCJ.mjs";
+import { A as RedisClient, E as TemplateRepository, G as getPriorityBucket, J as globalEmitter, Qt as CONSUMER_GROUPS, X as StreamConsumer, Y as PendingMessageScanner, Z as StreamProducer, Zt as buildStreamEvent, at as createDatabase, b as renderWithTemplate, c as BaseWorker, cn as registry, dt as scheduledPayloads, g as TemplateCache, gn as loadEnv, hn as getAiConfig, it as IdempotencyGuard, pn as AI_DEFAULTS, rn as STREAMS, tt as createLogger, u as startHealthReporter, vn as readBaseConfig } from "./src-CGmXRfsM.mjs";
 import { randomUUID } from "node:crypto";
 import { generateText } from "ai";
 //#region src/services/ai/main.ts
@@ -291,4 +291,4 @@ async function stopAiWorker() {
 //#endregion
 export { startAiWorker, stopAiWorker };
 
-//# sourceMappingURL=main-D6SG3Isk.mjs.map
+//# sourceMappingURL=main-DSmh1e-V.mjs.map

package/dist/{main-D6SG3Isk.mjs.map → main-DSmh1e-V.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"main-D6SG3Isk.mjs","names":[],"sources":["../src/services/ai/main.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { loadEnv, readBaseConfig } from \"@/index.js\";\nimport { createLogger } from \"@/index.js\";\nimport { RedisClient, type Redis } from \"@/index.js\";\nimport {\n  StreamConsumer,\n  PendingMessageScanner,\n  StreamProducer,\n  type StreamMessage,\n} from \"@/index.js\";\nimport { BaseWorker } from \"@/index.js\";\nimport {\n  STREAMS,\n  CONSUMER_GROUPS,\n  registry,\n  buildStreamEvent,\n  type NotificationAiPendingPayload,\n  type NotificationDispatchedPayload,\n  getAiConfig,\n  AI_DEFAULTS,\n} from \"@/index.js\";\nimport { generateText } from \"ai\";\nimport { type StreamName } from \"@/contracts/streams.js\";\nimport { IdempotencyGuard } from \"@/index.js\";\nimport { TemplateRepository } from \"@/index.js\";\nimport { createDatabase } from \"@/db/index.js\";\nimport { scheduledPayloads } from \"@/db/schema.js\";\nimport { getPriorityBucket, globalEmitter, type WorkerOptions } from \"@/shared/index.js\";\nimport { renderWithTemplate, TemplateCache } from \"@/templates/index.js\";\nimport { startHealthReporter } from \"@/workers/index.js\";\n\n// ─── Bootstrap ─────────────────────────────────────────────────────────────\n\nloadEnv();\nconst config = readBaseConfig();\n\nlet logger: ReturnType<typeof createLogger>;\nlet redis: RedisClient;\nlet sql: any;\nlet db: any;\nlet templateRepo: TemplateRepository;\n\nlet templateCache: TemplateCache;\n\nlet consumer: StreamConsumer;\nlet pendingScanner: PendingMessageScanner;\nlet worker: BaseWorker;\nlet healthInterval: NodeJS.Timeout | null = null;\nlet subscriber: any = null;\n\n/**\n * A model failure that retrying cannot fix (bad prompt, rejected request,\n * unsupported model). Thrown so the notification fails once instead of being\n * re-billed on every retry.\n */\nexport class PermanentAiError extends Error {\n  constructor(message: string, options?: ErrorOptions) {\n    super(message, options);\n    this.name = \"PermanentAiError\";\n  }\n}\n\n/** Timeouts, rate limits and 5xx are worth another attempt; 4xx are not. */\nexport function isRetryableAiError(err: unknown): boolean {\n  if (err instanceof PermanentAiError) return false;\n\n  const e = err as { name?: string; statusCode?: number; status?: number } | null;\n  if (!e) return false;\n\n  if (e.name === \"TimeoutError\" || e.name === \"AbortError\") return true;\n\n  const status = e.statusCode ?? e.status;\n  if (typeof status === \"number\") {\n    return status === 408 || status === 409 || status === 429 || status >= 500;\n  }\n\n  // Unclassifiable (network errors, transport failures) — assume transient.\n  return true;\n}\n\nexport interface AiWorkerOptions extends WorkerOptions {\n  registry: any;\n  idempotency: any;\n  redis: Redis;\n  generateAiContent: any;\n  templateCache: TemplateCache;\n  scheduledProducer: any;\n  outboundProducers: any;\n  db: any;\n}\n\nexport class AiWorker extends BaseWorker {\n  private readonly registry: any;\n  private readonly idempotency: any;\n  private readonly redisCli: Redis;\n  private readonly generateAiContent: any;\n  private readonly templateCache: TemplateCache;\n  private readonly scheduledProducer: any;\n  private readonly outboundProducers: any;\n  private readonly db: any;\n\n  constructor(options: AiWorkerOptions) {\n    super(options);\n    this.registry = options.registry;\n    this.idempotency = options.idempotency;\n    this.redisCli = options.redis;\n    this.generateAiContent = options.generateAiContent;\n    this.templateCache = options.templateCache;\n    this.scheduledProducer = options.scheduledProducer;\n    this.outboundProducers = options.outboundProducers;\n    this.db = options.db;\n  }\n  async process(message: StreamMessage): Promise<void> {\n    const { event } = message;\n\n    const payloadResult = this.registry.safeParsePayload(\"notification.ai_pending\", event.payload);\n    if (!payloadResult.success) {\n      this.logger.warn(\n        { messageId: message.id, issues: payloadResult.error.issues },\n        \"invalid notification.ai_pending payload — skipping\",\n      );\n      return;\n    }\n\n    const pending = payloadResult.data as NotificationAiPendingPayload;\n\n    // Idempotency\n    const idempotencyKey = `${pending.enrichedEventId}:${pending.recipientId}:${pending.channel}:ai`;\n    if (!(await this.idempotency.checkAndMark(idempotencyKey))) {\n      this.logger.debug(\n        { messageId: message.id, eventId: event.id },\n        \"duplicate ai task — skipping\",\n      );\n      return;\n    }\n    try {\n      // Execute AI prompts. Each key is a separate billed model call, so the\n      // count is capped rather than being driven by whatever the caller sent.\n      const promptEntries = Object.entries(pending.aiPrompts);\n      const maxPrompts =\n        getAiConfig().maxPromptsPerNotification ?? AI_DEFAULTS.maxPromptsPerNotification;\n      if (promptEntries.length > maxPrompts) {\n        this.logger.warn(\n          { messageId: message.id, requested: promptEntries.length, maxPrompts },\n          \"aiPrompts exceeds the per-notification cap — extra prompts ignored\",\n        );\n      }\n\n      const generatedVars: Record<string, string> = {};\n      for (const [key, prompt] of promptEntries.slice(0, maxPrompts)) {\n        generatedVars[key] = await this.generateAiContent(prompt, pending.templateVariables);\n      }\n\n      // Merge generated vars with original template vars\n      const finalVars = { ...pending.templateVariables, ...generatedVars };\n\n      const dbTemplate = pending.templateId\n        ? await this.templateCache.getCachedTemplate(pending.projectId, pending.templateId)\n        : null;\n\n      const rendered = renderWithTemplate(dbTemplate, finalVars);\n\n      const taskId = randomUUID();\n      const destination =\n        pending.channel === \"email\"\n          ? pending.recipient.email\n          : pending.channel === \"sms\" || pending.channel === \"whatsapp\"\n            ? pending.recipient.phone\n            : pending.channel === \"webhook\"\n              ? pending.recipient.webhook\n              : pending.channel === \"telegram\"\n                ? pending.recipient.telegram\n                : pending.channel === \"discord\"\n                  ? pending.recipient.discord\n                  : pending.channel === \"slack\"\n                    ? pending.recipient.slack\n                    : pending.channel === \"push\"\n                      ? (pending.recipient.pushTokens?.[0] ?? pending.recipient.pushToken)\n                      : undefined;\n      const resolvedDestination =\n        destination ?? (pending.channel === \"push\" ? undefined : pending.recipientId);\n\n      const taskPayload: NotificationDispatchedPayload = {\n        projectId: pending.projectId,\n        taskId,\n        enrichedEventId: pending.enrichedEventId,\n        recipientId: pending.recipientId,\n        channel: pending.channel,\n        priority: pending.priority,\n        templateId: pending.templateId,\n        templateVariables: pending.templateVariables,\n        aiPrompts: pending.aiPrompts,\n        recipient: pending.recipient,\n        renderedContent: rendered,\n        destination: resolvedDestination,\n        deliveryOptions: {\n          maxAttempts: 3,\n          timeoutMs: 10_000,\n        },\n        fallbackChain: pending.fallbackChain,\n      };\n\n      const envelope = buildStreamEvent(\n        \"notification.dispatched\",\n        taskPayload as Record<string, unknown>,\n        \"ai-worker\",\n        event.metadata.traceId,\n      );\n\n      // Route by scheduledAt\n      const now = Date.now();\n      const scheduledAt = pending.scheduledAt ? new Date(pending.scheduledAt).getTime() : now;\n\n      if (scheduledAt > now) {\n        await this.db.insert(scheduledPayloads).values({\n          taskId,\n          payload: taskPayload,\n        });\n\n        const scheduledEnvelope = buildStreamEvent(\n          \"notification.scheduled\",\n          {\n            projectId: pending.projectId,\n            enrichedEventId: pending.enrichedEventId,\n            taskId,\n            scheduledAt: pending.scheduledAt!,\n          },\n          \"ai-worker\",\n          event.metadata.traceId,\n        );\n\n        await this.scheduledProducer.publish(scheduledEnvelope);\n        this.logger.info(\n          {\n            messageId: message.id,\n            taskId,\n            scheduledAt: pending.scheduledAt,\n            traceId: event.metadata.traceId,\n          },\n          \"task scheduled and payload cached after AI generation\",\n        );\n      } else {\n        const p = getPriorityBucket(pending.priority);\n        const outboundProducer = this.outboundProducers[p] ?? this.outboundProducers[\"normal\"]!;\n\n        await outboundProducer.publish(envelope);\n        this.logger.info(\n          {\n            messageId: message.id,\n            taskId,\n            recipientId: pending.recipientId,\n            traceId: event.metadata.traceId,\n          },\n          \"task dispatched after AI generation\",\n        );\n      }\n    } catch (err) {\n      await this.idempotency.unmark(idempotencyKey);\n      if (err instanceof PermanentAiError || (err as Error)?.name === \"PermanentAiError\") {\n        // Retrying re-bills the same failing prompt. Fail the notification once.\n        this.logger.error(\n          { err, messageId: message.id, recipientId: pending.recipientId },\n          \"AI generation failed permanently — dropping notification without retry\",\n        );\n        globalEmitter.emit(\n          \"notification:failed\",\n          pending.enrichedEventId,\n          (err as Error).message,\n          pending.channel,\n        );\n        return;\n      }\n      throw err;\n    }\n  }\n}\n\nexport async function startAiWorker() {\n  logger = createLogger({ name: \"ai\", level: config.LOG_LEVEL });\n  redis = new RedisClient({ url: config.REDIS_URL, name: \"ai\", logger });\n  const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: \"ai\", logger });\n  sql = dbData.sql;\n  db = dbData.db;\n  templateRepo = new TemplateRepository(db);\n  templateCache = new TemplateCache(templateRepo);\n  consumer = new StreamConsumer({\n    redis: redis.native,\n    stream: STREAMS.AI_PENDING as StreamName,\n    group: CONSUMER_GROUPS.AI,\n    consumer: `ai-${process.pid}`,\n    dlqStream: STREAMS.DEAD_LETTER,\n    batchSize: config.WORKER_CONCURRENCY,\n    logger,\n  });\n\n  pendingScanner = new PendingMessageScanner({\n    redis: redis.native,\n    stream: STREAMS.AI_PENDING as StreamName,\n    group: CONSUMER_GROUPS.AI,\n    consumer: `ai-${process.pid}`,\n    logger,\n  });\n\n  const outboundProducers = {\n    critical: new StreamProducer({\n      redis: redis.native,\n      stream: STREAMS.OUTBOUND_CRITICAL,\n      logger,\n    }),\n    normal: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_NORMAL, logger }),\n    low: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_LOW, logger }),\n  };\n\n  const scheduledProducer = new StreamProducer({\n    redis: redis.native,\n    stream: STREAMS.SCHEDULED,\n    logger,\n  });\n\n  const idempotency = new IdempotencyGuard({\n    redis: redis.native,\n    keyPrefix: \"notif:processed:ai\",\n    ttlSeconds: 86_400,\n  });\n\n  // AI generation\n  async function generateAiContent(prompt: string, vars: Record<string, unknown>): Promise<string> {\n    const aiConfig = getAiConfig();\n    const interpolated = prompt.replace(/\\{\\{(\\w+)\\}\\}/g, (_, k: string) => String(vars[k] ?? \"\"));\n\n    if (!aiConfig || !aiConfig.aiModel) {\n      logger.warn(\"AI worker called but no AI model was provided to NotifkitServer\");\n      return `[AI Disabled] ${interpolated}`;\n    }\n\n    const maxOutputTokens = aiConfig.maxOutputTokens ?? AI_DEFAULTS.maxOutputTokens;\n    const timeoutMs = aiConfig.timeoutMs ?? AI_DEFAULTS.timeoutMs;\n\n    try {\n      const { text } = await generateText({\n        model: aiConfig.aiModel,\n        prompt: interpolated,\n        maxOutputTokens,\n        abortSignal: AbortSignal.timeout(timeoutMs),\n      });\n\n      return text;\n    } catch (err) {\n      // BaseWorker retries a throw up to maxRetriesBeforeDlq, and every retry is\n      // another billed call. Only re-throw for failures a retry could actually\n      // fix; a malformed prompt or a rejected request must not be re-billed.\n      if (isRetryableAiError(err)) {\n        logger.error({ err }, \"AI generation failed (transient) — will retry\");\n        throw err;\n      }\n      throw new PermanentAiError(err instanceof Error ? err.message : String(err), { cause: err });\n    }\n  }\n\n  worker = new AiWorker({\n    consumer,\n    pendingScanner,\n    logger,\n    concurrency: config.WORKER_CONCURRENCY,\n    registry,\n    idempotency,\n    redis: redis.native,\n    generateAiContent,\n    templateCache,\n    scheduledProducer,\n    outboundProducers,\n    db,\n  });\n\n  subscriber = redis.native.duplicate();\n  await subscriber.subscribe(\"template.invalidated\");\n  subscriber.on(\"message\", (channel: string, message: string) => {\n    if (channel === \"template.invalidated\") {\n      templateCache.invalidateKey(message);\n      logger.info({ cacheKey: message }, \"invalidated template cache\");\n    }\n  });\n\n  healthInterval = startHealthReporter(\"ai\", worker, redis, logger);\n\n  logger.info({ env: config.NODE_ENV, redis: config.REDIS_URL }, \"ai starting\");\n  await worker.start();\n}\n\nexport async function stopAiWorker(): Promise<void> {\n  logger?.info(\"shutdown initiated\");\n  if (healthInterval) {\n    clearInterval(healthInterval);\n    healthInterval = null;\n  }\n  if (subscriber) {\n    subscriber.disconnect();\n    subscriber = null;\n  }\n  if (worker) await worker.stop();\n  if (sql) await sql.end();\n  if (redis) await redis.disconnect();\n  logger?.info(\"ai stopped\");\n}\n"],"mappings":";;;;AAiCA,QAAQ;AACR,MAAM,SAAS,eAAe;AAE9B,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI;AAEJ,IAAI;AAEJ,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI,iBAAwC;AAC5C,IAAI,aAAkB;;;;;;AAOtB,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,SAAiB,SAAwB;EACnD,MAAM,SAAS,OAAO;EACtB,KAAK,OAAO;CACd;AACF;;AAGA,SAAgB,mBAAmB,KAAuB;CACxD,IAAI,eAAe,kBAAkB,OAAO;CAE5C,MAAM,IAAI;CACV,IAAI,CAAC,GAAG,OAAO;CAEf,IAAI,EAAE,SAAS,kBAAkB,EAAE,SAAS,cAAc,OAAO;CAEjE,MAAM,SAAS,EAAE,cAAc,EAAE;CACjC,IAAI,OAAO,WAAW,UACpB,OAAO,WAAW,OAAO,WAAW,OAAO,WAAW,OAAO,UAAU;CAIzE,OAAO;AACT;AAaA,IAAa,WAAb,cAA8B,WAAW;CACvC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAA0B;EACpC,MAAM,OAAO;EACb,KAAK,WAAW,QAAQ;EACxB,KAAK,cAAc,QAAQ;EAC3B,KAAK,WAAW,QAAQ;EACxB,KAAK,oBAAoB,QAAQ;EACjC,KAAK,gBAAgB,QAAQ;EAC7B,KAAK,oBAAoB,QAAQ;EACjC,KAAK,oBAAoB,QAAQ;EACjC,KAAK,KAAK,QAAQ;CACpB;CACA,MAAM,QAAQ,SAAuC;EACnD,MAAM,EAAE,UAAU;EAElB,MAAM,gBAAgB,KAAK,SAAS,iBAAiB,2BAA2B,MAAM,OAAO;EAC7F,IAAI,CAAC,cAAc,SAAS;GAC1B,KAAK,OAAO,KACV;IAAE,WAAW,QAAQ;IAAI,QAAQ,cAAc,MAAM;GAAO,GAC5D,oDACF;GACA;EACF;EAEA,MAAM,UAAU,cAAc;EAG9B,MAAM,iBAAiB,GAAG,QAAQ,gBAAgB,GAAG,QAAQ,YAAY,GAAG,QAAQ,QAAQ;EAC5F,IAAI,CAAE,MAAM,KAAK,YAAY,aAAa,cAAc,GAAI;GAC1D,KAAK,OAAO,MACV;IAAE,WAAW,QAAQ;IAAI,SAAS,MAAM;GAAG,GAC3C,8BACF;GACA;EACF;EACA,IAAI;GAGF,MAAM,gBAAgB,OAAO,QAAQ,QAAQ,SAAS;GACtD,MAAM,aACJ,YAAY,CAAC,CAAC,6BAA6B,YAAY;GACzD,IAAI,cAAc,SAAS,YACzB,KAAK,OAAO,KACV;IAAE,WAAW,QAAQ;IAAI,WAAW,cAAc;IAAQ;GAAW,GACrE,oEACF;GAGF,MAAM,gBAAwC,CAAC;GAC/C,KAAK,MAAM,CAAC,KAAK,WAAW,cAAc,MAAM,GAAG,UAAU,GAC3D,cAAc,OAAO,MAAM,KAAK,kBAAkB,QAAQ,QAAQ,iBAAiB;GAIrF,MAAM,YAAY;IAAE,GAAG,QAAQ;IAAmB,GAAG;GAAc;GAEnE,MAAM,aAAa,QAAQ,aACvB,MAAM,KAAK,cAAc,kBAAkB,QAAQ,WAAW,QAAQ,UAAU,IAChF;GAEJ,MAAM,WAAW,mBAAmB,YAAY,SAAS;GAEzD,MAAM,SAAS,WAAW;GAiB1B,MAAM,uBAfJ,QAAQ,YAAY,UAChB,QAAQ,UAAU,QAClB,QAAQ,YAAY,SAAS,QAAQ,YAAY,aAC/C,QAAQ,UAAU,QAClB,QAAQ,YAAY,YAClB,QAAQ,UAAU,UAClB,QAAQ,YAAY,aAClB,QAAQ,UAAU,WAClB,QAAQ,YAAY,YAClB,QAAQ,UAAU,UAClB,QAAQ,YAAY,UAClB,QAAQ,UAAU,QAClB,QAAQ,YAAY,SACjB,QAAQ,UAAU,aAAa,MAAM,QAAQ,UAAU,YACxD,KAAA,OAEA,QAAQ,YAAY,SAAS,KAAA,IAAY,QAAQ;GAEnE,MAAM,cAA6C;IACjD,WAAW,QAAQ;IACnB;IACA,iBAAiB,QAAQ;IACzB,aAAa,QAAQ;IACrB,SAAS,QAAQ;IACjB,UAAU,QAAQ;IAClB,YAAY,QAAQ;IACpB,mBAAmB,QAAQ;IAC3B,WAAW,QAAQ;IACnB,WAAW,QAAQ;IACnB,iBAAiB;IACjB,aAAa;IACb,iBAAiB;KACf,aAAa;KACb,WAAW;IACb;IACA,eAAe,QAAQ;GACzB;GAEA,MAAM,WAAW,iBACf,2BACA,aACA,aACA,MAAM,SAAS,OACjB;GAGA,MAAM,MAAM,KAAK,IAAI;GAGrB,KAFoB,QAAQ,cAAc,IAAI,KAAK,QAAQ,WAAW,CAAC,CAAC,QAAQ,IAAI,OAElE,KAAK;IACrB,MAAM,KAAK,GAAG,OAAO,iBAAiB,CAAC,CAAC,OAAO;KAC7C;KACA,SAAS;IACX,CAAC;IAED,MAAM,oBAAoB,iBACxB,0BACA;KACE,WAAW,QAAQ;KACnB,iBAAiB,QAAQ;KACzB;KACA,aAAa,QAAQ;IACvB,GACA,aACA,MAAM,SAAS,OACjB;IAEA,MAAM,KAAK,kBAAkB,QAAQ,iBAAiB;IACtD,KAAK,OAAO,KACV;KACE,WAAW,QAAQ;KACnB;KACA,aAAa,QAAQ;KACrB,SAAS,MAAM,SAAS;IAC1B,GACA,uDACF;GACF,OAAO;IACL,MAAM,IAAI,kBAAkB,QAAQ,QAAQ;IAG5C,OAFyB,KAAK,kBAAkB,MAAM,KAAK,kBAAkB,UAAA,CAEtD,QAAQ,QAAQ;IACvC,KAAK,OAAO,KACV;KACE,WAAW,QAAQ;KACnB;KACA,aAAa,QAAQ;KACrB,SAAS,MAAM,SAAS;IAC1B,GACA,qCACF;GACF;EACF,SAAS,KAAK;GACZ,MAAM,KAAK,YAAY,OAAO,cAAc;GAC5C,IAAI,eAAe,oBAAqB,KAAe,SAAS,oBAAoB;IAElF,KAAK,OAAO,MACV;KAAE;KAAK,WAAW,QAAQ;KAAI,aAAa,QAAQ;IAAY,GAC/D,wEACF;IACA,cAAc,KACZ,uBACA,QAAQ,iBACP,IAAc,SACf,QAAQ,OACV;IACA;GACF;GACA,MAAM;EACR;CACF;AACF;AAEA,eAAsB,gBAAgB;CACpC,SAAS,aAAa;EAAE,MAAM;EAAM,OAAO,OAAO;CAAU,CAAC;CAC7D,QAAQ,IAAI,YAAY;EAAE,KAAK,OAAO;EAAW,MAAM;EAAM;CAAO,CAAC;CACrE,MAAM,SAAS,eAAe;EAAE,KAAK,OAAO;EAAc,iBAAiB;EAAM;CAAO,CAAC;CACzF,MAAM,OAAO;CACb,KAAK,OAAO;CACZ,eAAe,IAAI,mBAAmB,EAAE;CACxC,gBAAgB,IAAI,cAAc,YAAY;CAC9C,WAAW,IAAI,eAAe;EAC5B,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB,OAAO,gBAAgB;EACvB,UAAU,MAAM,QAAQ;EACxB,WAAW,QAAQ;EACnB,WAAW,OAAO;EAClB;CACF,CAAC;CAED,iBAAiB,IAAI,sBAAsB;EACzC,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB,OAAO,gBAAgB;EACvB,UAAU,MAAM,QAAQ;EACxB;CACF,CAAC;CAED,MAAM,oBAAoB;EACxB,UAAU,IAAI,eAAe;GAC3B,OAAO,MAAM;GACb,QAAQ,QAAQ;GAChB;EACF,CAAC;EACD,QAAQ,IAAI,eAAe;GAAE,OAAO,MAAM;GAAQ,QAAQ,QAAQ;GAAiB;EAAO,CAAC;EAC3F,KAAK,IAAI,eAAe;GAAE,OAAO,MAAM;GAAQ,QAAQ,QAAQ;GAAc;EAAO,CAAC;CACvF;CAEA,MAAM,oBAAoB,IAAI,eAAe;EAC3C,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB;CACF,CAAC;CAED,MAAM,cAAc,IAAI,iBAAiB;EACvC,OAAO,MAAM;EACb,WAAW;EACX,YAAY;CACd,CAAC;CAGD,eAAe,kBAAkB,QAAgB,MAAgD;EAC/F,MAAM,WAAW,YAAY;EAC7B,MAAM,eAAe,OAAO,QAAQ,mBAAmB,GAAG,MAAc,OAAO,KAAK,MAAM,EAAE,CAAC;EAE7F,IAAI,CAAC,YAAY,CAAC,SAAS,SAAS;GAClC,OAAO,KAAK,iEAAiE;GAC7E,OAAO,iBAAiB;EAC1B;EAEA,MAAM,kBAAkB,SAAS,mBAAmB,YAAY;EAChE,MAAM,YAAY,SAAS,aAAa,YAAY;EAEpD,IAAI;GACF,MAAM,EAAE,SAAS,MAAM,aAAa;IAClC,OAAO,SAAS;IAChB,QAAQ;IACR;IACA,aAAa,YAAY,QAAQ,SAAS;GAC5C,CAAC;GAED,OAAO;EACT,SAAS,KAAK;GAIZ,IAAI,mBAAmB,GAAG,GAAG;IAC3B,OAAO,MAAM,EAAE,IAAI,GAAG,+CAA+C;IACrE,MAAM;GACR;GACA,MAAM,IAAI,iBAAiB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,CAAC;EAC7F;CACF;CAEA,SAAS,IAAI,SAAS;EACpB;EACA;EACA;EACA,aAAa,OAAO;EACpB;EACA;EACA,OAAO,MAAM;EACb;EACA;EACA;EACA;EACA;CACF,CAAC;CAED,aAAa,MAAM,OAAO,UAAU;CACpC,MAAM,WAAW,UAAU,sBAAsB;CACjD,WAAW,GAAG,YAAY,SAAiB,YAAoB;EAC7D,IAAI,YAAY,wBAAwB;GACtC,cAAc,cAAc,OAAO;GACnC,OAAO,KAAK,EAAE,UAAU,QAAQ,GAAG,4BAA4B;EACjE;CACF,CAAC;CAED,iBAAiB,oBAAoB,MAAM,QAAQ,OAAO,MAAM;CAEhE,OAAO,KAAK;EAAE,KAAK,OAAO;EAAU,OAAO,OAAO;CAAU,GAAG,aAAa;CAC5E,MAAM,OAAO,MAAM;AACrB;AAEA,eAAsB,eAA8B;CAClD,QAAQ,KAAK,oBAAoB;CACjC,IAAI,gBAAgB;EAClB,cAAc,cAAc;EAC5B,iBAAiB;CACnB;CACA,IAAI,YAAY;EACd,WAAW,WAAW;EACtB,aAAa;CACf;CACA,IAAI,QAAQ,MAAM,OAAO,KAAK;CAC9B,IAAI,KAAK,MAAM,IAAI,IAAI;CACvB,IAAI,OAAO,MAAM,MAAM,WAAW;CAClC,QAAQ,KAAK,YAAY;AAC3B"}
+{"version":3,"file":"main-DSmh1e-V.mjs","names":[],"sources":["../src/services/ai/main.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { loadEnv, readBaseConfig } from \"@/index.js\";\nimport { createLogger } from \"@/index.js\";\nimport { RedisClient, type Redis } from \"@/index.js\";\nimport {\n  StreamConsumer,\n  PendingMessageScanner,\n  StreamProducer,\n  type StreamMessage,\n} from \"@/index.js\";\nimport { BaseWorker } from \"@/index.js\";\nimport {\n  STREAMS,\n  CONSUMER_GROUPS,\n  registry,\n  buildStreamEvent,\n  type NotificationAiPendingPayload,\n  type NotificationDispatchedPayload,\n  getAiConfig,\n  AI_DEFAULTS,\n} from \"@/index.js\";\nimport { generateText } from \"ai\";\nimport { type StreamName } from \"@/contracts/streams.js\";\nimport { IdempotencyGuard } from \"@/index.js\";\nimport { TemplateRepository } from \"@/index.js\";\nimport { createDatabase } from \"@/db/index.js\";\nimport { scheduledPayloads } from \"@/db/schema.js\";\nimport { getPriorityBucket, globalEmitter, type WorkerOptions } from \"@/shared/index.js\";\nimport { renderWithTemplate, TemplateCache } from \"@/templates/index.js\";\nimport { startHealthReporter } from \"@/workers/index.js\";\n\n// ─── Bootstrap ─────────────────────────────────────────────────────────────\n\nloadEnv();\nconst config = readBaseConfig();\n\nlet logger: ReturnType<typeof createLogger>;\nlet redis: RedisClient;\nlet sql: any;\nlet db: any;\nlet templateRepo: TemplateRepository;\n\nlet templateCache: TemplateCache;\n\nlet consumer: StreamConsumer;\nlet pendingScanner: PendingMessageScanner;\nlet worker: BaseWorker;\nlet healthInterval: NodeJS.Timeout | null = null;\nlet subscriber: any = null;\n\n/**\n * A model failure that retrying cannot fix (bad prompt, rejected request,\n * unsupported model). Thrown so the notification fails once instead of being\n * re-billed on every retry.\n */\nexport class PermanentAiError extends Error {\n  constructor(message: string, options?: ErrorOptions) {\n    super(message, options);\n    this.name = \"PermanentAiError\";\n  }\n}\n\n/** Timeouts, rate limits and 5xx are worth another attempt; 4xx are not. */\nexport function isRetryableAiError(err: unknown): boolean {\n  if (err instanceof PermanentAiError) return false;\n\n  const e = err as { name?: string; statusCode?: number; status?: number } | null;\n  if (!e) return false;\n\n  if (e.name === \"TimeoutError\" || e.name === \"AbortError\") return true;\n\n  const status = e.statusCode ?? e.status;\n  if (typeof status === \"number\") {\n    return status === 408 || status === 409 || status === 429 || status >= 500;\n  }\n\n  // Unclassifiable (network errors, transport failures) — assume transient.\n  return true;\n}\n\nexport interface AiWorkerOptions extends WorkerOptions {\n  registry: any;\n  idempotency: any;\n  redis: Redis;\n  generateAiContent: any;\n  templateCache: TemplateCache;\n  scheduledProducer: any;\n  outboundProducers: any;\n  db: any;\n}\n\nexport class AiWorker extends BaseWorker {\n  private readonly registry: any;\n  private readonly idempotency: any;\n  private readonly redisCli: Redis;\n  private readonly generateAiContent: any;\n  private readonly templateCache: TemplateCache;\n  private readonly scheduledProducer: any;\n  private readonly outboundProducers: any;\n  private readonly db: any;\n\n  constructor(options: AiWorkerOptions) {\n    super(options);\n    this.registry = options.registry;\n    this.idempotency = options.idempotency;\n    this.redisCli = options.redis;\n    this.generateAiContent = options.generateAiContent;\n    this.templateCache = options.templateCache;\n    this.scheduledProducer = options.scheduledProducer;\n    this.outboundProducers = options.outboundProducers;\n    this.db = options.db;\n  }\n  async process(message: StreamMessage): Promise<void> {\n    const { event } = message;\n\n    const payloadResult = this.registry.safeParsePayload(\"notification.ai_pending\", event.payload);\n    if (!payloadResult.success) {\n      this.logger.warn(\n        { messageId: message.id, issues: payloadResult.error.issues },\n        \"invalid notification.ai_pending payload — skipping\",\n      );\n      return;\n    }\n\n    const pending = payloadResult.data as NotificationAiPendingPayload;\n\n    // Idempotency\n    const idempotencyKey = `${pending.enrichedEventId}:${pending.recipientId}:${pending.channel}:ai`;\n    if (!(await this.idempotency.checkAndMark(idempotencyKey))) {\n      this.logger.debug(\n        { messageId: message.id, eventId: event.id },\n        \"duplicate ai task — skipping\",\n      );\n      return;\n    }\n    try {\n      // Execute AI prompts. Each key is a separate billed model call, so the\n      // count is capped rather than being driven by whatever the caller sent.\n      const promptEntries = Object.entries(pending.aiPrompts);\n      const maxPrompts =\n        getAiConfig().maxPromptsPerNotification ?? AI_DEFAULTS.maxPromptsPerNotification;\n      if (promptEntries.length > maxPrompts) {\n        this.logger.warn(\n          { messageId: message.id, requested: promptEntries.length, maxPrompts },\n          \"aiPrompts exceeds the per-notification cap — extra prompts ignored\",\n        );\n      }\n\n      const generatedVars: Record<string, string> = {};\n      for (const [key, prompt] of promptEntries.slice(0, maxPrompts)) {\n        generatedVars[key] = await this.generateAiContent(prompt, pending.templateVariables);\n      }\n\n      // Merge generated vars with original template vars\n      const finalVars = { ...pending.templateVariables, ...generatedVars };\n\n      const dbTemplate = pending.templateId\n        ? await this.templateCache.getCachedTemplate(pending.projectId, pending.templateId)\n        : null;\n\n      const rendered = renderWithTemplate(dbTemplate, finalVars);\n\n      const taskId = randomUUID();\n      const destination =\n        pending.channel === \"email\"\n          ? pending.recipient.email\n          : pending.channel === \"sms\" || pending.channel === \"whatsapp\"\n            ? pending.recipient.phone\n            : pending.channel === \"webhook\"\n              ? pending.recipient.webhook\n              : pending.channel === \"telegram\"\n                ? pending.recipient.telegram\n                : pending.channel === \"discord\"\n                  ? pending.recipient.discord\n                  : pending.channel === \"slack\"\n                    ? pending.recipient.slack\n                    : pending.channel === \"push\"\n                      ? (pending.recipient.pushTokens?.[0] ?? pending.recipient.pushToken)\n                      : undefined;\n      const resolvedDestination =\n        destination ?? (pending.channel === \"push\" ? undefined : pending.recipientId);\n\n      const taskPayload: NotificationDispatchedPayload = {\n        projectId: pending.projectId,\n        taskId,\n        enrichedEventId: pending.enrichedEventId,\n        recipientId: pending.recipientId,\n        channel: pending.channel,\n        priority: pending.priority,\n        templateId: pending.templateId,\n        templateVariables: pending.templateVariables,\n        aiPrompts: pending.aiPrompts,\n        recipient: pending.recipient,\n        renderedContent: rendered,\n        destination: resolvedDestination,\n        deliveryOptions: {\n          maxAttempts: 3,\n          timeoutMs: 10_000,\n        },\n        fallbackChain: pending.fallbackChain,\n      };\n\n      const envelope = buildStreamEvent(\n        \"notification.dispatched\",\n        taskPayload as Record<string, unknown>,\n        \"ai-worker\",\n        event.metadata.traceId,\n      );\n\n      // Route by scheduledAt\n      const now = Date.now();\n      const scheduledAt = pending.scheduledAt ? new Date(pending.scheduledAt).getTime() : now;\n\n      if (scheduledAt > now) {\n        await this.db.insert(scheduledPayloads).values({\n          taskId,\n          payload: taskPayload,\n        });\n\n        const scheduledEnvelope = buildStreamEvent(\n          \"notification.scheduled\",\n          {\n            projectId: pending.projectId,\n            enrichedEventId: pending.enrichedEventId,\n            taskId,\n            scheduledAt: pending.scheduledAt!,\n          },\n          \"ai-worker\",\n          event.metadata.traceId,\n        );\n\n        await this.scheduledProducer.publish(scheduledEnvelope);\n        this.logger.info(\n          {\n            messageId: message.id,\n            taskId,\n            scheduledAt: pending.scheduledAt,\n            traceId: event.metadata.traceId,\n          },\n          \"task scheduled and payload cached after AI generation\",\n        );\n      } else {\n        const p = getPriorityBucket(pending.priority);\n        const outboundProducer = this.outboundProducers[p] ?? this.outboundProducers[\"normal\"]!;\n\n        await outboundProducer.publish(envelope);\n        this.logger.info(\n          {\n            messageId: message.id,\n            taskId,\n            recipientId: pending.recipientId,\n            traceId: event.metadata.traceId,\n          },\n          \"task dispatched after AI generation\",\n        );\n      }\n    } catch (err) {\n      await this.idempotency.unmark(idempotencyKey);\n      if (err instanceof PermanentAiError || (err as Error)?.name === \"PermanentAiError\") {\n        // Retrying re-bills the same failing prompt. Fail the notification once.\n        this.logger.error(\n          { err, messageId: message.id, recipientId: pending.recipientId },\n          \"AI generation failed permanently — dropping notification without retry\",\n        );\n        globalEmitter.emit(\n          \"notification:failed\",\n          pending.enrichedEventId,\n          (err as Error).message,\n          pending.channel,\n        );\n        return;\n      }\n      throw err;\n    }\n  }\n}\n\nexport async function startAiWorker() {\n  logger = createLogger({ name: \"ai\", level: config.LOG_LEVEL });\n  redis = new RedisClient({ url: config.REDIS_URL, name: \"ai\", logger });\n  const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: \"ai\", logger });\n  sql = dbData.sql;\n  db = dbData.db;\n  templateRepo = new TemplateRepository(db);\n  templateCache = new TemplateCache(templateRepo);\n  consumer = new StreamConsumer({\n    redis: redis.native,\n    stream: STREAMS.AI_PENDING as StreamName,\n    group: CONSUMER_GROUPS.AI,\n    consumer: `ai-${process.pid}`,\n    dlqStream: STREAMS.DEAD_LETTER,\n    batchSize: config.WORKER_CONCURRENCY,\n    logger,\n  });\n\n  pendingScanner = new PendingMessageScanner({\n    redis: redis.native,\n    stream: STREAMS.AI_PENDING as StreamName,\n    group: CONSUMER_GROUPS.AI,\n    consumer: `ai-${process.pid}`,\n    logger,\n  });\n\n  const outboundProducers = {\n    critical: new StreamProducer({\n      redis: redis.native,\n      stream: STREAMS.OUTBOUND_CRITICAL,\n      logger,\n    }),\n    normal: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_NORMAL, logger }),\n    low: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_LOW, logger }),\n  };\n\n  const scheduledProducer = new StreamProducer({\n    redis: redis.native,\n    stream: STREAMS.SCHEDULED,\n    logger,\n  });\n\n  const idempotency = new IdempotencyGuard({\n    redis: redis.native,\n    keyPrefix: \"notif:processed:ai\",\n    ttlSeconds: 86_400,\n  });\n\n  // AI generation\n  async function generateAiContent(prompt: string, vars: Record<string, unknown>): Promise<string> {\n    const aiConfig = getAiConfig();\n    const interpolated = prompt.replace(/\\{\\{(\\w+)\\}\\}/g, (_, k: string) => String(vars[k] ?? \"\"));\n\n    if (!aiConfig || !aiConfig.aiModel) {\n      logger.warn(\"AI worker called but no AI model was provided to NotifkitServer\");\n      return `[AI Disabled] ${interpolated}`;\n    }\n\n    const maxOutputTokens = aiConfig.maxOutputTokens ?? AI_DEFAULTS.maxOutputTokens;\n    const timeoutMs = aiConfig.timeoutMs ?? AI_DEFAULTS.timeoutMs;\n\n    try {\n      const { text } = await generateText({\n        model: aiConfig.aiModel,\n        prompt: interpolated,\n        maxOutputTokens,\n        abortSignal: AbortSignal.timeout(timeoutMs),\n      });\n\n      return text;\n    } catch (err) {\n      // BaseWorker retries a throw up to maxRetriesBeforeDlq, and every retry is\n      // another billed call. Only re-throw for failures a retry could actually\n      // fix; a malformed prompt or a rejected request must not be re-billed.\n      if (isRetryableAiError(err)) {\n        logger.error({ err }, \"AI generation failed (transient) — will retry\");\n        throw err;\n      }\n      throw new PermanentAiError(err instanceof Error ? err.message : String(err), { cause: err });\n    }\n  }\n\n  worker = new AiWorker({\n    consumer,\n    pendingScanner,\n    logger,\n    concurrency: config.WORKER_CONCURRENCY,\n    registry,\n    idempotency,\n    redis: redis.native,\n    generateAiContent,\n    templateCache,\n    scheduledProducer,\n    outboundProducers,\n    db,\n  });\n\n  subscriber = redis.native.duplicate();\n  await subscriber.subscribe(\"template.invalidated\");\n  subscriber.on(\"message\", (channel: string, message: string) => {\n    if (channel === \"template.invalidated\") {\n      templateCache.invalidateKey(message);\n      logger.info({ cacheKey: message }, \"invalidated template cache\");\n    }\n  });\n\n  healthInterval = startHealthReporter(\"ai\", worker, redis, logger);\n\n  logger.info({ env: config.NODE_ENV, redis: config.REDIS_URL }, \"ai starting\");\n  await worker.start();\n}\n\nexport async function stopAiWorker(): Promise<void> {\n  logger?.info(\"shutdown initiated\");\n  if (healthInterval) {\n    clearInterval(healthInterval);\n    healthInterval = null;\n  }\n  if (subscriber) {\n    subscriber.disconnect();\n    subscriber = null;\n  }\n  if (worker) await worker.stop();\n  if (sql) await sql.end();\n  if (redis) await redis.disconnect();\n  logger?.info(\"ai stopped\");\n}\n"],"mappings":";;;;AAiCA,QAAQ;AACR,MAAM,SAAS,eAAe;AAE9B,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI;AAEJ,IAAI;AAEJ,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI,iBAAwC;AAC5C,IAAI,aAAkB;;;;;;AAOtB,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,SAAiB,SAAwB;EACnD,MAAM,SAAS,OAAO;EACtB,KAAK,OAAO;CACd;AACF;;AAGA,SAAgB,mBAAmB,KAAuB;CACxD,IAAI,eAAe,kBAAkB,OAAO;CAE5C,MAAM,IAAI;CACV,IAAI,CAAC,GAAG,OAAO;CAEf,IAAI,EAAE,SAAS,kBAAkB,EAAE,SAAS,cAAc,OAAO;CAEjE,MAAM,SAAS,EAAE,cAAc,EAAE;CACjC,IAAI,OAAO,WAAW,UACpB,OAAO,WAAW,OAAO,WAAW,OAAO,WAAW,OAAO,UAAU;CAIzE,OAAO;AACT;AAaA,IAAa,WAAb,cAA8B,WAAW;CACvC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAA0B;EACpC,MAAM,OAAO;EACb,KAAK,WAAW,QAAQ;EACxB,KAAK,cAAc,QAAQ;EAC3B,KAAK,WAAW,QAAQ;EACxB,KAAK,oBAAoB,QAAQ;EACjC,KAAK,gBAAgB,QAAQ;EAC7B,KAAK,oBAAoB,QAAQ;EACjC,KAAK,oBAAoB,QAAQ;EACjC,KAAK,KAAK,QAAQ;CACpB;CACA,MAAM,QAAQ,SAAuC;EACnD,MAAM,EAAE,UAAU;EAElB,MAAM,gBAAgB,KAAK,SAAS,iBAAiB,2BAA2B,MAAM,OAAO;EAC7F,IAAI,CAAC,cAAc,SAAS;GAC1B,KAAK,OAAO,KACV;IAAE,WAAW,QAAQ;IAAI,QAAQ,cAAc,MAAM;GAAO,GAC5D,oDACF;GACA;EACF;EAEA,MAAM,UAAU,cAAc;EAG9B,MAAM,iBAAiB,GAAG,QAAQ,gBAAgB,GAAG,QAAQ,YAAY,GAAG,QAAQ,QAAQ;EAC5F,IAAI,CAAE,MAAM,KAAK,YAAY,aAAa,cAAc,GAAI;GAC1D,KAAK,OAAO,MACV;IAAE,WAAW,QAAQ;IAAI,SAAS,MAAM;GAAG,GAC3C,8BACF;GACA;EACF;EACA,IAAI;GAGF,MAAM,gBAAgB,OAAO,QAAQ,QAAQ,SAAS;GACtD,MAAM,aACJ,YAAY,CAAC,CAAC,6BAA6B,YAAY;GACzD,IAAI,cAAc,SAAS,YACzB,KAAK,OAAO,KACV;IAAE,WAAW,QAAQ;IAAI,WAAW,cAAc;IAAQ;GAAW,GACrE,oEACF;GAGF,MAAM,gBAAwC,CAAC;GAC/C,KAAK,MAAM,CAAC,KAAK,WAAW,cAAc,MAAM,GAAG,UAAU,GAC3D,cAAc,OAAO,MAAM,KAAK,kBAAkB,QAAQ,QAAQ,iBAAiB;GAIrF,MAAM,YAAY;IAAE,GAAG,QAAQ;IAAmB,GAAG;GAAc;GAEnE,MAAM,aAAa,QAAQ,aACvB,MAAM,KAAK,cAAc,kBAAkB,QAAQ,WAAW,QAAQ,UAAU,IAChF;GAEJ,MAAM,WAAW,mBAAmB,YAAY,SAAS;GAEzD,MAAM,SAAS,WAAW;GAiB1B,MAAM,uBAfJ,QAAQ,YAAY,UAChB,QAAQ,UAAU,QAClB,QAAQ,YAAY,SAAS,QAAQ,YAAY,aAC/C,QAAQ,UAAU,QAClB,QAAQ,YAAY,YAClB,QAAQ,UAAU,UAClB,QAAQ,YAAY,aAClB,QAAQ,UAAU,WAClB,QAAQ,YAAY,YAClB,QAAQ,UAAU,UAClB,QAAQ,YAAY,UAClB,QAAQ,UAAU,QAClB,QAAQ,YAAY,SACjB,QAAQ,UAAU,aAAa,MAAM,QAAQ,UAAU,YACxD,KAAA,OAEA,QAAQ,YAAY,SAAS,KAAA,IAAY,QAAQ;GAEnE,MAAM,cAA6C;IACjD,WAAW,QAAQ;IACnB;IACA,iBAAiB,QAAQ;IACzB,aAAa,QAAQ;IACrB,SAAS,QAAQ;IACjB,UAAU,QAAQ;IAClB,YAAY,QAAQ;IACpB,mBAAmB,QAAQ;IAC3B,WAAW,QAAQ;IACnB,WAAW,QAAQ;IACnB,iBAAiB;IACjB,aAAa;IACb,iBAAiB;KACf,aAAa;KACb,WAAW;IACb;IACA,eAAe,QAAQ;GACzB;GAEA,MAAM,WAAW,iBACf,2BACA,aACA,aACA,MAAM,SAAS,OACjB;GAGA,MAAM,MAAM,KAAK,IAAI;GAGrB,KAFoB,QAAQ,cAAc,IAAI,KAAK,QAAQ,WAAW,CAAC,CAAC,QAAQ,IAAI,OAElE,KAAK;IACrB,MAAM,KAAK,GAAG,OAAO,iBAAiB,CAAC,CAAC,OAAO;KAC7C;KACA,SAAS;IACX,CAAC;IAED,MAAM,oBAAoB,iBACxB,0BACA;KACE,WAAW,QAAQ;KACnB,iBAAiB,QAAQ;KACzB;KACA,aAAa,QAAQ;IACvB,GACA,aACA,MAAM,SAAS,OACjB;IAEA,MAAM,KAAK,kBAAkB,QAAQ,iBAAiB;IACtD,KAAK,OAAO,KACV;KACE,WAAW,QAAQ;KACnB;KACA,aAAa,QAAQ;KACrB,SAAS,MAAM,SAAS;IAC1B,GACA,uDACF;GACF,OAAO;IACL,MAAM,IAAI,kBAAkB,QAAQ,QAAQ;IAG5C,OAFyB,KAAK,kBAAkB,MAAM,KAAK,kBAAkB,UAAA,CAEtD,QAAQ,QAAQ;IACvC,KAAK,OAAO,KACV;KACE,WAAW,QAAQ;KACnB;KACA,aAAa,QAAQ;KACrB,SAAS,MAAM,SAAS;IAC1B,GACA,qCACF;GACF;EACF,SAAS,KAAK;GACZ,MAAM,KAAK,YAAY,OAAO,cAAc;GAC5C,IAAI,eAAe,oBAAqB,KAAe,SAAS,oBAAoB;IAElF,KAAK,OAAO,MACV;KAAE;KAAK,WAAW,QAAQ;KAAI,aAAa,QAAQ;IAAY,GAC/D,wEACF;IACA,cAAc,KACZ,uBACA,QAAQ,iBACP,IAAc,SACf,QAAQ,OACV;IACA;GACF;GACA,MAAM;EACR;CACF;AACF;AAEA,eAAsB,gBAAgB;CACpC,SAAS,aAAa;EAAE,MAAM;EAAM,OAAO,OAAO;CAAU,CAAC;CAC7D,QAAQ,IAAI,YAAY;EAAE,KAAK,OAAO;EAAW,MAAM;EAAM;CAAO,CAAC;CACrE,MAAM,SAAS,eAAe;EAAE,KAAK,OAAO;EAAc,iBAAiB;EAAM;CAAO,CAAC;CACzF,MAAM,OAAO;CACb,KAAK,OAAO;CACZ,eAAe,IAAI,mBAAmB,EAAE;CACxC,gBAAgB,IAAI,cAAc,YAAY;CAC9C,WAAW,IAAI,eAAe;EAC5B,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB,OAAO,gBAAgB;EACvB,UAAU,MAAM,QAAQ;EACxB,WAAW,QAAQ;EACnB,WAAW,OAAO;EAClB;CACF,CAAC;CAED,iBAAiB,IAAI,sBAAsB;EACzC,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB,OAAO,gBAAgB;EACvB,UAAU,MAAM,QAAQ;EACxB;CACF,CAAC;CAED,MAAM,oBAAoB;EACxB,UAAU,IAAI,eAAe;GAC3B,OAAO,MAAM;GACb,QAAQ,QAAQ;GAChB;EACF,CAAC;EACD,QAAQ,IAAI,eAAe;GAAE,OAAO,MAAM;GAAQ,QAAQ,QAAQ;GAAiB;EAAO,CAAC;EAC3F,KAAK,IAAI,eAAe;GAAE,OAAO,MAAM;GAAQ,QAAQ,QAAQ;GAAc;EAAO,CAAC;CACvF;CAEA,MAAM,oBAAoB,IAAI,eAAe;EAC3C,OAAO,MAAM;EACb,QAAQ,QAAQ;EAChB;CACF,CAAC;CAED,MAAM,cAAc,IAAI,iBAAiB;EACvC,OAAO,MAAM;EACb,WAAW;EACX,YAAY;CACd,CAAC;CAGD,eAAe,kBAAkB,QAAgB,MAAgD;EAC/F,MAAM,WAAW,YAAY;EAC7B,MAAM,eAAe,OAAO,QAAQ,mBAAmB,GAAG,MAAc,OAAO,KAAK,MAAM,EAAE,CAAC;EAE7F,IAAI,CAAC,YAAY,CAAC,SAAS,SAAS;GAClC,OAAO,KAAK,iEAAiE;GAC7E,OAAO,iBAAiB;EAC1B;EAEA,MAAM,kBAAkB,SAAS,mBAAmB,YAAY;EAChE,MAAM,YAAY,SAAS,aAAa,YAAY;EAEpD,IAAI;GACF,MAAM,EAAE,SAAS,MAAM,aAAa;IAClC,OAAO,SAAS;IAChB,QAAQ;IACR;IACA,aAAa,YAAY,QAAQ,SAAS;GAC5C,CAAC;GAED,OAAO;EACT,SAAS,KAAK;GAIZ,IAAI,mBAAmB,GAAG,GAAG;IAC3B,OAAO,MAAM,EAAE,IAAI,GAAG,+CAA+C;IACrE,MAAM;GACR;GACA,MAAM,IAAI,iBAAiB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,CAAC;EAC7F;CACF;CAEA,SAAS,IAAI,SAAS;EACpB;EACA;EACA;EACA,aAAa,OAAO;EACpB;EACA;EACA,OAAO,MAAM;EACb;EACA;EACA;EACA;EACA;CACF,CAAC;CAED,aAAa,MAAM,OAAO,UAAU;CACpC,MAAM,WAAW,UAAU,sBAAsB;CACjD,WAAW,GAAG,YAAY,SAAiB,YAAoB;EAC7D,IAAI,YAAY,wBAAwB;GACtC,cAAc,cAAc,OAAO;GACnC,OAAO,KAAK,EAAE,UAAU,QAAQ,GAAG,4BAA4B;EACjE;CACF,CAAC;CAED,iBAAiB,oBAAoB,MAAM,QAAQ,OAAO,MAAM;CAEhE,OAAO,KAAK;EAAE,KAAK,OAAO;EAAU,OAAO,OAAO;CAAU,GAAG,aAAa;CAC5E,MAAM,OAAO,MAAM;AACrB;AAEA,eAAsB,eAA8B;CAClD,QAAQ,KAAK,oBAAoB;CACjC,IAAI,gBAAgB;EAClB,cAAc,cAAc;EAC5B,iBAAiB;CACnB;CACA,IAAI,YAAY;EACd,WAAW,WAAW;EACtB,aAAa;CACf;CACA,IAAI,QAAQ,MAAM,OAAO,KAAK;CAC9B,IAAI,KAAK,MAAM,IAAI,IAAI;CACvB,IAAI,OAAO,MAAM,MAAM,WAAW;CAClC,QAAQ,KAAK,YAAY;AAC3B"}