@lota-sdk/core 0.4.9 → 0.4.11

package/src/services/plan/plan-deadline.service.ts

@@ -17,6 +17,8 @@ import type { SurrealDBService } from '../../db/service'
 import { TABLES } from '../../db/tables'
 import { makeEffectTryPromiseWithMessage } from '../../effect/helpers'
 import { DatabaseServiceTag } from '../../effect/services'
+import type { PlanSchedulerQueueRuntime } from '../../queues/plan-scheduler.queue'
+import { LotaQueuesServiceTag } from '../../queues/queues.service'
 import { nowDate, unsafeDateFrom } from '../../utils/date-time'
 import type { makePlanEventDeliveryService } from './plan-event-delivery.service'
 import { PlanEventDeliveryServiceTag } from './plan-event-delivery.service'
@@ -86,6 +88,7 @@ interface PlanDeadlineDeps {
   planExecutorService: ReturnType<typeof makePlanExecutorService>
   planEventDeliveryService: ReturnType<typeof makePlanEventDeliveryService>
   planRunService: ReturnType<typeof makePlanRunService>
+  planSchedulerQueue: PlanSchedulerQueueRuntime
 }
 
 class PlanDeadlineError extends Schema.TaggedErrorClass<PlanDeadlineError>()('PlanDeadlineError', {
@@ -94,12 +97,12 @@ class PlanDeadlineError extends Schema.TaggedErrorClass<PlanDeadlineError>()('Pl
 }) {}
 
 export function makePlanDeadlineService(deps: PlanDeadlineDeps) {
-  const { db, planEventDeliveryService, planExecutorService, planRunService } = deps
+  const { db, planEventDeliveryService, planExecutorService, planRunService, planSchedulerQueue } = deps
   const effectTryPromise = makeEffectTryPromiseWithMessage(
     (message, cause) => new PlanDeadlineError({ message, cause }),
   )
-  const loadPlanSchedulerQueue = () =>
-    effectTryPromise(() => import('../../queues/plan-scheduler.queue'), 'Failed to load plan scheduler queue module.')
+  const loadPlanSchedulerQueue = (): Effect.Effect<PlanSchedulerQueueRuntime, PlanDeadlineError> =>
+    Effect.succeed(planSchedulerQueue)
 
   function loadNodeSpec(nodeRun: PlanNodeRunRecord): Effect.Effect<PlanNodeSpecRecord | null, PlanDeadlineError> {
     return db
@@ -111,50 +114,48 @@ export function makePlanDeadlineService(deps: PlanDeadlineDeps) {
       .pipe(Effect.mapError((cause) => new PlanDeadlineError({ message: 'Failed to load plan node spec.', cause })))
   }
 
-  function collectDeadlineSweepEffect(now: Date) {
-    return Effect.gen(function* () {
-      const activeNodeRuns = yield* effectTryPromise(
-        () =>
-          db.queryMany(
-            new BoundQuery(`SELECT * FROM ${TABLES.PLAN_NODE_RUN} WHERE status IN $statuses`, {
-              statuses: ['running', 'awaiting-human'],
-            }),
-            PlanNodeRunSchema,
-          ),
-        'Failed to load active plan node runs for deadline sweep.',
-      )
-      const results = yield* Effect.forEach(
-        activeNodeRuns,
-        (nodeRun) =>
-          Effect.gen(function* () {
-            const nodeSpec = yield* loadNodeSpec(nodeRun).pipe(
-              Effect.mapError(
-                (cause) =>
-                  new PlanDeadlineError({
-                    message: `Failed to load node spec for ${nodeRun.nodeId}.`,
-                    cause: cause.cause,
-                  }),
-              ),
-            )
-            if (!nodeSpec?.deadline) {
-              return null
-            }
-
-            const evaluation = evaluateDeadline({
-              deadline: nodeSpec.deadline,
-              nodeStartedAt: unsafeDateFrom(nodeRun.startedAt ?? nodeRun.createdAt),
-              now,
-            })
-
-            return { nodeRun, nodeSpec, evaluation }
+  const collectDeadlineSweepEffect = Effect.fn('PlanDeadline.collectDeadlineSweep')(function* (now: Date) {
+    const activeNodeRuns = yield* effectTryPromise(
+      () =>
+        db.queryMany(
+          new BoundQuery(`SELECT * FROM ${TABLES.PLAN_NODE_RUN} WHERE status IN $statuses`, {
+            statuses: ['running', 'awaiting-human'],
           }),
-        { concurrency: 10 },
-      )
-      return { entries: results.filter((entry): entry is NonNullable<typeof entry> => entry !== null) }
-    })
-  }
+          PlanNodeRunSchema,
+        ),
+      'Failed to load active plan node runs for deadline sweep.',
+    ).pipe(Effect.withSpan('PlanDeadline.loadActiveNodeRuns'))
+    const results = yield* Effect.forEach(
+      activeNodeRuns,
+      (nodeRun) =>
+        Effect.gen(function* () {
+          const nodeSpec = yield* loadNodeSpec(nodeRun).pipe(
+            Effect.mapError(
+              (cause) =>
+                new PlanDeadlineError({
+                  message: `Failed to load node spec for ${nodeRun.nodeId}.`,
+                  cause: cause.cause,
+                }),
+            ),
+          )
+          if (!nodeSpec?.deadline) {
+            return null
+          }
 
-  function emitDeadlineEventEffect(params: {
+          const evaluation = evaluateDeadline({
+            deadline: nodeSpec.deadline,
+            nodeStartedAt: unsafeDateFrom(nodeRun.startedAt ?? nodeRun.createdAt),
+            now,
+          })
+
+          return { nodeRun, nodeSpec, evaluation }
+        }),
+      { concurrency: 10 },
+    ).pipe(Effect.withSpan('PlanDeadline.evaluateNodeRuns'))
+    return { entries: results.filter((entry): entry is NonNullable<typeof entry> => entry !== null) }
+  })
+
+  const emitDeadlineEventEffect = Effect.fn('PlanDeadline.emitDeadlineEvent')(function* (params: {
     run: PlanRunRecord
     nodeRun: PlanNodeRunRecord
     eventType: Extract<PlanEventType, 'deadline-warning' | 'deadline-missed' | 'escalation-triggered'>
@@ -167,68 +168,69 @@ export function makePlanDeadlineService(deps: PlanDeadlineDeps) {
         ? params.detail.dedupeKey.trim()
         : null
 
-    return Effect.gen(function* () {
-      const existing =
-        dedupeKey === null
-          ? []
-          : yield* effectTryPromise(
-              () =>
-                db.queryMany(
-                  new BoundQuery(
-                    `SELECT * FROM ${TABLES.PLAN_EVENT}
+    const existing =
+      dedupeKey === null
+        ? []
+        : yield* effectTryPromise(
+            () =>
+              db.queryMany(
+                new BoundQuery(
+                  `SELECT * FROM ${TABLES.PLAN_EVENT}
                     WHERE runId = $runId
                       AND nodeId = $nodeId
                       AND eventType = $eventType
                       AND detail.dedupeKey = $dedupeKey
                     LIMIT 1`,
-                    {
-                      runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
-                      nodeId: params.nodeRun.nodeId,
-                      eventType: params.eventType,
-                      dedupeKey,
-                    },
-                  ),
-                  PlanEventSchema,
+                  {
+                    runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
+                    nodeId: params.nodeRun.nodeId,
+                    eventType: params.eventType,
+                    dedupeKey,
+                  },
                 ),
-              'Failed to load existing deadline event.',
-            )
-      if (existing.length > 0) {
-        return
-      }
+                PlanEventSchema,
+              ),
+            'Failed to load existing deadline event.',
+          )
+    if (existing.length > 0) {
+      return
+    }
 
-      const spec = yield* planRunService
-        .getPlanSpecById(params.run.planSpecId)
-        .pipe(
-          Effect.mapError(
-            (cause) => new PlanDeadlineError({ message: 'Failed to load plan spec for deadline event.', cause }),
-          ),
-        )
-      const event = yield* effectTryPromise(
-        () =>
-          db.create(
-            TABLES.PLAN_EVENT,
-            {
-              id: new RecordId(TABLES.PLAN_EVENT, Bun.randomUUIDv7()),
-              planSpecId: ensureRecordId(spec.id, TABLES.PLAN_SPEC),
-              runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
-              nodeId: params.nodeRun.nodeId,
-              eventType: params.eventType,
-              message: params.message,
-              emittedBy: params.emittedBy,
-              detail: params.detail,
-            },
-            PlanEventSchema,
-          ),
-        'Failed to create deadline event.',
+    const spec = yield* planRunService
+      .getPlanSpecById(params.run.planSpecId)
+      .pipe(
+        Effect.mapError(
+          (cause) => new PlanDeadlineError({ message: 'Failed to load plan spec for deadline event.', cause }),
+        ),
       )
-      yield* effectTryPromise(
-        () => planEventDeliveryService.dispatchEvent(PlanEventSchema.parse(event)),
-        'Failed to dispatch deadline event.',
+    const event = yield* effectTryPromise(
+      () =>
+        db.create(
+          TABLES.PLAN_EVENT,
+          {
+            id: new RecordId(TABLES.PLAN_EVENT, Bun.randomUUIDv7()),
+            planSpecId: ensureRecordId(spec.id, TABLES.PLAN_SPEC),
+            runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
+            nodeId: params.nodeRun.nodeId,
+            eventType: params.eventType,
+            message: params.message,
+            emittedBy: params.emittedBy,
+            detail: params.detail,
+          },
+          PlanEventSchema,
+        ),
+      'Failed to create deadline event.',
+    )
+    yield* planEventDeliveryService
+      .dispatchEvent(PlanEventSchema.parse(event))
+      .pipe(
+        Effect.mapError(
+          (error) => new PlanDeadlineError({ message: 'Failed to dispatch deadline event.', cause: error }),
+        ),
       )
-    })
-  }
+  })
 
-  function applyDeadlineMissActionEffect(params: {
+  const applyDeadlineMissActionEffect = Effect.fn('PlanDeadline.applyDeadlineMissAction')(function* (params: {
     runId: RecordIdInput
     nodeId: string
     threadId: string
@@ -237,105 +239,99 @@ export function makePlanDeadlineService(deps: PlanDeadlineDeps) {
     emittedBy: string
   }) {
     const runIdStr = recordIdToString(params.runId, TABLES.PLAN_RUN)
-    return Effect.gen(function* () {
-      const [run, nodeRun] = yield* Effect.all([
-        planRunService
-          .getRunById(params.runId)
-          .pipe(
-            Effect.mapError(
-              (cause) => new PlanDeadlineError({ message: 'Failed to load plan run for deadline action.', cause }),
-            ),
+    const [run, nodeRun] = yield* Effect.all([
+      planRunService
+        .getRunById(params.runId)
+        .pipe(
+          Effect.mapError(
+            (cause) => new PlanDeadlineError({ message: 'Failed to load plan run for deadline action.', cause }),
           ),
-        planRunService
-          .getNodeRunByNodeId(params.runId, params.nodeId)
-          .pipe(
-            Effect.mapError(
-              (cause) => new PlanDeadlineError({ message: 'Failed to load plan node run for deadline action.', cause }),
-            ),
+        ),
+      planRunService
+        .getNodeRunByNodeId(params.runId, params.nodeId)
+        .pipe(
+          Effect.mapError(
+            (cause) => new PlanDeadlineError({ message: 'Failed to load plan node run for deadline action.', cause }),
           ),
-      ])
-      const blockNode = (message: string): Effect.Effect<void, PlanDeadlineError> =>
-        effectTryPromise(
-          () =>
-            planExecutorService.blockNodeOnDispatchFailure({
-              threadId: params.threadId,
-              runId: runIdStr,
-              nodeId: params.nodeId,
-              emittedBy: params.emittedBy,
-              message,
-              failureClass: 'timeout_exceeded',
-            }),
-          `Failed to ${message.toLowerCase()}.`,
+        ),
+    ])
+    const blockNode = (message: string): Effect.Effect<void, PlanDeadlineError> =>
+      planExecutorService
+        .blockNodeOnDispatchFailure({
+          threadId: params.threadId,
+          runId: runIdStr,
+          nodeId: params.nodeId,
+          emittedBy: params.emittedBy,
+          message,
+          failureClass: 'timeout_exceeded',
+        })
+        .pipe(
+          Effect.asVoid,
+          Effect.mapError((cause) => new PlanDeadlineError({ message: `Failed to ${message.toLowerCase()}.`, cause })),
         )
 
-      switch (params.action) {
-        case 'notify':
-          yield* emitDeadlineEventEffect({
-            run,
-            nodeRun,
-            eventType: 'deadline-missed',
-            emittedBy: params.emittedBy,
-            message: `Node "${params.nodeId}" has missed its deadline.`,
-            detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:notify` },
-          })
-          return
+    switch (params.action) {
+      case 'notify':
+        yield* emitDeadlineEventEffect({
+          run,
+          nodeRun,
+          eventType: 'deadline-missed',
+          emittedBy: params.emittedBy,
+          message: `Node "${params.nodeId}" has missed its deadline.`,
+          detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:notify` },
+        })
+        return
 
-        case 'escalate':
-          yield* emitDeadlineEventEffect({
-            run,
-            nodeRun,
-            eventType: 'escalation-triggered',
-            emittedBy: params.emittedBy,
-            message: `Node "${params.nodeId}" has missed its deadline and requires escalation.`,
-            detail: {
-              title: 'Deadline escalation',
-              dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:escalate`,
-              missedDeadline: true,
-            },
-          })
-          return
+      case 'escalate':
+        yield* emitDeadlineEventEffect({
+          run,
+          nodeRun,
+          eventType: 'escalation-triggered',
+          emittedBy: params.emittedBy,
+          message: `Node "${params.nodeId}" has missed its deadline and requires escalation.`,
+          detail: {
+            title: 'Deadline escalation',
+            dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:escalate`,
+            missedDeadline: true,
+          },
+        })
+        return
 
-        case 'block':
-          yield* emitDeadlineEventEffect({
-            run,
-            nodeRun,
-            eventType: 'deadline-missed',
-            emittedBy: params.emittedBy,
-            message: `Node "${params.nodeId}" has missed its deadline.`,
-            detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:block` },
-          })
-          yield* blockNode('Deadline missed — node blocked')
-          return
+      case 'block':
+        yield* emitDeadlineEventEffect({
+          run,
+          nodeRun,
+          eventType: 'deadline-missed',
+          emittedBy: params.emittedBy,
+          message: `Node "${params.nodeId}" has missed its deadline.`,
+          detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:block` },
+        })
+        yield* blockNode('Deadline missed — node blocked')
+        return
 
-        case 'fail':
-          yield* emitDeadlineEventEffect({
-            run,
-            nodeRun,
-            eventType: 'deadline-missed',
-            emittedBy: params.emittedBy,
-            message: `Node "${params.nodeId}" has missed its deadline.`,
-            detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:fail` },
-          })
-          yield* blockNode('Deadline missed — node failed')
-          return
-      }
-    })
-  }
+      case 'fail':
+        yield* emitDeadlineEventEffect({
+          run,
+          nodeRun,
+          eventType: 'deadline-missed',
+          emittedBy: params.emittedBy,
+          message: `Node "${params.nodeId}" has missed its deadline.`,
+          detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:fail` },
+        })
+        yield* blockNode('Deadline missed — node failed')
+        return
+    }
+  })
 
-  function maybeEmitEscalationPolicyEventEffect(params: {
-    run: PlanRunRecord
-    nodeRun: PlanNodeRunRecord
-    nodeSpec: PlanNodeSpecRecord
-    now: Date
-  }) {
-    const escalation = params.nodeSpec.escalation
-    if (!escalation) return Effect.void
+  const maybeEmitEscalationPolicyEventEffect = Effect.fn('PlanDeadline.maybeEmitEscalationPolicyEvent')(
+    function* (params: { run: PlanRunRecord; nodeRun: PlanNodeRunRecord; nodeSpec: PlanNodeSpecRecord; now: Date }) {
+      const escalation = params.nodeSpec.escalation
+      if (!escalation) return
 
-    const startedAt = unsafeDateFrom(params.nodeRun.startedAt ?? params.nodeRun.createdAt)
-    const runIdStr = recordIdToString(params.run.id, TABLES.PLAN_RUN)
-    const baseKey = `plan-escalation:${runIdStr}:${params.nodeRun.nodeId}`
+      const startedAt = unsafeDateFrom(params.nodeRun.startedAt ?? params.nodeRun.createdAt)
+      const runIdStr = recordIdToString(params.run.id, TABLES.PLAN_RUN)
+      const baseKey = `plan-escalation:${runIdStr}:${params.nodeRun.nodeId}`
 
-    return Effect.gen(function* () {
       if (escalation.autoEscalateAfterMinutes) {
         const thresholdMs = escalation.autoEscalateAfterMinutes * 60_000
         if (params.now.getTime() - startedAt.getTime() >= thresholdMs) {
@@ -378,146 +374,140 @@ export function makePlanDeadlineService(deps: PlanDeadlineDeps) {
           })
         }
       }
-    })
-  }
+    },
+  )
 
-  const enqueueDeadlineCheckEffect = (scheduledFor: Date) =>
-    Effect.gen(function* () {
-      const { enqueueDeadlineCheck } = yield* loadPlanSchedulerQueue()
-      yield* effectTryPromise(() => enqueueDeadlineCheck(scheduledFor), 'Failed to enqueue deadline check.')
-    })
+  const enqueueDeadlineCheckEffect = Effect.fn('PlanDeadline.enqueueDeadlineCheck')(function* (scheduledFor: Date) {
+    const { enqueueDeadlineCheck } = yield* loadPlanSchedulerQueue()
+    yield* effectTryPromise(() => enqueueDeadlineCheck(scheduledFor), 'Failed to enqueue deadline check.')
+  })
 
-  const checkDeadlinesEffect = (now?: Date) => {
+  const checkDeadlinesEffect = Effect.fn('PlanDeadline.checkDeadlines')(function* (now?: Date) {
     const currentTime = now ?? nowDate()
 
-    return Effect.gen(function* () {
-      const sweep = yield* collectDeadlineSweepEffect(currentTime)
-      if (sweep.entries.length === 0) {
-        return
-      }
-
-      const runCache = new Map<string, PlanRunRecord>()
+    const sweep = yield* collectDeadlineSweepEffect(currentTime)
+    if (sweep.entries.length === 0) {
+      return
+    }
 
-      const handleEntry = (entry: (typeof sweep.entries)[number]): Effect.Effect<void, PlanDeadlineError> =>
-        Effect.gen(function* () {
-          const deadline = entry.nodeSpec.deadline
-          if (!deadline) {
-            return
-          }
+    const runCache = new Map<string, PlanRunRecord>()
 
-          const runIdStr = recordIdToString(entry.nodeRun.runId, TABLES.PLAN_RUN)
-          const cachedRun = runCache.get(runIdStr)
-          const run =
-            cachedRun ??
-            (yield* effectTryPromise(
-              () =>
-                db.findOne(
-                  TABLES.PLAN_RUN,
-                  { id: ensureRecordId(entry.nodeRun.runId, TABLES.PLAN_RUN) },
-                  PlanRunSchema,
-                ),
-              'Failed to load plan run during deadline check.',
-            ))
-          if (!run) {
-            return
-          }
-          runCache.set(runIdStr, run)
+    const handleEntry = (entry: (typeof sweep.entries)[number]): Effect.Effect<void, PlanDeadlineError> =>
+      Effect.gen(function* () {
+        const deadline = entry.nodeSpec.deadline
+        if (!deadline) {
+          return
+        }
 
-          const dedupeKeyBase = `plan-deadline:${runIdStr}:${entry.nodeRun.nodeId}`
-          const actionEffect =
-            entry.evaluation.status === 'warning'
+        const runIdStr = recordIdToString(entry.nodeRun.runId, TABLES.PLAN_RUN)
+        const cachedRun = runCache.get(runIdStr)
+        const run =
+          cachedRun ??
+          (yield* effectTryPromise(
+            () =>
+              db.findOne(TABLES.PLAN_RUN, { id: ensureRecordId(entry.nodeRun.runId, TABLES.PLAN_RUN) }, PlanRunSchema),
+            'Failed to load plan run during deadline check.',
+          ))
+        if (!run) {
+          return
+        }
+        runCache.set(runIdStr, run)
+
+        const dedupeKeyBase = `plan-deadline:${runIdStr}:${entry.nodeRun.nodeId}`
+        const actionEffect =
+          entry.evaluation.status === 'warning'
+            ? emitDeadlineEventEffect({
+                run,
+                nodeRun: entry.nodeRun,
+                eventType: 'deadline-warning',
+                emittedBy: 'plan-deadline-checker',
+                message:
+                  entry.evaluation.activeReminder?.message ??
+                  `Node "${entry.nodeRun.nodeId}" is approaching its deadline.`,
+                detail: {
+                  title: 'Deadline approaching',
+                  reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
+                  dedupeKey: `${dedupeKeyBase}:warning:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
+                },
+              })
+            : entry.evaluation.status === 'escalated'
               ? emitDeadlineEventEffect({
                   run,
                   nodeRun: entry.nodeRun,
-                  eventType: 'deadline-warning',
+                  eventType: 'escalation-triggered',
                   emittedBy: 'plan-deadline-checker',
                   message:
                     entry.evaluation.activeReminder?.message ??
-                    `Node "${entry.nodeRun.nodeId}" is approaching its deadline.`,
+                    `Node "${entry.nodeRun.nodeId}" deadline requires escalation.`,
                   detail: {
-                    title: 'Deadline approaching',
+                    title: 'Deadline escalation',
                     reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
-                    dedupeKey: `${dedupeKeyBase}:warning:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
+                    dedupeKey: `${dedupeKeyBase}:escalated:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
                   },
                 })
-              : entry.evaluation.status === 'escalated'
-                ? emitDeadlineEventEffect({
-                    run,
-                    nodeRun: entry.nodeRun,
-                    eventType: 'escalation-triggered',
-                    emittedBy: 'plan-deadline-checker',
-                    message:
-                      entry.evaluation.activeReminder?.message ??
-                      `Node "${entry.nodeRun.nodeId}" deadline requires escalation.`,
-                    detail: {
-                      title: 'Deadline escalation',
-                      reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
-                      dedupeKey: `${dedupeKeyBase}:escalated:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
-                    },
-                  })
-                : applyDeadlineMissActionEffect({
-                    runId: entry.nodeRun.runId,
-                    nodeId: entry.nodeRun.nodeId,
-                    threadId: recordIdToString(run.threadId, TABLES.THREAD),
-                    organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
-                    action: deadline.missAction,
-                    emittedBy: 'plan-deadline-checker',
-                  })
-
-          yield* actionEffect
-          yield* maybeEmitEscalationPolicyEventEffect({
-            run,
-            nodeRun: entry.nodeRun,
-            nodeSpec: entry.nodeSpec,
-            now: currentTime,
-          })
-        })
-
-      yield* Effect.forEach(
-        sweep.entries.filter((entry) => entry.evaluation.status !== 'ok'),
-        handleEntry,
-        { concurrency: 3, discard: true },
-      )
-
-      const nextTriggerAt =
-        sweep.entries
-          .map((entry) => entry.evaluation.nextTriggerAt)
-          .filter((value): value is Date => {
-            if (!value) return false
-            return value.getTime() > currentTime.getTime()
-          })
-          .sort((a, b) => a.getTime() - b.getTime())
-          .at(0) ?? null
+              : applyDeadlineMissActionEffect({
+                  runId: entry.nodeRun.runId,
+                  nodeId: entry.nodeRun.nodeId,
+                  threadId: recordIdToString(run.threadId, TABLES.THREAD),
+                  organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
+                  action: deadline.missAction,
+                  emittedBy: 'plan-deadline-checker',
+                })
 
-      if (nextTriggerAt) {
-        yield* enqueueDeadlineCheckEffect(nextTriggerAt)
-      }
-    })
-  }
+        yield* actionEffect
+        yield* maybeEmitEscalationPolicyEventEffect({
+          run,
+          nodeRun: entry.nodeRun,
+          nodeSpec: entry.nodeSpec,
+          now: currentTime,
+        })
+      })
+
+    yield* Effect.forEach(
+      sweep.entries.filter((entry) => entry.evaluation.status !== 'ok'),
+      handleEntry,
+      { concurrency: 3, discard: true },
+    ).pipe(Effect.withSpan('PlanDeadline.processDeadlineEntries'))
+
+    const nextTriggerAt =
+      sweep.entries
+        .map((entry) => entry.evaluation.nextTriggerAt)
+        .filter((value): value is Date => {
+          if (!value) return false
+          return value.getTime() > currentTime.getTime()
+        })
+        .sort((a, b) => a.getTime() - b.getTime())
+        .at(0) ?? null
 
-  const recoverDeadlineChecksEffect = (now = nowDate()) =>
-    Effect.gen(function* () {
-      const sweep = yield* collectDeadlineSweepEffect(now)
-      const hasDueAction = sweep.entries.some((entry) => entry.evaluation.status !== 'ok')
-      if (hasDueAction) {
-        yield* enqueueDeadlineCheckEffect(now)
-        return
-      }
+    if (nextTriggerAt) {
+      yield* enqueueDeadlineCheckEffect(nextTriggerAt)
+    }
+  })
+
+  const recoverDeadlineChecksEffect = Effect.fn('PlanDeadline.recoverDeadlineChecks')(function* (
+    now: Date = nowDate(),
+  ) {
+    const sweep = yield* collectDeadlineSweepEffect(now)
+    const hasDueAction = sweep.entries.some((entry) => entry.evaluation.status !== 'ok')
+    if (hasDueAction) {
+      yield* enqueueDeadlineCheckEffect(now)
+      return
+    }
 
-      const nextTriggerAt =
-        sweep.entries
-          .map((entry) => entry.evaluation.nextTriggerAt)
-          .filter((value): value is Date => {
-            if (!value) return false
-            return value.getTime() > now.getTime()
-          })
-          .sort((a, b) => a.getTime() - b.getTime())
-          .at(0) ?? null
+    const nextTriggerAt =
+      sweep.entries
+        .map((entry) => entry.evaluation.nextTriggerAt)
+        .filter((value): value is Date => {
+          if (!value) return false
+          return value.getTime() > now.getTime()
+        })
+        .sort((a, b) => a.getTime() - b.getTime())
+        .at(0) ?? null
 
-      if (nextTriggerAt) {
-        yield* enqueueDeadlineCheckEffect(nextTriggerAt)
-      }
-    })
+    if (nextTriggerAt) {
+      yield* enqueueDeadlineCheckEffect(nextTriggerAt)
+    }
+  })
 
   return {
     evaluateDeadline,
@@ -533,7 +523,7 @@ export function makePlanDeadlineService(deps: PlanDeadlineDeps) {
 export class PlanDeadlineServiceTag extends Context.Service<
   PlanDeadlineServiceTag,
   ReturnType<typeof makePlanDeadlineService>
->()('PlanDeadlineService') {}
+>()('@lota-sdk/core/PlanDeadlineService') {}
 
 export const PlanDeadlineServiceLive = Layer.effect(
   PlanDeadlineServiceTag,
@@ -542,6 +532,13 @@ export const PlanDeadlineServiceLive = Layer.effect(
     const planExecutorService = yield* PlanExecutorServiceTag
     const planEventDeliveryService = yield* PlanEventDeliveryServiceTag
     const planRunService = yield* PlanRunServiceTag
-    return makePlanDeadlineService({ db, planExecutorService, planEventDeliveryService, planRunService })
+    const queues = yield* LotaQueuesServiceTag
+    return makePlanDeadlineService({
+      db,
+      planExecutorService,
+      planEventDeliveryService,
+      planRunService,
+      planSchedulerQueue: queues.planScheduler,
+    })
   }),
 )