@lota-sdk/core 0.4.7 → 0.4.9

package/src/services/plan/plan-deadline.service.ts

@@ -0,0 +1,547 @@
+import type {
+  DeadlineAction,
+  DeadlineReminder,
+  DeadlineSpec,
+  PlanEventType,
+  PlanNodeRunRecord,
+  PlanNodeSpecRecord,
+  PlanRunRecord,
+} from '@lota-sdk/shared'
+import { PlanEventSchema, PlanNodeRunSchema, PlanNodeSpecRecordSchema, PlanRunSchema } from '@lota-sdk/shared'
+import { Context, Schema, Effect, Layer } from 'effect'
+import { BoundQuery, RecordId } from 'surrealdb'
+
+import type { RecordIdInput } from '../../db/record-id'
+import { ensureRecordId, recordIdToString } from '../../db/record-id'
+import type { SurrealDBService } from '../../db/service'
+import { TABLES } from '../../db/tables'
+import { makeEffectTryPromiseWithMessage } from '../../effect/helpers'
+import { DatabaseServiceTag } from '../../effect/services'
+import { nowDate, unsafeDateFrom } from '../../utils/date-time'
+import type { makePlanEventDeliveryService } from './plan-event-delivery.service'
+import { PlanEventDeliveryServiceTag } from './plan-event-delivery.service'
+import type { makePlanExecutorService } from './plan-executor.service'
+import { PlanExecutorServiceTag } from './plan-executor.service'
+import type { makePlanRunService } from './plan-run.service'
+import { PlanRunServiceTag } from './plan-run.service'
+
+export type DeadlineEvaluationStatus = 'ok' | 'warning' | 'escalated' | 'missed'
+
+export interface DeadlineEvaluationResult {
+  status: DeadlineEvaluationStatus
+  activeReminder?: DeadlineReminder
+  nextTriggerAt?: Date | null
+}
+
+function resolveDeadlineTime(deadline: DeadlineSpec, nodeStartedAt: Date): Date | null {
+  return (
+    (deadline.dueAt ? unsafeDateFrom(deadline.dueAt) : null) ??
+    (deadline.durationMs !== undefined ? unsafeDateFrom(nodeStartedAt.getTime() + deadline.durationMs) : null)
+  )
+}
+
+/** Pure deadline evaluation — exported for use by other services without needing the full PlanDeadlineService. */
+export function evaluateDeadline(params: {
+  deadline: DeadlineSpec
+  nodeStartedAt: Date
+  now?: Date
+}): DeadlineEvaluationResult {
+  const now = params.now ?? nowDate()
+  const deadlineTime = resolveDeadlineTime(params.deadline, params.nodeStartedAt)
+
+  if (!deadlineTime) {
+    return { status: 'ok', nextTriggerAt: null }
+  }
+
+  const reminderEntries = [...params.deadline.reminders]
+    .map((reminder) => ({ reminder, triggerAt: unsafeDateFrom(deadlineTime.getTime() - reminder.beforeMs) }))
+    .sort((a, b) => a.triggerAt.getTime() - b.triggerAt.getTime())
+
+  let activeReminder: DeadlineReminder | undefined
+  let nextTriggerAt: Date | null = null
+
+  for (const entry of reminderEntries) {
+    if (now.getTime() < entry.triggerAt.getTime()) {
+      nextTriggerAt = entry.triggerAt
+      break
+    }
+
+    activeReminder = entry.reminder
+  }
+
+  if (now.getTime() >= deadlineTime.getTime()) {
+    return { status: 'missed', nextTriggerAt: null }
+  }
+
+  if (activeReminder) {
+    const status: DeadlineEvaluationStatus = activeReminder.action === 'escalate' ? 'escalated' : 'warning'
+    return { status, activeReminder, nextTriggerAt: nextTriggerAt ?? deadlineTime }
+  }
+
+  return { status: 'ok', nextTriggerAt: nextTriggerAt ?? deadlineTime }
+}
+
+interface PlanDeadlineDeps {
+  db: SurrealDBService
+  planExecutorService: ReturnType<typeof makePlanExecutorService>
+  planEventDeliveryService: ReturnType<typeof makePlanEventDeliveryService>
+  planRunService: ReturnType<typeof makePlanRunService>
+}
+
+class PlanDeadlineError extends Schema.TaggedErrorClass<PlanDeadlineError>()('PlanDeadlineError', {
+  message: Schema.String,
+  cause: Schema.optional(Schema.Defect),
+}) {}
+
+export function makePlanDeadlineService(deps: PlanDeadlineDeps) {
+  const { db, planEventDeliveryService, planExecutorService, planRunService } = deps
+  const effectTryPromise = makeEffectTryPromiseWithMessage(
+    (message, cause) => new PlanDeadlineError({ message, cause }),
+  )
+  const loadPlanSchedulerQueue = () =>
+    effectTryPromise(() => import('../../queues/plan-scheduler.queue'), 'Failed to load plan scheduler queue module.')
+
+  function loadNodeSpec(nodeRun: PlanNodeRunRecord): Effect.Effect<PlanNodeSpecRecord | null, PlanDeadlineError> {
+    return db
+      .findOne(
+        TABLES.PLAN_NODE_SPEC,
+        { planSpecId: ensureRecordId(nodeRun.planSpecId, TABLES.PLAN_SPEC), nodeId: nodeRun.nodeId },
+        PlanNodeSpecRecordSchema,
+      )
+      .pipe(Effect.mapError((cause) => new PlanDeadlineError({ message: 'Failed to load plan node spec.', cause })))
+  }
+
+  function collectDeadlineSweepEffect(now: Date) {
+    return Effect.gen(function* () {
+      const activeNodeRuns = yield* effectTryPromise(
+        () =>
+          db.queryMany(
+            new BoundQuery(`SELECT * FROM ${TABLES.PLAN_NODE_RUN} WHERE status IN $statuses`, {
+              statuses: ['running', 'awaiting-human'],
+            }),
+            PlanNodeRunSchema,
+          ),
+        'Failed to load active plan node runs for deadline sweep.',
+      )
+      const results = yield* Effect.forEach(
+        activeNodeRuns,
+        (nodeRun) =>
+          Effect.gen(function* () {
+            const nodeSpec = yield* loadNodeSpec(nodeRun).pipe(
+              Effect.mapError(
+                (cause) =>
+                  new PlanDeadlineError({
+                    message: `Failed to load node spec for ${nodeRun.nodeId}.`,
+                    cause: cause.cause,
+                  }),
+              ),
+            )
+            if (!nodeSpec?.deadline) {
+              return null
+            }
+
+            const evaluation = evaluateDeadline({
+              deadline: nodeSpec.deadline,
+              nodeStartedAt: unsafeDateFrom(nodeRun.startedAt ?? nodeRun.createdAt),
+              now,
+            })
+
+            return { nodeRun, nodeSpec, evaluation }
+          }),
+        { concurrency: 10 },
+      )
+      return { entries: results.filter((entry): entry is NonNullable<typeof entry> => entry !== null) }
+    })
+  }
+
+  function emitDeadlineEventEffect(params: {
+    run: PlanRunRecord
+    nodeRun: PlanNodeRunRecord
+    eventType: Extract<PlanEventType, 'deadline-warning' | 'deadline-missed' | 'escalation-triggered'>
+    emittedBy: string
+    message: string
+    detail: Record<string, unknown>
+  }) {
+    const dedupeKey =
+      typeof params.detail.dedupeKey === 'string' && params.detail.dedupeKey.trim().length > 0
+        ? params.detail.dedupeKey.trim()
+        : null
+
+    return Effect.gen(function* () {
+      const existing =
+        dedupeKey === null
+          ? []
+          : yield* effectTryPromise(
+              () =>
+                db.queryMany(
+                  new BoundQuery(
+                    `SELECT * FROM ${TABLES.PLAN_EVENT}
+                    WHERE runId = $runId
+                      AND nodeId = $nodeId
+                      AND eventType = $eventType
+                      AND detail.dedupeKey = $dedupeKey
+                    LIMIT 1`,
+                    {
+                      runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
+                      nodeId: params.nodeRun.nodeId,
+                      eventType: params.eventType,
+                      dedupeKey,
+                    },
+                  ),
+                  PlanEventSchema,
+                ),
+              'Failed to load existing deadline event.',
+            )
+      if (existing.length > 0) {
+        return
+      }
+
+      const spec = yield* planRunService
+        .getPlanSpecById(params.run.planSpecId)
+        .pipe(
+          Effect.mapError(
+            (cause) => new PlanDeadlineError({ message: 'Failed to load plan spec for deadline event.', cause }),
+          ),
+        )
+      const event = yield* effectTryPromise(
+        () =>
+          db.create(
+            TABLES.PLAN_EVENT,
+            {
+              id: new RecordId(TABLES.PLAN_EVENT, Bun.randomUUIDv7()),
+              planSpecId: ensureRecordId(spec.id, TABLES.PLAN_SPEC),
+              runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
+              nodeId: params.nodeRun.nodeId,
+              eventType: params.eventType,
+              message: params.message,
+              emittedBy: params.emittedBy,
+              detail: params.detail,
+            },
+            PlanEventSchema,
+          ),
+        'Failed to create deadline event.',
+      )
+      yield* effectTryPromise(
+        () => planEventDeliveryService.dispatchEvent(PlanEventSchema.parse(event)),
+        'Failed to dispatch deadline event.',
+      )
+    })
+  }
+
+  function applyDeadlineMissActionEffect(params: {
+    runId: RecordIdInput
+    nodeId: string
+    threadId: string
+    organizationId: string
+    action: DeadlineAction
+    emittedBy: string
+  }) {
+    const runIdStr = recordIdToString(params.runId, TABLES.PLAN_RUN)
+    return Effect.gen(function* () {
+      const [run, nodeRun] = yield* Effect.all([
+        planRunService
+          .getRunById(params.runId)
+          .pipe(
+            Effect.mapError(
+              (cause) => new PlanDeadlineError({ message: 'Failed to load plan run for deadline action.', cause }),
+            ),
+          ),
+        planRunService
+          .getNodeRunByNodeId(params.runId, params.nodeId)
+          .pipe(
+            Effect.mapError(
+              (cause) => new PlanDeadlineError({ message: 'Failed to load plan node run for deadline action.', cause }),
+            ),
+          ),
+      ])
+      const blockNode = (message: string): Effect.Effect<void, PlanDeadlineError> =>
+        effectTryPromise(
+          () =>
+            planExecutorService.blockNodeOnDispatchFailure({
+              threadId: params.threadId,
+              runId: runIdStr,
+              nodeId: params.nodeId,
+              emittedBy: params.emittedBy,
+              message,
+              failureClass: 'timeout_exceeded',
+            }),
+          `Failed to ${message.toLowerCase()}.`,
+        )
+
+      switch (params.action) {
+        case 'notify':
+          yield* emitDeadlineEventEffect({
+            run,
+            nodeRun,
+            eventType: 'deadline-missed',
+            emittedBy: params.emittedBy,
+            message: `Node "${params.nodeId}" has missed its deadline.`,
+            detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:notify` },
+          })
+          return
+
+        case 'escalate':
+          yield* emitDeadlineEventEffect({
+            run,
+            nodeRun,
+            eventType: 'escalation-triggered',
+            emittedBy: params.emittedBy,
+            message: `Node "${params.nodeId}" has missed its deadline and requires escalation.`,
+            detail: {
+              title: 'Deadline escalation',
+              dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:escalate`,
+              missedDeadline: true,
+            },
+          })
+          return
+
+        case 'block':
+          yield* emitDeadlineEventEffect({
+            run,
+            nodeRun,
+            eventType: 'deadline-missed',
+            emittedBy: params.emittedBy,
+            message: `Node "${params.nodeId}" has missed its deadline.`,
+            detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:block` },
+          })
+          yield* blockNode('Deadline missed — node blocked')
+          return
+
+        case 'fail':
+          yield* emitDeadlineEventEffect({
+            run,
+            nodeRun,
+            eventType: 'deadline-missed',
+            emittedBy: params.emittedBy,
+            message: `Node "${params.nodeId}" has missed its deadline.`,
+            detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:fail` },
+          })
+          yield* blockNode('Deadline missed — node failed')
+          return
+      }
+    })
+  }
+
+  function maybeEmitEscalationPolicyEventEffect(params: {
+    run: PlanRunRecord
+    nodeRun: PlanNodeRunRecord
+    nodeSpec: PlanNodeSpecRecord
+    now: Date
+  }) {
+    const escalation = params.nodeSpec.escalation
+    if (!escalation) return Effect.void
+
+    const startedAt = unsafeDateFrom(params.nodeRun.startedAt ?? params.nodeRun.createdAt)
+    const runIdStr = recordIdToString(params.run.id, TABLES.PLAN_RUN)
+    const baseKey = `plan-escalation:${runIdStr}:${params.nodeRun.nodeId}`
+
+    return Effect.gen(function* () {
+      if (escalation.autoEscalateAfterMinutes) {
+        const thresholdMs = escalation.autoEscalateAfterMinutes * 60_000
+        if (params.now.getTime() - startedAt.getTime() >= thresholdMs) {
+          yield* emitDeadlineEventEffect({
+            run: params.run,
+            nodeRun: params.nodeRun,
+            eventType: 'escalation-triggered',
+            emittedBy: 'plan-deadline-checker',
+            message: `Node "${params.nodeRun.nodeId}" exceeded its auto-escalation threshold.`,
+            detail: {
+              title: 'Execution escalation',
+              dedupeKey: `${baseKey}:auto:${escalation.autoEscalateAfterMinutes}`,
+              autoEscalateAfterMinutes: escalation.autoEscalateAfterMinutes,
+              ...(escalation.escalateToAgent ? { escalateToAgent: escalation.escalateToAgent } : {}),
+              ...(escalation.escalateToUser ? { escalateToUser: escalation.escalateToUser } : {}),
+            },
+          })
+        }
+      }
+
+      if (escalation.deadlineThresholdMinutes && params.nodeSpec.deadline) {
+        const dueAt = resolveDeadlineTime(params.nodeSpec.deadline, startedAt)
+        if (!dueAt) return
+
+        const thresholdMs = escalation.deadlineThresholdMinutes * 60_000
+        if (dueAt.getTime() - params.now.getTime() <= thresholdMs && dueAt.getTime() > params.now.getTime()) {
+          yield* emitDeadlineEventEffect({
+            run: params.run,
+            nodeRun: params.nodeRun,
+            eventType: 'escalation-triggered',
+            emittedBy: 'plan-deadline-checker',
+            message: `Node "${params.nodeRun.nodeId}" is inside its escalation threshold.`,
+            detail: {
+              title: 'Deadline escalation threshold',
+              dedupeKey: `${baseKey}:threshold:${escalation.deadlineThresholdMinutes}`,
+              deadlineThresholdMinutes: escalation.deadlineThresholdMinutes,
+              ...(escalation.escalateToAgent ? { escalateToAgent: escalation.escalateToAgent } : {}),
+              ...(escalation.escalateToUser ? { escalateToUser: escalation.escalateToUser } : {}),
+            },
+          })
+        }
+      }
+    })
+  }
+
+  const enqueueDeadlineCheckEffect = (scheduledFor: Date) =>
+    Effect.gen(function* () {
+      const { enqueueDeadlineCheck } = yield* loadPlanSchedulerQueue()
+      yield* effectTryPromise(() => enqueueDeadlineCheck(scheduledFor), 'Failed to enqueue deadline check.')
+    })
+
+  const checkDeadlinesEffect = (now?: Date) => {
+    const currentTime = now ?? nowDate()
+
+    return Effect.gen(function* () {
+      const sweep = yield* collectDeadlineSweepEffect(currentTime)
+      if (sweep.entries.length === 0) {
+        return
+      }
+
+      const runCache = new Map<string, PlanRunRecord>()
+
+      const handleEntry = (entry: (typeof sweep.entries)[number]): Effect.Effect<void, PlanDeadlineError> =>
+        Effect.gen(function* () {
+          const deadline = entry.nodeSpec.deadline
+          if (!deadline) {
+            return
+          }
+
+          const runIdStr = recordIdToString(entry.nodeRun.runId, TABLES.PLAN_RUN)
+          const cachedRun = runCache.get(runIdStr)
+          const run =
+            cachedRun ??
+            (yield* effectTryPromise(
+              () =>
+                db.findOne(
+                  TABLES.PLAN_RUN,
+                  { id: ensureRecordId(entry.nodeRun.runId, TABLES.PLAN_RUN) },
+                  PlanRunSchema,
+                ),
+              'Failed to load plan run during deadline check.',
+            ))
+          if (!run) {
+            return
+          }
+          runCache.set(runIdStr, run)
+
+          const dedupeKeyBase = `plan-deadline:${runIdStr}:${entry.nodeRun.nodeId}`
+          const actionEffect =
+            entry.evaluation.status === 'warning'
+              ? emitDeadlineEventEffect({
+                  run,
+                  nodeRun: entry.nodeRun,
+                  eventType: 'deadline-warning',
+                  emittedBy: 'plan-deadline-checker',
+                  message:
+                    entry.evaluation.activeReminder?.message ??
+                    `Node "${entry.nodeRun.nodeId}" is approaching its deadline.`,
+                  detail: {
+                    title: 'Deadline approaching',
+                    reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
+                    dedupeKey: `${dedupeKeyBase}:warning:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
+                  },
+                })
+              : entry.evaluation.status === 'escalated'
+                ? emitDeadlineEventEffect({
+                    run,
+                    nodeRun: entry.nodeRun,
+                    eventType: 'escalation-triggered',
+                    emittedBy: 'plan-deadline-checker',
+                    message:
+                      entry.evaluation.activeReminder?.message ??
+                      `Node "${entry.nodeRun.nodeId}" deadline requires escalation.`,
+                    detail: {
+                      title: 'Deadline escalation',
+                      reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
+                      dedupeKey: `${dedupeKeyBase}:escalated:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
+                    },
+                  })
+                : applyDeadlineMissActionEffect({
+                    runId: entry.nodeRun.runId,
+                    nodeId: entry.nodeRun.nodeId,
+                    threadId: recordIdToString(run.threadId, TABLES.THREAD),
+                    organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
+                    action: deadline.missAction,
+                    emittedBy: 'plan-deadline-checker',
+                  })
+
+          yield* actionEffect
+          yield* maybeEmitEscalationPolicyEventEffect({
+            run,
+            nodeRun: entry.nodeRun,
+            nodeSpec: entry.nodeSpec,
+            now: currentTime,
+          })
+        })
+
+      yield* Effect.forEach(
+        sweep.entries.filter((entry) => entry.evaluation.status !== 'ok'),
+        handleEntry,
+        { concurrency: 3, discard: true },
+      )
+
+      const nextTriggerAt =
+        sweep.entries
+          .map((entry) => entry.evaluation.nextTriggerAt)
+          .filter((value): value is Date => {
+            if (!value) return false
+            return value.getTime() > currentTime.getTime()
+          })
+          .sort((a, b) => a.getTime() - b.getTime())
+          .at(0) ?? null
+
+      if (nextTriggerAt) {
+        yield* enqueueDeadlineCheckEffect(nextTriggerAt)
+      }
+    })
+  }
+
+  const recoverDeadlineChecksEffect = (now = nowDate()) =>
+    Effect.gen(function* () {
+      const sweep = yield* collectDeadlineSweepEffect(now)
+      const hasDueAction = sweep.entries.some((entry) => entry.evaluation.status !== 'ok')
+      if (hasDueAction) {
+        yield* enqueueDeadlineCheckEffect(now)
+        return
+      }
+
+      const nextTriggerAt =
+        sweep.entries
+          .map((entry) => entry.evaluation.nextTriggerAt)
+          .filter((value): value is Date => {
+            if (!value) return false
+            return value.getTime() > now.getTime()
+          })
+          .sort((a, b) => a.getTime() - b.getTime())
+          .at(0) ?? null
+
+      if (nextTriggerAt) {
+        yield* enqueueDeadlineCheckEffect(nextTriggerAt)
+      }
+    })
+
+  return {
+    evaluateDeadline,
+    checkDeadlines: checkDeadlinesEffect,
+    recoverDeadlineChecks: recoverDeadlineChecksEffect,
+    applyDeadlineMissAction: applyDeadlineMissActionEffect,
+    maybeEmitEscalationPolicyEvent: maybeEmitEscalationPolicyEventEffect,
+    emitDeadlineEvent: emitDeadlineEventEffect,
+    enqueueDeadlineCheck: enqueueDeadlineCheckEffect,
+  }
+}
+
+export class PlanDeadlineServiceTag extends Context.Service<
+  PlanDeadlineServiceTag,
+  ReturnType<typeof makePlanDeadlineService>
+>()('PlanDeadlineService') {}
+
+export const PlanDeadlineServiceLive = Layer.effect(
+  PlanDeadlineServiceTag,
+  Effect.gen(function* () {
+    const db = yield* DatabaseServiceTag
+    const planExecutorService = yield* PlanExecutorServiceTag
+    const planEventDeliveryService = yield* PlanEventDeliveryServiceTag
+    const planRunService = yield* PlanRunServiceTag
+    return makePlanDeadlineService({ db, planExecutorService, planEventDeliveryService, planRunService })
+  }),
+)