@lota-sdk/core 0.4.8 → 0.4.10

package/src/services/plan/plan-deadline.service.ts

@@ -0,0 +1,533 @@
+import type {
+  DeadlineAction,
+  DeadlineReminder,
+  DeadlineSpec,
+  PlanEventType,
+  PlanNodeRunRecord,
+  PlanNodeSpecRecord,
+  PlanRunRecord,
+} from '@lota-sdk/shared'
+import { PlanEventSchema, PlanNodeRunSchema, PlanNodeSpecRecordSchema, PlanRunSchema } from '@lota-sdk/shared'
+import { Context, Schema, Effect, Layer } from 'effect'
+import { BoundQuery, RecordId } from 'surrealdb'
+
+import type { RecordIdInput } from '../../db/record-id'
+import { ensureRecordId, recordIdToString } from '../../db/record-id'
+import type { SurrealDBService } from '../../db/service'
+import { TABLES } from '../../db/tables'
+import { makeEffectTryPromiseWithMessage } from '../../effect/helpers'
+import { DatabaseServiceTag } from '../../effect/services'
+import { nowDate, unsafeDateFrom } from '../../utils/date-time'
+import type { makePlanEventDeliveryService } from './plan-event-delivery.service'
+import { PlanEventDeliveryServiceTag } from './plan-event-delivery.service'
+import type { makePlanExecutorService } from './plan-executor.service'
+import { PlanExecutorServiceTag } from './plan-executor.service'
+import type { makePlanRunService } from './plan-run.service'
+import { PlanRunServiceTag } from './plan-run.service'
+
+export type DeadlineEvaluationStatus = 'ok' | 'warning' | 'escalated' | 'missed'
+
+export interface DeadlineEvaluationResult {
+  status: DeadlineEvaluationStatus
+  activeReminder?: DeadlineReminder
+  nextTriggerAt?: Date | null
+}
+
+function resolveDeadlineTime(deadline: DeadlineSpec, nodeStartedAt: Date): Date | null {
+  return (
+    (deadline.dueAt ? unsafeDateFrom(deadline.dueAt) : null) ??
+    (deadline.durationMs !== undefined ? unsafeDateFrom(nodeStartedAt.getTime() + deadline.durationMs) : null)
+  )
+}
+
+/** Pure deadline evaluation — exported for use by other services without needing the full PlanDeadlineService. */
+export function evaluateDeadline(params: {
+  deadline: DeadlineSpec
+  nodeStartedAt: Date
+  now?: Date
+}): DeadlineEvaluationResult {
+  const now = params.now ?? nowDate()
+  const deadlineTime = resolveDeadlineTime(params.deadline, params.nodeStartedAt)
+
+  if (!deadlineTime) {
+    return { status: 'ok', nextTriggerAt: null }
+  }
+
+  const reminderEntries = [...params.deadline.reminders]
+    .map((reminder) => ({ reminder, triggerAt: unsafeDateFrom(deadlineTime.getTime() - reminder.beforeMs) }))
+    .sort((a, b) => a.triggerAt.getTime() - b.triggerAt.getTime())
+
+  let activeReminder: DeadlineReminder | undefined
+  let nextTriggerAt: Date | null = null
+
+  for (const entry of reminderEntries) {
+    if (now.getTime() < entry.triggerAt.getTime()) {
+      nextTriggerAt = entry.triggerAt
+      break
+    }
+
+    activeReminder = entry.reminder
+  }
+
+  if (now.getTime() >= deadlineTime.getTime()) {
+    return { status: 'missed', nextTriggerAt: null }
+  }
+
+  if (activeReminder) {
+    const status: DeadlineEvaluationStatus = activeReminder.action === 'escalate' ? 'escalated' : 'warning'
+    return { status, activeReminder, nextTriggerAt: nextTriggerAt ?? deadlineTime }
+  }
+
+  return { status: 'ok', nextTriggerAt: nextTriggerAt ?? deadlineTime }
+}
+
+interface PlanDeadlineDeps {
+  db: SurrealDBService
+  planExecutorService: ReturnType<typeof makePlanExecutorService>
+  planEventDeliveryService: ReturnType<typeof makePlanEventDeliveryService>
+  planRunService: ReturnType<typeof makePlanRunService>
+}
+
+class PlanDeadlineError extends Schema.TaggedErrorClass<PlanDeadlineError>()('PlanDeadlineError', {
+  message: Schema.String,
+  cause: Schema.optional(Schema.Defect),
+}) {}
+
+export function makePlanDeadlineService(deps: PlanDeadlineDeps) {
+  const { db, planEventDeliveryService, planExecutorService, planRunService } = deps
+  const effectTryPromise = makeEffectTryPromiseWithMessage(
+    (message, cause) => new PlanDeadlineError({ message, cause }),
+  )
+  const loadPlanSchedulerQueue = () =>
+    effectTryPromise(() => import('../../queues/plan-scheduler.queue'), 'Failed to load plan scheduler queue module.')
+
+  function loadNodeSpec(nodeRun: PlanNodeRunRecord): Effect.Effect<PlanNodeSpecRecord | null, PlanDeadlineError> {
+    return db
+      .findOne(
+        TABLES.PLAN_NODE_SPEC,
+        { planSpecId: ensureRecordId(nodeRun.planSpecId, TABLES.PLAN_SPEC), nodeId: nodeRun.nodeId },
+        PlanNodeSpecRecordSchema,
+      )
+      .pipe(Effect.mapError((cause) => new PlanDeadlineError({ message: 'Failed to load plan node spec.', cause })))
+  }
+
+  const collectDeadlineSweepEffect = Effect.fn('PlanDeadline.collectDeadlineSweep')(function* (now: Date) {
+    const activeNodeRuns = yield* effectTryPromise(
+      () =>
+        db.queryMany(
+          new BoundQuery(`SELECT * FROM ${TABLES.PLAN_NODE_RUN} WHERE status IN $statuses`, {
+            statuses: ['running', 'awaiting-human'],
+          }),
+          PlanNodeRunSchema,
+        ),
+      'Failed to load active plan node runs for deadline sweep.',
+    ).pipe(Effect.withSpan('PlanDeadline.loadActiveNodeRuns'))
+    const results = yield* Effect.forEach(
+      activeNodeRuns,
+      (nodeRun) =>
+        Effect.gen(function* () {
+          const nodeSpec = yield* loadNodeSpec(nodeRun).pipe(
+            Effect.mapError(
+              (cause) =>
+                new PlanDeadlineError({
+                  message: `Failed to load node spec for ${nodeRun.nodeId}.`,
+                  cause: cause.cause,
+                }),
+            ),
+          )
+          if (!nodeSpec?.deadline) {
+            return null
+          }
+
+          const evaluation = evaluateDeadline({
+            deadline: nodeSpec.deadline,
+            nodeStartedAt: unsafeDateFrom(nodeRun.startedAt ?? nodeRun.createdAt),
+            now,
+          })
+
+          return { nodeRun, nodeSpec, evaluation }
+        }),
+      { concurrency: 10 },
+    ).pipe(Effect.withSpan('PlanDeadline.evaluateNodeRuns'))
+    return { entries: results.filter((entry): entry is NonNullable<typeof entry> => entry !== null) }
+  })
+
+  const emitDeadlineEventEffect = Effect.fn('PlanDeadline.emitDeadlineEvent')(function* (params: {
+    run: PlanRunRecord
+    nodeRun: PlanNodeRunRecord
+    eventType: Extract<PlanEventType, 'deadline-warning' | 'deadline-missed' | 'escalation-triggered'>
+    emittedBy: string
+    message: string
+    detail: Record<string, unknown>
+  }) {
+    const dedupeKey =
+      typeof params.detail.dedupeKey === 'string' && params.detail.dedupeKey.trim().length > 0
+        ? params.detail.dedupeKey.trim()
+        : null
+
+    const existing =
+      dedupeKey === null
+        ? []
+        : yield* effectTryPromise(
+            () =>
+              db.queryMany(
+                new BoundQuery(
+                  `SELECT * FROM ${TABLES.PLAN_EVENT}
+                    WHERE runId = $runId
+                      AND nodeId = $nodeId
+                      AND eventType = $eventType
+                      AND detail.dedupeKey = $dedupeKey
+                    LIMIT 1`,
+                  {
+                    runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
+                    nodeId: params.nodeRun.nodeId,
+                    eventType: params.eventType,
+                    dedupeKey,
+                  },
+                ),
+                PlanEventSchema,
+              ),
+            'Failed to load existing deadline event.',
+          )
+    if (existing.length > 0) {
+      return
+    }
+
+    const spec = yield* planRunService
+      .getPlanSpecById(params.run.planSpecId)
+      .pipe(
+        Effect.mapError(
+          (cause) => new PlanDeadlineError({ message: 'Failed to load plan spec for deadline event.', cause }),
+        ),
+      )
+    const event = yield* effectTryPromise(
+      () =>
+        db.create(
+          TABLES.PLAN_EVENT,
+          {
+            id: new RecordId(TABLES.PLAN_EVENT, Bun.randomUUIDv7()),
+            planSpecId: ensureRecordId(spec.id, TABLES.PLAN_SPEC),
+            runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
+            nodeId: params.nodeRun.nodeId,
+            eventType: params.eventType,
+            message: params.message,
+            emittedBy: params.emittedBy,
+            detail: params.detail,
+          },
+          PlanEventSchema,
+        ),
+      'Failed to create deadline event.',
+    )
+    yield* planEventDeliveryService
+      .dispatchEventEffect(PlanEventSchema.parse(event))
+      .pipe(
+        Effect.mapError(
+          (error) => new PlanDeadlineError({ message: 'Failed to dispatch deadline event.', cause: error }),
+        ),
+      )
+  })
+
+  const applyDeadlineMissActionEffect = Effect.fn('PlanDeadline.applyDeadlineMissAction')(function* (params: {
+    runId: RecordIdInput
+    nodeId: string
+    threadId: string
+    organizationId: string
+    action: DeadlineAction
+    emittedBy: string
+  }) {
+    const runIdStr = recordIdToString(params.runId, TABLES.PLAN_RUN)
+    const [run, nodeRun] = yield* Effect.all([
+      planRunService
+        .getRunById(params.runId)
+        .pipe(
+          Effect.mapError(
+            (cause) => new PlanDeadlineError({ message: 'Failed to load plan run for deadline action.', cause }),
+          ),
+        ),
+      planRunService
+        .getNodeRunByNodeId(params.runId, params.nodeId)
+        .pipe(
+          Effect.mapError(
+            (cause) => new PlanDeadlineError({ message: 'Failed to load plan node run for deadline action.', cause }),
+          ),
+        ),
+    ])
+    const blockNode = (message: string): Effect.Effect<void, PlanDeadlineError> =>
+      effectTryPromise(
+        () =>
+          planExecutorService.blockNodeOnDispatchFailure({
+            threadId: params.threadId,
+            runId: runIdStr,
+            nodeId: params.nodeId,
+            emittedBy: params.emittedBy,
+            message,
+            failureClass: 'timeout_exceeded',
+          }),
+        `Failed to ${message.toLowerCase()}.`,
+      )
+
+    switch (params.action) {
+      case 'notify':
+        yield* emitDeadlineEventEffect({
+          run,
+          nodeRun,
+          eventType: 'deadline-missed',
+          emittedBy: params.emittedBy,
+          message: `Node "${params.nodeId}" has missed its deadline.`,
+          detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:notify` },
+        })
+        return
+
+      case 'escalate':
+        yield* emitDeadlineEventEffect({
+          run,
+          nodeRun,
+          eventType: 'escalation-triggered',
+          emittedBy: params.emittedBy,
+          message: `Node "${params.nodeId}" has missed its deadline and requires escalation.`,
+          detail: {
+            title: 'Deadline escalation',
+            dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:escalate`,
+            missedDeadline: true,
+          },
+        })
+        return
+
+      case 'block':
+        yield* emitDeadlineEventEffect({
+          run,
+          nodeRun,
+          eventType: 'deadline-missed',
+          emittedBy: params.emittedBy,
+          message: `Node "${params.nodeId}" has missed its deadline.`,
+          detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:block` },
+        })
+        yield* blockNode('Deadline missed — node blocked')
+        return
+
+      case 'fail':
+        yield* emitDeadlineEventEffect({
+          run,
+          nodeRun,
+          eventType: 'deadline-missed',
+          emittedBy: params.emittedBy,
+          message: `Node "${params.nodeId}" has missed its deadline.`,
+          detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:fail` },
+        })
+        yield* blockNode('Deadline missed — node failed')
+        return
+    }
+  })
+
+  const maybeEmitEscalationPolicyEventEffect = Effect.fn('PlanDeadline.maybeEmitEscalationPolicyEvent')(
+    function* (params: { run: PlanRunRecord; nodeRun: PlanNodeRunRecord; nodeSpec: PlanNodeSpecRecord; now: Date }) {
+      const escalation = params.nodeSpec.escalation
+      if (!escalation) return
+
+      const startedAt = unsafeDateFrom(params.nodeRun.startedAt ?? params.nodeRun.createdAt)
+      const runIdStr = recordIdToString(params.run.id, TABLES.PLAN_RUN)
+      const baseKey = `plan-escalation:${runIdStr}:${params.nodeRun.nodeId}`
+
+      if (escalation.autoEscalateAfterMinutes) {
+        const thresholdMs = escalation.autoEscalateAfterMinutes * 60_000
+        if (params.now.getTime() - startedAt.getTime() >= thresholdMs) {
+          yield* emitDeadlineEventEffect({
+            run: params.run,
+            nodeRun: params.nodeRun,
+            eventType: 'escalation-triggered',
+            emittedBy: 'plan-deadline-checker',
+            message: `Node "${params.nodeRun.nodeId}" exceeded its auto-escalation threshold.`,
+            detail: {
+              title: 'Execution escalation',
+              dedupeKey: `${baseKey}:auto:${escalation.autoEscalateAfterMinutes}`,
+              autoEscalateAfterMinutes: escalation.autoEscalateAfterMinutes,
+              ...(escalation.escalateToAgent ? { escalateToAgent: escalation.escalateToAgent } : {}),
+              ...(escalation.escalateToUser ? { escalateToUser: escalation.escalateToUser } : {}),
+            },
+          })
+        }
+      }
+
+      if (escalation.deadlineThresholdMinutes && params.nodeSpec.deadline) {
+        const dueAt = resolveDeadlineTime(params.nodeSpec.deadline, startedAt)
+        if (!dueAt) return
+
+        const thresholdMs = escalation.deadlineThresholdMinutes * 60_000
+        if (dueAt.getTime() - params.now.getTime() <= thresholdMs && dueAt.getTime() > params.now.getTime()) {
+          yield* emitDeadlineEventEffect({
+            run: params.run,
+            nodeRun: params.nodeRun,
+            eventType: 'escalation-triggered',
+            emittedBy: 'plan-deadline-checker',
+            message: `Node "${params.nodeRun.nodeId}" is inside its escalation threshold.`,
+            detail: {
+              title: 'Deadline escalation threshold',
+              dedupeKey: `${baseKey}:threshold:${escalation.deadlineThresholdMinutes}`,
+              deadlineThresholdMinutes: escalation.deadlineThresholdMinutes,
+              ...(escalation.escalateToAgent ? { escalateToAgent: escalation.escalateToAgent } : {}),
+              ...(escalation.escalateToUser ? { escalateToUser: escalation.escalateToUser } : {}),
+            },
+          })
+        }
+      }
+    },
+  )
+
+  const enqueueDeadlineCheckEffect = Effect.fn('PlanDeadline.enqueueDeadlineCheck')(function* (scheduledFor: Date) {
+    const { enqueueDeadlineCheck } = yield* loadPlanSchedulerQueue()
+    yield* effectTryPromise(() => enqueueDeadlineCheck(scheduledFor), 'Failed to enqueue deadline check.')
+  })
+
+  const checkDeadlinesEffect = Effect.fn('PlanDeadline.checkDeadlines')(function* (now?: Date) {
+    const currentTime = now ?? nowDate()
+
+    const sweep = yield* collectDeadlineSweepEffect(currentTime)
+    if (sweep.entries.length === 0) {
+      return
+    }
+
+    const runCache = new Map<string, PlanRunRecord>()
+
+    const handleEntry = (entry: (typeof sweep.entries)[number]): Effect.Effect<void, PlanDeadlineError> =>
+      Effect.gen(function* () {
+        const deadline = entry.nodeSpec.deadline
+        if (!deadline) {
+          return
+        }
+
+        const runIdStr = recordIdToString(entry.nodeRun.runId, TABLES.PLAN_RUN)
+        const cachedRun = runCache.get(runIdStr)
+        const run =
+          cachedRun ??
+          (yield* effectTryPromise(
+            () =>
+              db.findOne(TABLES.PLAN_RUN, { id: ensureRecordId(entry.nodeRun.runId, TABLES.PLAN_RUN) }, PlanRunSchema),
+            'Failed to load plan run during deadline check.',
+          ))
+        if (!run) {
+          return
+        }
+        runCache.set(runIdStr, run)
+
+        const dedupeKeyBase = `plan-deadline:${runIdStr}:${entry.nodeRun.nodeId}`
+        const actionEffect =
+          entry.evaluation.status === 'warning'
+            ? emitDeadlineEventEffect({
+                run,
+                nodeRun: entry.nodeRun,
+                eventType: 'deadline-warning',
+                emittedBy: 'plan-deadline-checker',
+                message:
+                  entry.evaluation.activeReminder?.message ??
+                  `Node "${entry.nodeRun.nodeId}" is approaching its deadline.`,
+                detail: {
+                  title: 'Deadline approaching',
+                  reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
+                  dedupeKey: `${dedupeKeyBase}:warning:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
+                },
+              })
+            : entry.evaluation.status === 'escalated'
+              ? emitDeadlineEventEffect({
+                  run,
+                  nodeRun: entry.nodeRun,
+                  eventType: 'escalation-triggered',
+                  emittedBy: 'plan-deadline-checker',
+                  message:
+                    entry.evaluation.activeReminder?.message ??
+                    `Node "${entry.nodeRun.nodeId}" deadline requires escalation.`,
+                  detail: {
+                    title: 'Deadline escalation',
+                    reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
+                    dedupeKey: `${dedupeKeyBase}:escalated:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
+                  },
+                })
+              : applyDeadlineMissActionEffect({
+                  runId: entry.nodeRun.runId,
+                  nodeId: entry.nodeRun.nodeId,
+                  threadId: recordIdToString(run.threadId, TABLES.THREAD),
+                  organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
+                  action: deadline.missAction,
+                  emittedBy: 'plan-deadline-checker',
+                })
+
+        yield* actionEffect
+        yield* maybeEmitEscalationPolicyEventEffect({
+          run,
+          nodeRun: entry.nodeRun,
+          nodeSpec: entry.nodeSpec,
+          now: currentTime,
+        })
+      })
+
+    yield* Effect.forEach(
+      sweep.entries.filter((entry) => entry.evaluation.status !== 'ok'),
+      handleEntry,
+      { concurrency: 3, discard: true },
+    ).pipe(Effect.withSpan('PlanDeadline.processDeadlineEntries'))
+
+    const nextTriggerAt =
+      sweep.entries
+        .map((entry) => entry.evaluation.nextTriggerAt)
+        .filter((value): value is Date => {
+          if (!value) return false
+          return value.getTime() > currentTime.getTime()
+        })
+        .sort((a, b) => a.getTime() - b.getTime())
+        .at(0) ?? null
+
+    if (nextTriggerAt) {
+      yield* enqueueDeadlineCheckEffect(nextTriggerAt)
+    }
+  })
+
+  const recoverDeadlineChecksEffect = Effect.fn('PlanDeadline.recoverDeadlineChecks')(function* (
+    now: Date = nowDate(),
+  ) {
+    const sweep = yield* collectDeadlineSweepEffect(now)
+    const hasDueAction = sweep.entries.some((entry) => entry.evaluation.status !== 'ok')
+    if (hasDueAction) {
+      yield* enqueueDeadlineCheckEffect(now)
+      return
+    }
+
+    const nextTriggerAt =
+      sweep.entries
+        .map((entry) => entry.evaluation.nextTriggerAt)
+        .filter((value): value is Date => {
+          if (!value) return false
+          return value.getTime() > now.getTime()
+        })
+        .sort((a, b) => a.getTime() - b.getTime())
+        .at(0) ?? null
+
+    if (nextTriggerAt) {
+      yield* enqueueDeadlineCheckEffect(nextTriggerAt)
+    }
+  })
+
+  return {
+    evaluateDeadline,
+    checkDeadlines: checkDeadlinesEffect,
+    recoverDeadlineChecks: recoverDeadlineChecksEffect,
+    applyDeadlineMissAction: applyDeadlineMissActionEffect,
+    maybeEmitEscalationPolicyEvent: maybeEmitEscalationPolicyEventEffect,
+    emitDeadlineEvent: emitDeadlineEventEffect,
+    enqueueDeadlineCheck: enqueueDeadlineCheckEffect,
+  }
+}
+
+export class PlanDeadlineServiceTag extends Context.Service<
+  PlanDeadlineServiceTag,
+  ReturnType<typeof makePlanDeadlineService>
+>()('@lota-sdk/core/PlanDeadlineService') {}
+
+export const PlanDeadlineServiceLive = Layer.effect(
+  PlanDeadlineServiceTag,
+  Effect.gen(function* () {
+    const db = yield* DatabaseServiceTag
+    const planExecutorService = yield* PlanExecutorServiceTag
+    const planEventDeliveryService = yield* PlanEventDeliveryServiceTag
+    const planRunService = yield* PlanRunServiceTag
+    return makePlanDeadlineService({ db, planExecutorService, planEventDeliveryService, planRunService })
+  }),
+)