@lota-sdk/core 0.1.20 → 0.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/infrastructure/schema/02_execution_plan.surql +4 -0
  2. package/package.json +6 -6
  3. package/src/ai-gateway/ai-gateway.ts +2 -4
  4. package/src/create-runtime.ts +8 -0
  5. package/src/queues/document-processor.queue.ts +11 -8
  6. package/src/queues/index.ts +1 -0
  7. package/src/queues/plan-agent-heartbeat.queue.ts +100 -0
  8. package/src/queues/queue-factory.ts +12 -11
  9. package/src/redis/redis-lease-lock.ts +1 -1
  10. package/src/runtime/agent-runtime-policy.ts +41 -4
  11. package/src/runtime/execution-plan-visibility.ts +23 -0
  12. package/src/runtime/execution-plan.ts +1 -0
  13. package/src/runtime/runtime-extensions.ts +26 -0
  14. package/src/runtime/runtime-worker-registry.ts +9 -1
  15. package/src/services/agent-executor.service.ts +6 -0
  16. package/src/services/execution-plan.service.ts +51 -36
  17. package/src/services/index.ts +3 -0
  18. package/src/services/ownership-dispatcher.service.ts +50 -8
  19. package/src/services/plan-agent-heartbeat.service.ts +136 -0
  20. package/src/services/plan-agent-query.service.ts +238 -0
  21. package/src/services/plan-builder.service.ts +11 -1
  22. package/src/services/plan-compiler.service.ts +2 -0
  23. package/src/services/plan-deadline.service.ts +186 -44
  24. package/src/services/plan-event-delivery.service.ts +170 -0
  25. package/src/services/plan-executor.service.ts +107 -3
  26. package/src/services/plan-helpers.ts +13 -0
  27. package/src/services/plan-run.service.ts +4 -0
  28. package/src/services/plan-template.service.ts +0 -1
  29. package/src/services/workstream-turn-preparation.service.ts +452 -176
  30. package/src/services/workstream-turn.ts +101 -1
  31. package/src/services/workstream.service.ts +76 -16
  32. package/src/tools/execution-plan.tool.ts +0 -2
@@ -0,0 +1,238 @@
1
+ import type { PlanExecutionVisibility, PlanNodeSpecRecord, PlanRunRecord, PlanSpecRecord } from '@lota-sdk/shared'
2
+ import { PlanRunSchema } from '@lota-sdk/shared'
3
+
4
+ import type { RecordIdInput } from '../db/record-id'
5
+ import { ensureRecordId, recordIdToString } from '../db/record-id'
6
+ import { databaseService } from '../db/service'
7
+ import { TABLES } from '../db/tables'
8
+ import { resolvePlanNodeExecutionVisibility } from '../runtime/execution-plan-visibility'
9
+ import { planDeadlineService } from './plan-deadline.service'
10
+ import { planRunService } from './plan-run.service'
11
+
12
+ const ACTIVE_PLAN_RUN_STATUSES = ['running', 'awaiting-human', 'blocked'] as const
13
+ const ACTIONABLE_NODE_STATUSES = new Set(['ready', 'running'])
14
+ const DEADLINE_TRACKED_NODE_STATUSES = new Set(['ready', 'running', 'awaiting-human'])
15
+
16
+ export interface ActionablePlanAgentNode {
17
+ organizationId: string
18
+ workstreamId: string
19
+ runId: string
20
+ nodeId: string
21
+ agentId: string
22
+ status: 'ready' | 'running'
23
+ visibility: PlanExecutionVisibility
24
+ }
25
+
26
+ export interface ApproachingDeadlineNode {
27
+ organizationId: string
28
+ workstreamId: string
29
+ runId: string
30
+ nodeId: string
31
+ agentId?: string
32
+ visibility?: PlanExecutionVisibility
33
+ dueAt?: string
34
+ status: 'warning' | 'escalated' | 'missed'
35
+ nextTriggerAt?: string | null
36
+ }
37
+
38
+ export interface RecentlyUnblockedNode {
39
+ organizationId: string
40
+ workstreamId: string
41
+ runId: string
42
+ nodeId: string
43
+ agentId: string
44
+ visibility: PlanExecutionVisibility
45
+ unblockedAt: string
46
+ sourceEventType: 'node-unblocked' | 'approval-resolved'
47
+ }
48
+
49
+ function isVisibleAgentNode(params: {
50
+ nodeSpec: PlanNodeSpecRecord
51
+ spec: PlanSpecRecord
52
+ }): { agentId: string; visibility: PlanExecutionVisibility } | null {
53
+ if (params.nodeSpec.owner.executorType !== 'agent') {
54
+ return null
55
+ }
56
+
57
+ const visibility = resolvePlanNodeExecutionVisibility(params.spec, params.nodeSpec)
58
+ const isVisible = visibility === 'visible'
59
+ if (!isVisible) {
60
+ return null
61
+ }
62
+
63
+ return { agentId: params.nodeSpec.owner.ref, visibility }
64
+ }
65
+
66
+ class PlanAgentQueryService {
67
+ async getActionableNodesForAgent(params: {
68
+ agentId?: string
69
+ organizationId?: RecordIdInput
70
+ }): Promise<ActionablePlanAgentNode[]> {
71
+ const runs = await this.listActiveRuns(params.organizationId)
72
+ const actionable: ActionablePlanAgentNode[] = []
73
+
74
+ for (const run of runs) {
75
+ const spec = await planRunService.getPlanSpecById(run.planSpecId)
76
+ const currentNodeId = run.currentNodeId
77
+ if (!currentNodeId) {
78
+ continue
79
+ }
80
+
81
+ const [nodeSpec, nodeRun] = await Promise.all([
82
+ planRunService.getNodeSpecByNodeId(spec.id, currentNodeId),
83
+ planRunService.getNodeRunByNodeId(run.id, currentNodeId),
84
+ ])
85
+ if (!ACTIONABLE_NODE_STATUSES.has(nodeRun.status)) {
86
+ continue
87
+ }
88
+
89
+ const visibleTarget = isVisibleAgentNode({ nodeSpec, spec })
90
+ if (!visibleTarget) {
91
+ continue
92
+ }
93
+ if (params.agentId && params.agentId !== visibleTarget.agentId) {
94
+ continue
95
+ }
96
+
97
+ actionable.push({
98
+ organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
99
+ workstreamId: recordIdToString(run.workstreamId, TABLES.WORKSTREAM),
100
+ runId: recordIdToString(run.id, TABLES.PLAN_RUN),
101
+ nodeId: nodeSpec.nodeId,
102
+ agentId: visibleTarget.agentId,
103
+ status: nodeRun.status as 'ready' | 'running',
104
+ visibility: visibleTarget.visibility,
105
+ })
106
+ }
107
+
108
+ return actionable
109
+ }
110
+
111
+ async getApproachingDeadlines(params?: {
112
+ organizationId?: RecordIdInput
113
+ withinMinutes?: number
114
+ }): Promise<ApproachingDeadlineNode[]> {
115
+ const now = new Date()
116
+ const maxWindowMs = (params?.withinMinutes ?? 60) * 60_000
117
+ const runs = await this.listActiveRuns(params?.organizationId)
118
+ const matches: ApproachingDeadlineNode[] = []
119
+
120
+ for (const run of runs) {
121
+ const spec = await planRunService.getPlanSpecById(run.planSpecId)
122
+ const [nodeSpecs, nodeRuns] = await Promise.all([
123
+ planRunService.listNodeSpecs(spec.id),
124
+ planRunService.listNodeRuns(run.id),
125
+ ])
126
+ const nodeRunsById = new Map(nodeRuns.map((nodeRun) => [nodeRun.nodeId, nodeRun]))
127
+
128
+ for (const nodeSpec of nodeSpecs) {
129
+ const nodeRun = nodeRunsById.get(nodeSpec.nodeId)
130
+ if (!nodeRun || !nodeSpec.deadline || !DEADLINE_TRACKED_NODE_STATUSES.has(nodeRun.status)) {
131
+ continue
132
+ }
133
+
134
+ const evaluation = planDeadlineService.evaluateDeadline({
135
+ deadline: nodeSpec.deadline,
136
+ nodeStartedAt: new Date(nodeRun.startedAt ?? nodeRun.createdAt),
137
+ now,
138
+ })
139
+ if (evaluation.status === 'ok') {
140
+ continue
141
+ }
142
+
143
+ const nextTriggerTime = evaluation.nextTriggerAt?.getTime()
144
+ if (nextTriggerTime && nextTriggerTime - now.getTime() > maxWindowMs && evaluation.status !== 'missed') {
145
+ continue
146
+ }
147
+
148
+ const visibleTarget = isVisibleAgentNode({ nodeSpec, spec })
149
+ matches.push({
150
+ organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
151
+ workstreamId: recordIdToString(run.workstreamId, TABLES.WORKSTREAM),
152
+ runId: recordIdToString(run.id, TABLES.PLAN_RUN),
153
+ nodeId: nodeSpec.nodeId,
154
+ ...(visibleTarget ? { agentId: visibleTarget.agentId, visibility: visibleTarget.visibility } : {}),
155
+ ...(nodeSpec.deadline.dueAt ? { dueAt: nodeSpec.deadline.dueAt } : {}),
156
+ status: evaluation.status,
157
+ nextTriggerAt: evaluation.nextTriggerAt?.toISOString() ?? null,
158
+ })
159
+ }
160
+ }
161
+
162
+ return matches
163
+ }
164
+
165
+ async getRecentlyUnblockedNodes(params?: {
166
+ organizationId?: RecordIdInput
167
+ sinceMinutes?: number
168
+ agentId?: string
169
+ }): Promise<RecentlyUnblockedNode[]> {
170
+ const since = new Date(Date.now() - (params?.sinceMinutes ?? 30) * 60_000)
171
+ const runs = await this.listActiveRuns(params?.organizationId)
172
+ const matches: RecentlyUnblockedNode[] = []
173
+
174
+ for (const run of runs) {
175
+ const spec = await planRunService.getPlanSpecById(run.planSpecId)
176
+ const [nodeSpecs, events] = await Promise.all([
177
+ planRunService.listNodeSpecs(spec.id),
178
+ planRunService.listEvents(run.id, 200),
179
+ ])
180
+ const nodeSpecsById = new Map(nodeSpecs.map((nodeSpec) => [nodeSpec.nodeId, nodeSpec]))
181
+
182
+ for (const event of events) {
183
+ if (
184
+ (event.eventType !== 'node-unblocked' && event.eventType !== 'approval-resolved') ||
185
+ !event.nodeId ||
186
+ new Date(event.createdAt).getTime() < since.getTime()
187
+ ) {
188
+ continue
189
+ }
190
+
191
+ const currentNodeId = run.currentNodeId ?? event.nodeId
192
+ const nodeSpec = nodeSpecsById.get(currentNodeId)
193
+ if (!nodeSpec) {
194
+ continue
195
+ }
196
+
197
+ const visibleTarget = isVisibleAgentNode({ nodeSpec, spec })
198
+ if (!visibleTarget) {
199
+ continue
200
+ }
201
+ if (params?.agentId && params.agentId !== visibleTarget.agentId) {
202
+ continue
203
+ }
204
+
205
+ matches.push({
206
+ organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
207
+ workstreamId: recordIdToString(run.workstreamId, TABLES.WORKSTREAM),
208
+ runId: recordIdToString(run.id, TABLES.PLAN_RUN),
209
+ nodeId: currentNodeId,
210
+ agentId: visibleTarget.agentId,
211
+ visibility: visibleTarget.visibility,
212
+ unblockedAt: new Date(event.createdAt).toISOString(),
213
+ sourceEventType: event.eventType,
214
+ })
215
+ }
216
+ }
217
+
218
+ return matches
219
+ }
220
+
221
+ private async listActiveRuns(organizationId?: RecordIdInput): Promise<PlanRunRecord[]> {
222
+ const bindings = {
223
+ statuses: [...ACTIVE_PLAN_RUN_STATUSES],
224
+ ...(organizationId ? { organizationId: ensureRecordId(organizationId, TABLES.ORGANIZATION) } : {}),
225
+ }
226
+
227
+ const whereOrganization = organizationId ? ' AND organizationId = $organizationId' : ''
228
+ return databaseService.queryMany(
229
+ {
230
+ query: `SELECT * FROM ${TABLES.PLAN_RUN} WHERE status INSIDE $statuses${whereOrganization} ORDER BY updatedAt DESC`,
231
+ bindings,
232
+ },
233
+ PlanRunSchema,
234
+ )
235
+ }
236
+ }
237
+
238
+ export const planAgentQueryService = new PlanAgentQueryService()
@@ -1,5 +1,7 @@
1
1
  import type { PlanDraft } from '@lota-sdk/shared'
2
2
 
3
+ import { isExecutableConditionExpression } from './plan-helpers'
4
+
3
5
  function buildImplicitLinearEdges(draft: PlanDraft) {
4
6
  if (draft.edges.length > 0 || draft.nodes.length <= 1) {
5
7
  return draft.edges
@@ -46,7 +48,15 @@ class PlanBuilderService {
46
48
  webPolicy: node.contextPolicy.webPolicy,
47
49
  },
48
50
  })),
49
- edges: draft.edges.map((edge) => ({ ...edge, map: { ...edge.map } })),
51
+ edges: draft.edges.map((edge) => {
52
+ const normalizedWhen = edge.when?.trim()
53
+ const { when: _when, ...edgeWithoutWhen } = edge
54
+ return {
55
+ ...edgeWithoutWhen,
56
+ ...(normalizedWhen && isExecutableConditionExpression(normalizedWhen) ? { when: normalizedWhen } : {}),
57
+ map: { ...edge.map },
58
+ }
59
+ }),
50
60
  schemas: structuredClone(draft.schemas),
51
61
  entryNodeIds: [...(draft.entryNodeIds ?? [])],
52
62
  }
@@ -72,8 +72,10 @@ class PlanCompilerService {
72
72
  attachmentPolicy: node.contextPolicy.attachmentPolicy,
73
73
  webPolicy: node.contextPolicy.webPolicy,
74
74
  },
75
+ executionVisibility: node.executionVisibility,
75
76
  ...(node.schedule ? { schedule: node.schedule } : {}),
76
77
  ...(node.deadline ? { deadline: node.deadline } : {}),
78
+ ...(node.escalation ? { escalation: node.escalation } : {}),
77
79
  ...(node.monitoringConfig ? { monitoringConfig: node.monitoringConfig } : {}),
78
80
  ...(node.delayAfterPredecessorMs ? { delayAfterPredecessorMs: node.delayAfterPredecessorMs } : {}),
79
81
  ...(node.deliberationConfig ? { deliberationConfig: node.deliberationConfig } : {}),
@@ -2,17 +2,20 @@ import type {
2
2
  DeadlineAction,
3
3
  DeadlineReminder,
4
4
  DeadlineSpec,
5
+ PlanEventType,
5
6
  PlanNodeRunRecord,
6
7
  PlanNodeSpecRecord,
7
8
  PlanRunRecord,
8
9
  } from '@lota-sdk/shared'
9
- import { PlanNodeRunSchema, PlanNodeSpecRecordSchema, PlanRunSchema } from '@lota-sdk/shared'
10
+ import { PlanEventSchema, PlanNodeRunSchema, PlanNodeSpecRecordSchema, PlanRunSchema } from '@lota-sdk/shared'
11
+ import { RecordId } from 'surrealdb'
10
12
 
11
13
  import type { RecordIdInput } from '../db/record-id'
12
14
  import { ensureRecordId, recordIdToString } from '../db/record-id'
13
15
  import { databaseService } from '../db/service'
14
16
  import { TABLES } from '../db/tables'
15
- import { getNotificationService } from './notification.service'
17
+ import { planEventDeliveryService } from './plan-event-delivery.service'
18
+ import { planRunService } from './plan-run.service'
16
19
 
17
20
  export type DeadlineEvaluationStatus = 'ok' | 'warning' | 'escalated' | 'missed'
18
21
 
@@ -89,45 +92,53 @@ class PlanDeadlineService {
89
92
  runCache.set(runIdStr, run)
90
93
  }
91
94
 
92
- const organizationId = recordIdToString(run.organizationId, TABLES.ORGANIZATION)
93
- const workstreamId = recordIdToString(run.workstreamId, TABLES.WORKSTREAM)
94
- const notificationService = getNotificationService()
95
95
  const dedupeKeyBase = `plan-deadline:${runIdStr}:${entry.nodeRun.nodeId}`
96
96
 
97
97
  if (entry.evaluation.status === 'warning') {
98
- await notificationService.remind({
99
- organizationId,
100
- workstreamId,
101
- runId: runIdStr,
102
- nodeId: entry.nodeRun.nodeId,
103
- severity: 'warning',
104
- title: 'Deadline approaching',
105
- body:
98
+ await this.emitDeadlineEvent({
99
+ run,
100
+ nodeRun: entry.nodeRun,
101
+ eventType: 'deadline-warning',
102
+ emittedBy: 'plan-deadline-checker',
103
+ message:
106
104
  entry.evaluation.activeReminder?.message ?? `Node "${entry.nodeRun.nodeId}" is approaching its deadline.`,
107
- dedupeKey: `${dedupeKeyBase}:warning:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
105
+ detail: {
106
+ title: 'Deadline approaching',
107
+ reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
108
+ dedupeKey: `${dedupeKeyBase}:warning:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
109
+ },
108
110
  })
109
111
  } else if (entry.evaluation.status === 'escalated') {
110
- await notificationService.escalate({
111
- organizationId,
112
- workstreamId,
113
- runId: runIdStr,
114
- nodeId: entry.nodeRun.nodeId,
115
- severity: 'urgent',
116
- title: 'Deadline escalation',
117
- body:
112
+ await this.emitDeadlineEvent({
113
+ run,
114
+ nodeRun: entry.nodeRun,
115
+ eventType: 'escalation-triggered',
116
+ emittedBy: 'plan-deadline-checker',
117
+ message:
118
118
  entry.evaluation.activeReminder?.message ?? `Node "${entry.nodeRun.nodeId}" deadline requires escalation.`,
119
- dedupeKey: `${dedupeKeyBase}:escalated:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
119
+ detail: {
120
+ title: 'Deadline escalation',
121
+ reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
122
+ dedupeKey: `${dedupeKeyBase}:escalated:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
123
+ },
120
124
  })
121
125
  } else {
122
126
  await this.applyDeadlineMissAction({
123
127
  runId: entry.nodeRun.runId,
124
128
  nodeId: entry.nodeRun.nodeId,
125
- workstreamId,
126
- organizationId,
129
+ workstreamId: recordIdToString(run.workstreamId, TABLES.WORKSTREAM),
130
+ organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
127
131
  action: deadline.missAction,
128
132
  emittedBy: 'plan-deadline-checker',
129
133
  })
130
134
  }
135
+
136
+ await this.maybeEmitEscalationPolicyEvent({
137
+ run,
138
+ nodeRun: entry.nodeRun,
139
+ nodeSpec: entry.nodeSpec,
140
+ now: currentTime,
141
+ })
131
142
  }
132
143
 
133
144
  const nextTriggerAt =
@@ -222,37 +233,46 @@ class PlanDeadlineService {
222
233
  action: DeadlineAction
223
234
  emittedBy: string
224
235
  }): Promise<void> {
225
- const notificationService = getNotificationService()
226
236
  const runIdStr = recordIdToString(params.runId, TABLES.PLAN_RUN)
237
+ const run = await planRunService.getRunById(params.runId)
238
+ const nodeRun = await planRunService.getNodeRunByNodeId(params.runId, params.nodeId)
227
239
 
228
240
  switch (params.action) {
229
241
  case 'notify':
230
- await notificationService.notify({
231
- organizationId: params.organizationId,
232
- workstreamId: params.workstreamId,
233
- runId: runIdStr,
234
- nodeId: params.nodeId,
235
- severity: 'warning',
236
- title: 'Deadline missed',
237
- body: `Node "${params.nodeId}" has missed its deadline.`,
238
- dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:notify`,
242
+ await this.emitDeadlineEvent({
243
+ run,
244
+ nodeRun,
245
+ eventType: 'deadline-missed',
246
+ emittedBy: params.emittedBy,
247
+ message: `Node "${params.nodeId}" has missed its deadline.`,
248
+ detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:notify` },
239
249
  })
240
250
  break
241
251
 
242
252
  case 'escalate':
243
- await notificationService.escalate({
244
- organizationId: params.organizationId,
245
- workstreamId: params.workstreamId,
246
- runId: runIdStr,
247
- nodeId: params.nodeId,
248
- severity: 'urgent',
249
- title: 'Deadline escalation',
250
- body: `Node "${params.nodeId}" has missed its deadline and requires escalation.`,
251
- dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:escalate`,
253
+ await this.emitDeadlineEvent({
254
+ run,
255
+ nodeRun,
256
+ eventType: 'escalation-triggered',
257
+ emittedBy: params.emittedBy,
258
+ message: `Node "${params.nodeId}" has missed its deadline and requires escalation.`,
259
+ detail: {
260
+ title: 'Deadline escalation',
261
+ dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:escalate`,
262
+ missedDeadline: true,
263
+ },
252
264
  })
253
265
  break
254
266
 
255
267
  case 'block': {
268
+ await this.emitDeadlineEvent({
269
+ run,
270
+ nodeRun,
271
+ eventType: 'deadline-missed',
272
+ emittedBy: params.emittedBy,
273
+ message: `Node "${params.nodeId}" has missed its deadline.`,
274
+ detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:block` },
275
+ })
256
276
  const { planExecutorService } = await import('./plan-executor.service')
257
277
  await planExecutorService.blockNodeOnDispatchFailure({
258
278
  workstreamId: params.workstreamId,
@@ -266,6 +286,14 @@ class PlanDeadlineService {
266
286
  }
267
287
 
268
288
  case 'fail': {
289
+ await this.emitDeadlineEvent({
290
+ run,
291
+ nodeRun,
292
+ eventType: 'deadline-missed',
293
+ emittedBy: params.emittedBy,
294
+ message: `Node "${params.nodeId}" has missed its deadline.`,
295
+ detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:fail` },
296
+ })
269
297
  const { planExecutorService } = await import('./plan-executor.service')
270
298
  await planExecutorService.blockNodeOnDispatchFailure({
271
299
  workstreamId: params.workstreamId,
@@ -280,6 +308,120 @@ class PlanDeadlineService {
280
308
  }
281
309
  }
282
310
 
311
+ private async maybeEmitEscalationPolicyEvent(params: {
312
+ run: PlanRunRecord
313
+ nodeRun: PlanNodeRunRecord
314
+ nodeSpec: PlanNodeSpecRecord
315
+ now: Date
316
+ }): Promise<void> {
317
+ const escalation = params.nodeSpec.escalation
318
+ if (!escalation) return
319
+
320
+ const startedAt = new Date(params.nodeRun.startedAt ?? params.nodeRun.createdAt)
321
+ const runIdStr = recordIdToString(params.run.id, TABLES.PLAN_RUN)
322
+ const baseKey = `plan-escalation:${runIdStr}:${params.nodeRun.nodeId}`
323
+
324
+ if (escalation.autoEscalateAfterMinutes) {
325
+ const thresholdMs = escalation.autoEscalateAfterMinutes * 60_000
326
+ if (params.now.getTime() - startedAt.getTime() >= thresholdMs) {
327
+ await this.emitDeadlineEvent({
328
+ run: params.run,
329
+ nodeRun: params.nodeRun,
330
+ eventType: 'escalation-triggered',
331
+ emittedBy: 'plan-deadline-checker',
332
+ message: `Node "${params.nodeRun.nodeId}" exceeded its auto-escalation threshold.`,
333
+ detail: {
334
+ title: 'Execution escalation',
335
+ dedupeKey: `${baseKey}:auto:${escalation.autoEscalateAfterMinutes}`,
336
+ autoEscalateAfterMinutes: escalation.autoEscalateAfterMinutes,
337
+ ...(escalation.escalateToAgent ? { escalateToAgent: escalation.escalateToAgent } : {}),
338
+ ...(escalation.escalateToUser ? { escalateToUser: escalation.escalateToUser } : {}),
339
+ },
340
+ })
341
+ }
342
+ }
343
+
344
+ if (escalation.deadlineThresholdMinutes && params.nodeSpec.deadline) {
345
+ const dueAt = this.resolveDeadlineTime(params.nodeSpec.deadline, startedAt)
346
+ if (!dueAt) return
347
+
348
+ const thresholdMs = escalation.deadlineThresholdMinutes * 60_000
349
+ if (dueAt.getTime() - params.now.getTime() <= thresholdMs && dueAt.getTime() > params.now.getTime()) {
350
+ await this.emitDeadlineEvent({
351
+ run: params.run,
352
+ nodeRun: params.nodeRun,
353
+ eventType: 'escalation-triggered',
354
+ emittedBy: 'plan-deadline-checker',
355
+ message: `Node "${params.nodeRun.nodeId}" is inside its escalation threshold.`,
356
+ detail: {
357
+ title: 'Deadline escalation threshold',
358
+ dedupeKey: `${baseKey}:threshold:${escalation.deadlineThresholdMinutes}`,
359
+ deadlineThresholdMinutes: escalation.deadlineThresholdMinutes,
360
+ ...(escalation.escalateToAgent ? { escalateToAgent: escalation.escalateToAgent } : {}),
361
+ ...(escalation.escalateToUser ? { escalateToUser: escalation.escalateToUser } : {}),
362
+ },
363
+ })
364
+ }
365
+ }
366
+ }
367
+
368
+ private async emitDeadlineEvent(params: {
369
+ run: PlanRunRecord
370
+ nodeRun: PlanNodeRunRecord
371
+ eventType: Extract<PlanEventType, 'deadline-warning' | 'deadline-missed' | 'escalation-triggered'>
372
+ emittedBy: string
373
+ message: string
374
+ detail: Record<string, unknown>
375
+ }): Promise<void> {
376
+ const dedupeKey =
377
+ typeof params.detail.dedupeKey === 'string' && params.detail.dedupeKey.trim().length > 0
378
+ ? params.detail.dedupeKey.trim()
379
+ : null
380
+
381
+ if (dedupeKey) {
382
+ const existing = await databaseService.queryMany(
383
+ {
384
+ query: `SELECT * FROM ${TABLES.PLAN_EVENT}
385
+ WHERE runId = $runId
386
+ AND nodeId = $nodeId
387
+ AND eventType = $eventType
388
+ AND detail.dedupeKey = $dedupeKey
389
+ LIMIT 1`,
390
+ bindings: {
391
+ runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
392
+ nodeId: params.nodeRun.nodeId,
393
+ eventType: params.eventType,
394
+ dedupeKey,
395
+ },
396
+ },
397
+ PlanEventSchema,
398
+ )
399
+ if (existing.length > 0) {
400
+ return
401
+ }
402
+ }
403
+
404
+ const spec = await planRunService.getPlanSpecById(params.run.planSpecId)
405
+ const event = PlanEventSchema.parse(
406
+ await databaseService.create(
407
+ TABLES.PLAN_EVENT,
408
+ {
409
+ id: new RecordId(TABLES.PLAN_EVENT, Bun.randomUUIDv7()),
410
+ planSpecId: ensureRecordId(spec.id, TABLES.PLAN_SPEC),
411
+ runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
412
+ nodeId: params.nodeRun.nodeId,
413
+ eventType: params.eventType,
414
+ message: params.message,
415
+ emittedBy: params.emittedBy,
416
+ detail: params.detail,
417
+ },
418
+ PlanEventSchema,
419
+ ),
420
+ )
421
+
422
+ await planEventDeliveryService.dispatchEvent(event)
423
+ }
424
+
283
425
  private async enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
284
426
  const { enqueueDeadlineCheck } = await import('../queues/plan-scheduler.queue')
285
427
  await enqueueDeadlineCheck(scheduledFor)