@lota-sdk/core 0.1.21 → 0.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/infrastructure/schema/02_execution_plan.surql +4 -0
- package/package.json +3 -3
- package/src/ai-gateway/ai-gateway.ts +4 -0
- package/src/create-runtime.ts +8 -0
- package/src/queues/index.ts +1 -0
- package/src/queues/plan-agent-heartbeat.queue.ts +100 -0
- package/src/redis/redis-lease-lock.ts +1 -1
- package/src/runtime/agent-runtime-policy.ts +41 -4
- package/src/runtime/execution-plan-visibility.ts +23 -0
- package/src/runtime/execution-plan.ts +1 -0
- package/src/runtime/runtime-extensions.ts +26 -0
- package/src/runtime/runtime-worker-registry.ts +9 -1
- package/src/services/agent-executor.service.ts +6 -0
- package/src/services/execution-plan.service.ts +51 -36
- package/src/services/index.ts +3 -0
- package/src/services/ownership-dispatcher.service.ts +50 -8
- package/src/services/plan-agent-heartbeat.service.ts +136 -0
- package/src/services/plan-agent-query.service.ts +238 -0
- package/src/services/plan-builder.service.ts +11 -1
- package/src/services/plan-compiler.service.ts +2 -0
- package/src/services/plan-deadline.service.ts +186 -44
- package/src/services/plan-event-delivery.service.ts +170 -0
- package/src/services/plan-executor.service.ts +107 -3
- package/src/services/plan-helpers.ts +13 -0
- package/src/services/plan-run.service.ts +4 -0
- package/src/services/plan-template.service.ts +0 -1
- package/src/services/workstream-turn-preparation.service.ts +452 -176
- package/src/services/workstream-turn.ts +101 -1
- package/src/services/workstream.service.ts +76 -16
- package/src/tools/execution-plan.tool.ts +0 -2
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import type { PlanExecutionVisibility, PlanNodeSpecRecord, PlanRunRecord, PlanSpecRecord } from '@lota-sdk/shared'
|
|
2
|
+
import { PlanRunSchema } from '@lota-sdk/shared'
|
|
3
|
+
|
|
4
|
+
import type { RecordIdInput } from '../db/record-id'
|
|
5
|
+
import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
6
|
+
import { databaseService } from '../db/service'
|
|
7
|
+
import { TABLES } from '../db/tables'
|
|
8
|
+
import { resolvePlanNodeExecutionVisibility } from '../runtime/execution-plan-visibility'
|
|
9
|
+
import { planDeadlineService } from './plan-deadline.service'
|
|
10
|
+
import { planRunService } from './plan-run.service'
|
|
11
|
+
|
|
12
|
+
const ACTIVE_PLAN_RUN_STATUSES = ['running', 'awaiting-human', 'blocked'] as const
|
|
13
|
+
const ACTIONABLE_NODE_STATUSES = new Set(['ready', 'running'])
|
|
14
|
+
const DEADLINE_TRACKED_NODE_STATUSES = new Set(['ready', 'running', 'awaiting-human'])
|
|
15
|
+
|
|
16
|
+
export interface ActionablePlanAgentNode {
|
|
17
|
+
organizationId: string
|
|
18
|
+
workstreamId: string
|
|
19
|
+
runId: string
|
|
20
|
+
nodeId: string
|
|
21
|
+
agentId: string
|
|
22
|
+
status: 'ready' | 'running'
|
|
23
|
+
visibility: PlanExecutionVisibility
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface ApproachingDeadlineNode {
|
|
27
|
+
organizationId: string
|
|
28
|
+
workstreamId: string
|
|
29
|
+
runId: string
|
|
30
|
+
nodeId: string
|
|
31
|
+
agentId?: string
|
|
32
|
+
visibility?: PlanExecutionVisibility
|
|
33
|
+
dueAt?: string
|
|
34
|
+
status: 'warning' | 'escalated' | 'missed'
|
|
35
|
+
nextTriggerAt?: string | null
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface RecentlyUnblockedNode {
|
|
39
|
+
organizationId: string
|
|
40
|
+
workstreamId: string
|
|
41
|
+
runId: string
|
|
42
|
+
nodeId: string
|
|
43
|
+
agentId: string
|
|
44
|
+
visibility: PlanExecutionVisibility
|
|
45
|
+
unblockedAt: string
|
|
46
|
+
sourceEventType: 'node-unblocked' | 'approval-resolved'
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function isVisibleAgentNode(params: {
|
|
50
|
+
nodeSpec: PlanNodeSpecRecord
|
|
51
|
+
spec: PlanSpecRecord
|
|
52
|
+
}): { agentId: string; visibility: PlanExecutionVisibility } | null {
|
|
53
|
+
if (params.nodeSpec.owner.executorType !== 'agent') {
|
|
54
|
+
return null
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const visibility = resolvePlanNodeExecutionVisibility(params.spec, params.nodeSpec)
|
|
58
|
+
const isVisible = visibility === 'visible'
|
|
59
|
+
if (!isVisible) {
|
|
60
|
+
return null
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return { agentId: params.nodeSpec.owner.ref, visibility }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
class PlanAgentQueryService {
|
|
67
|
+
async getActionableNodesForAgent(params: {
|
|
68
|
+
agentId?: string
|
|
69
|
+
organizationId?: RecordIdInput
|
|
70
|
+
}): Promise<ActionablePlanAgentNode[]> {
|
|
71
|
+
const runs = await this.listActiveRuns(params.organizationId)
|
|
72
|
+
const actionable: ActionablePlanAgentNode[] = []
|
|
73
|
+
|
|
74
|
+
for (const run of runs) {
|
|
75
|
+
const spec = await planRunService.getPlanSpecById(run.planSpecId)
|
|
76
|
+
const currentNodeId = run.currentNodeId
|
|
77
|
+
if (!currentNodeId) {
|
|
78
|
+
continue
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const [nodeSpec, nodeRun] = await Promise.all([
|
|
82
|
+
planRunService.getNodeSpecByNodeId(spec.id, currentNodeId),
|
|
83
|
+
planRunService.getNodeRunByNodeId(run.id, currentNodeId),
|
|
84
|
+
])
|
|
85
|
+
if (!ACTIONABLE_NODE_STATUSES.has(nodeRun.status)) {
|
|
86
|
+
continue
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const visibleTarget = isVisibleAgentNode({ nodeSpec, spec })
|
|
90
|
+
if (!visibleTarget) {
|
|
91
|
+
continue
|
|
92
|
+
}
|
|
93
|
+
if (params.agentId && params.agentId !== visibleTarget.agentId) {
|
|
94
|
+
continue
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
actionable.push({
|
|
98
|
+
organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
|
|
99
|
+
workstreamId: recordIdToString(run.workstreamId, TABLES.WORKSTREAM),
|
|
100
|
+
runId: recordIdToString(run.id, TABLES.PLAN_RUN),
|
|
101
|
+
nodeId: nodeSpec.nodeId,
|
|
102
|
+
agentId: visibleTarget.agentId,
|
|
103
|
+
status: nodeRun.status as 'ready' | 'running',
|
|
104
|
+
visibility: visibleTarget.visibility,
|
|
105
|
+
})
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return actionable
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async getApproachingDeadlines(params?: {
|
|
112
|
+
organizationId?: RecordIdInput
|
|
113
|
+
withinMinutes?: number
|
|
114
|
+
}): Promise<ApproachingDeadlineNode[]> {
|
|
115
|
+
const now = new Date()
|
|
116
|
+
const maxWindowMs = (params?.withinMinutes ?? 60) * 60_000
|
|
117
|
+
const runs = await this.listActiveRuns(params?.organizationId)
|
|
118
|
+
const matches: ApproachingDeadlineNode[] = []
|
|
119
|
+
|
|
120
|
+
for (const run of runs) {
|
|
121
|
+
const spec = await planRunService.getPlanSpecById(run.planSpecId)
|
|
122
|
+
const [nodeSpecs, nodeRuns] = await Promise.all([
|
|
123
|
+
planRunService.listNodeSpecs(spec.id),
|
|
124
|
+
planRunService.listNodeRuns(run.id),
|
|
125
|
+
])
|
|
126
|
+
const nodeRunsById = new Map(nodeRuns.map((nodeRun) => [nodeRun.nodeId, nodeRun]))
|
|
127
|
+
|
|
128
|
+
for (const nodeSpec of nodeSpecs) {
|
|
129
|
+
const nodeRun = nodeRunsById.get(nodeSpec.nodeId)
|
|
130
|
+
if (!nodeRun || !nodeSpec.deadline || !DEADLINE_TRACKED_NODE_STATUSES.has(nodeRun.status)) {
|
|
131
|
+
continue
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const evaluation = planDeadlineService.evaluateDeadline({
|
|
135
|
+
deadline: nodeSpec.deadline,
|
|
136
|
+
nodeStartedAt: new Date(nodeRun.startedAt ?? nodeRun.createdAt),
|
|
137
|
+
now,
|
|
138
|
+
})
|
|
139
|
+
if (evaluation.status === 'ok') {
|
|
140
|
+
continue
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const nextTriggerTime = evaluation.nextTriggerAt?.getTime()
|
|
144
|
+
if (nextTriggerTime && nextTriggerTime - now.getTime() > maxWindowMs && evaluation.status !== 'missed') {
|
|
145
|
+
continue
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const visibleTarget = isVisibleAgentNode({ nodeSpec, spec })
|
|
149
|
+
matches.push({
|
|
150
|
+
organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
|
|
151
|
+
workstreamId: recordIdToString(run.workstreamId, TABLES.WORKSTREAM),
|
|
152
|
+
runId: recordIdToString(run.id, TABLES.PLAN_RUN),
|
|
153
|
+
nodeId: nodeSpec.nodeId,
|
|
154
|
+
...(visibleTarget ? { agentId: visibleTarget.agentId, visibility: visibleTarget.visibility } : {}),
|
|
155
|
+
...(nodeSpec.deadline.dueAt ? { dueAt: nodeSpec.deadline.dueAt } : {}),
|
|
156
|
+
status: evaluation.status,
|
|
157
|
+
nextTriggerAt: evaluation.nextTriggerAt?.toISOString() ?? null,
|
|
158
|
+
})
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return matches
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async getRecentlyUnblockedNodes(params?: {
|
|
166
|
+
organizationId?: RecordIdInput
|
|
167
|
+
sinceMinutes?: number
|
|
168
|
+
agentId?: string
|
|
169
|
+
}): Promise<RecentlyUnblockedNode[]> {
|
|
170
|
+
const since = new Date(Date.now() - (params?.sinceMinutes ?? 30) * 60_000)
|
|
171
|
+
const runs = await this.listActiveRuns(params?.organizationId)
|
|
172
|
+
const matches: RecentlyUnblockedNode[] = []
|
|
173
|
+
|
|
174
|
+
for (const run of runs) {
|
|
175
|
+
const spec = await planRunService.getPlanSpecById(run.planSpecId)
|
|
176
|
+
const [nodeSpecs, events] = await Promise.all([
|
|
177
|
+
planRunService.listNodeSpecs(spec.id),
|
|
178
|
+
planRunService.listEvents(run.id, 200),
|
|
179
|
+
])
|
|
180
|
+
const nodeSpecsById = new Map(nodeSpecs.map((nodeSpec) => [nodeSpec.nodeId, nodeSpec]))
|
|
181
|
+
|
|
182
|
+
for (const event of events) {
|
|
183
|
+
if (
|
|
184
|
+
(event.eventType !== 'node-unblocked' && event.eventType !== 'approval-resolved') ||
|
|
185
|
+
!event.nodeId ||
|
|
186
|
+
new Date(event.createdAt).getTime() < since.getTime()
|
|
187
|
+
) {
|
|
188
|
+
continue
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const currentNodeId = run.currentNodeId ?? event.nodeId
|
|
192
|
+
const nodeSpec = nodeSpecsById.get(currentNodeId)
|
|
193
|
+
if (!nodeSpec) {
|
|
194
|
+
continue
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const visibleTarget = isVisibleAgentNode({ nodeSpec, spec })
|
|
198
|
+
if (!visibleTarget) {
|
|
199
|
+
continue
|
|
200
|
+
}
|
|
201
|
+
if (params?.agentId && params.agentId !== visibleTarget.agentId) {
|
|
202
|
+
continue
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
matches.push({
|
|
206
|
+
organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
|
|
207
|
+
workstreamId: recordIdToString(run.workstreamId, TABLES.WORKSTREAM),
|
|
208
|
+
runId: recordIdToString(run.id, TABLES.PLAN_RUN),
|
|
209
|
+
nodeId: currentNodeId,
|
|
210
|
+
agentId: visibleTarget.agentId,
|
|
211
|
+
visibility: visibleTarget.visibility,
|
|
212
|
+
unblockedAt: new Date(event.createdAt).toISOString(),
|
|
213
|
+
sourceEventType: event.eventType,
|
|
214
|
+
})
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return matches
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
private async listActiveRuns(organizationId?: RecordIdInput): Promise<PlanRunRecord[]> {
|
|
222
|
+
const bindings = {
|
|
223
|
+
statuses: [...ACTIVE_PLAN_RUN_STATUSES],
|
|
224
|
+
...(organizationId ? { organizationId: ensureRecordId(organizationId, TABLES.ORGANIZATION) } : {}),
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const whereOrganization = organizationId ? ' AND organizationId = $organizationId' : ''
|
|
228
|
+
return databaseService.queryMany(
|
|
229
|
+
{
|
|
230
|
+
query: `SELECT * FROM ${TABLES.PLAN_RUN} WHERE status INSIDE $statuses${whereOrganization} ORDER BY updatedAt DESC`,
|
|
231
|
+
bindings,
|
|
232
|
+
},
|
|
233
|
+
PlanRunSchema,
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
export const planAgentQueryService = new PlanAgentQueryService()
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { PlanDraft } from '@lota-sdk/shared'
|
|
2
2
|
|
|
3
|
+
import { isExecutableConditionExpression } from './plan-helpers'
|
|
4
|
+
|
|
3
5
|
function buildImplicitLinearEdges(draft: PlanDraft) {
|
|
4
6
|
if (draft.edges.length > 0 || draft.nodes.length <= 1) {
|
|
5
7
|
return draft.edges
|
|
@@ -46,7 +48,15 @@ class PlanBuilderService {
|
|
|
46
48
|
webPolicy: node.contextPolicy.webPolicy,
|
|
47
49
|
},
|
|
48
50
|
})),
|
|
49
|
-
edges: draft.edges.map((edge) =>
|
|
51
|
+
edges: draft.edges.map((edge) => {
|
|
52
|
+
const normalizedWhen = edge.when?.trim()
|
|
53
|
+
const { when: _when, ...edgeWithoutWhen } = edge
|
|
54
|
+
return {
|
|
55
|
+
...edgeWithoutWhen,
|
|
56
|
+
...(normalizedWhen && isExecutableConditionExpression(normalizedWhen) ? { when: normalizedWhen } : {}),
|
|
57
|
+
map: { ...edge.map },
|
|
58
|
+
}
|
|
59
|
+
}),
|
|
50
60
|
schemas: structuredClone(draft.schemas),
|
|
51
61
|
entryNodeIds: [...(draft.entryNodeIds ?? [])],
|
|
52
62
|
}
|
|
@@ -72,8 +72,10 @@ class PlanCompilerService {
|
|
|
72
72
|
attachmentPolicy: node.contextPolicy.attachmentPolicy,
|
|
73
73
|
webPolicy: node.contextPolicy.webPolicy,
|
|
74
74
|
},
|
|
75
|
+
executionVisibility: node.executionVisibility,
|
|
75
76
|
...(node.schedule ? { schedule: node.schedule } : {}),
|
|
76
77
|
...(node.deadline ? { deadline: node.deadline } : {}),
|
|
78
|
+
...(node.escalation ? { escalation: node.escalation } : {}),
|
|
77
79
|
...(node.monitoringConfig ? { monitoringConfig: node.monitoringConfig } : {}),
|
|
78
80
|
...(node.delayAfterPredecessorMs ? { delayAfterPredecessorMs: node.delayAfterPredecessorMs } : {}),
|
|
79
81
|
...(node.deliberationConfig ? { deliberationConfig: node.deliberationConfig } : {}),
|
|
@@ -2,17 +2,20 @@ import type {
|
|
|
2
2
|
DeadlineAction,
|
|
3
3
|
DeadlineReminder,
|
|
4
4
|
DeadlineSpec,
|
|
5
|
+
PlanEventType,
|
|
5
6
|
PlanNodeRunRecord,
|
|
6
7
|
PlanNodeSpecRecord,
|
|
7
8
|
PlanRunRecord,
|
|
8
9
|
} from '@lota-sdk/shared'
|
|
9
|
-
import { PlanNodeRunSchema, PlanNodeSpecRecordSchema, PlanRunSchema } from '@lota-sdk/shared'
|
|
10
|
+
import { PlanEventSchema, PlanNodeRunSchema, PlanNodeSpecRecordSchema, PlanRunSchema } from '@lota-sdk/shared'
|
|
11
|
+
import { RecordId } from 'surrealdb'
|
|
10
12
|
|
|
11
13
|
import type { RecordIdInput } from '../db/record-id'
|
|
12
14
|
import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
13
15
|
import { databaseService } from '../db/service'
|
|
14
16
|
import { TABLES } from '../db/tables'
|
|
15
|
-
import {
|
|
17
|
+
import { planEventDeliveryService } from './plan-event-delivery.service'
|
|
18
|
+
import { planRunService } from './plan-run.service'
|
|
16
19
|
|
|
17
20
|
export type DeadlineEvaluationStatus = 'ok' | 'warning' | 'escalated' | 'missed'
|
|
18
21
|
|
|
@@ -89,45 +92,53 @@ class PlanDeadlineService {
|
|
|
89
92
|
runCache.set(runIdStr, run)
|
|
90
93
|
}
|
|
91
94
|
|
|
92
|
-
const organizationId = recordIdToString(run.organizationId, TABLES.ORGANIZATION)
|
|
93
|
-
const workstreamId = recordIdToString(run.workstreamId, TABLES.WORKSTREAM)
|
|
94
|
-
const notificationService = getNotificationService()
|
|
95
95
|
const dedupeKeyBase = `plan-deadline:${runIdStr}:${entry.nodeRun.nodeId}`
|
|
96
96
|
|
|
97
97
|
if (entry.evaluation.status === 'warning') {
|
|
98
|
-
await
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
title: 'Deadline approaching',
|
|
105
|
-
body:
|
|
98
|
+
await this.emitDeadlineEvent({
|
|
99
|
+
run,
|
|
100
|
+
nodeRun: entry.nodeRun,
|
|
101
|
+
eventType: 'deadline-warning',
|
|
102
|
+
emittedBy: 'plan-deadline-checker',
|
|
103
|
+
message:
|
|
106
104
|
entry.evaluation.activeReminder?.message ?? `Node "${entry.nodeRun.nodeId}" is approaching its deadline.`,
|
|
107
|
-
|
|
105
|
+
detail: {
|
|
106
|
+
title: 'Deadline approaching',
|
|
107
|
+
reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
|
|
108
|
+
dedupeKey: `${dedupeKeyBase}:warning:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
|
|
109
|
+
},
|
|
108
110
|
})
|
|
109
111
|
} else if (entry.evaluation.status === 'escalated') {
|
|
110
|
-
await
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
title: 'Deadline escalation',
|
|
117
|
-
body:
|
|
112
|
+
await this.emitDeadlineEvent({
|
|
113
|
+
run,
|
|
114
|
+
nodeRun: entry.nodeRun,
|
|
115
|
+
eventType: 'escalation-triggered',
|
|
116
|
+
emittedBy: 'plan-deadline-checker',
|
|
117
|
+
message:
|
|
118
118
|
entry.evaluation.activeReminder?.message ?? `Node "${entry.nodeRun.nodeId}" deadline requires escalation.`,
|
|
119
|
-
|
|
119
|
+
detail: {
|
|
120
|
+
title: 'Deadline escalation',
|
|
121
|
+
reminderBeforeMs: entry.evaluation.activeReminder?.beforeMs ?? null,
|
|
122
|
+
dedupeKey: `${dedupeKeyBase}:escalated:${entry.evaluation.activeReminder?.beforeMs ?? 'default'}`,
|
|
123
|
+
},
|
|
120
124
|
})
|
|
121
125
|
} else {
|
|
122
126
|
await this.applyDeadlineMissAction({
|
|
123
127
|
runId: entry.nodeRun.runId,
|
|
124
128
|
nodeId: entry.nodeRun.nodeId,
|
|
125
|
-
workstreamId,
|
|
126
|
-
organizationId,
|
|
129
|
+
workstreamId: recordIdToString(run.workstreamId, TABLES.WORKSTREAM),
|
|
130
|
+
organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
|
|
127
131
|
action: deadline.missAction,
|
|
128
132
|
emittedBy: 'plan-deadline-checker',
|
|
129
133
|
})
|
|
130
134
|
}
|
|
135
|
+
|
|
136
|
+
await this.maybeEmitEscalationPolicyEvent({
|
|
137
|
+
run,
|
|
138
|
+
nodeRun: entry.nodeRun,
|
|
139
|
+
nodeSpec: entry.nodeSpec,
|
|
140
|
+
now: currentTime,
|
|
141
|
+
})
|
|
131
142
|
}
|
|
132
143
|
|
|
133
144
|
const nextTriggerAt =
|
|
@@ -222,37 +233,46 @@ class PlanDeadlineService {
|
|
|
222
233
|
action: DeadlineAction
|
|
223
234
|
emittedBy: string
|
|
224
235
|
}): Promise<void> {
|
|
225
|
-
const notificationService = getNotificationService()
|
|
226
236
|
const runIdStr = recordIdToString(params.runId, TABLES.PLAN_RUN)
|
|
237
|
+
const run = await planRunService.getRunById(params.runId)
|
|
238
|
+
const nodeRun = await planRunService.getNodeRunByNodeId(params.runId, params.nodeId)
|
|
227
239
|
|
|
228
240
|
switch (params.action) {
|
|
229
241
|
case 'notify':
|
|
230
|
-
await
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
title: 'Deadline missed',
|
|
237
|
-
body: `Node "${params.nodeId}" has missed its deadline.`,
|
|
238
|
-
dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:notify`,
|
|
242
|
+
await this.emitDeadlineEvent({
|
|
243
|
+
run,
|
|
244
|
+
nodeRun,
|
|
245
|
+
eventType: 'deadline-missed',
|
|
246
|
+
emittedBy: params.emittedBy,
|
|
247
|
+
message: `Node "${params.nodeId}" has missed its deadline.`,
|
|
248
|
+
detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:notify` },
|
|
239
249
|
})
|
|
240
250
|
break
|
|
241
251
|
|
|
242
252
|
case 'escalate':
|
|
243
|
-
await
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
253
|
+
await this.emitDeadlineEvent({
|
|
254
|
+
run,
|
|
255
|
+
nodeRun,
|
|
256
|
+
eventType: 'escalation-triggered',
|
|
257
|
+
emittedBy: params.emittedBy,
|
|
258
|
+
message: `Node "${params.nodeId}" has missed its deadline and requires escalation.`,
|
|
259
|
+
detail: {
|
|
260
|
+
title: 'Deadline escalation',
|
|
261
|
+
dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:escalate`,
|
|
262
|
+
missedDeadline: true,
|
|
263
|
+
},
|
|
252
264
|
})
|
|
253
265
|
break
|
|
254
266
|
|
|
255
267
|
case 'block': {
|
|
268
|
+
await this.emitDeadlineEvent({
|
|
269
|
+
run,
|
|
270
|
+
nodeRun,
|
|
271
|
+
eventType: 'deadline-missed',
|
|
272
|
+
emittedBy: params.emittedBy,
|
|
273
|
+
message: `Node "${params.nodeId}" has missed its deadline.`,
|
|
274
|
+
detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:block` },
|
|
275
|
+
})
|
|
256
276
|
const { planExecutorService } = await import('./plan-executor.service')
|
|
257
277
|
await planExecutorService.blockNodeOnDispatchFailure({
|
|
258
278
|
workstreamId: params.workstreamId,
|
|
@@ -266,6 +286,14 @@ class PlanDeadlineService {
|
|
|
266
286
|
}
|
|
267
287
|
|
|
268
288
|
case 'fail': {
|
|
289
|
+
await this.emitDeadlineEvent({
|
|
290
|
+
run,
|
|
291
|
+
nodeRun,
|
|
292
|
+
eventType: 'deadline-missed',
|
|
293
|
+
emittedBy: params.emittedBy,
|
|
294
|
+
message: `Node "${params.nodeId}" has missed its deadline.`,
|
|
295
|
+
detail: { title: 'Deadline missed', dedupeKey: `plan-deadline:${runIdStr}:${params.nodeId}:missed:fail` },
|
|
296
|
+
})
|
|
269
297
|
const { planExecutorService } = await import('./plan-executor.service')
|
|
270
298
|
await planExecutorService.blockNodeOnDispatchFailure({
|
|
271
299
|
workstreamId: params.workstreamId,
|
|
@@ -280,6 +308,120 @@ class PlanDeadlineService {
|
|
|
280
308
|
}
|
|
281
309
|
}
|
|
282
310
|
|
|
311
|
+
private async maybeEmitEscalationPolicyEvent(params: {
|
|
312
|
+
run: PlanRunRecord
|
|
313
|
+
nodeRun: PlanNodeRunRecord
|
|
314
|
+
nodeSpec: PlanNodeSpecRecord
|
|
315
|
+
now: Date
|
|
316
|
+
}): Promise<void> {
|
|
317
|
+
const escalation = params.nodeSpec.escalation
|
|
318
|
+
if (!escalation) return
|
|
319
|
+
|
|
320
|
+
const startedAt = new Date(params.nodeRun.startedAt ?? params.nodeRun.createdAt)
|
|
321
|
+
const runIdStr = recordIdToString(params.run.id, TABLES.PLAN_RUN)
|
|
322
|
+
const baseKey = `plan-escalation:${runIdStr}:${params.nodeRun.nodeId}`
|
|
323
|
+
|
|
324
|
+
if (escalation.autoEscalateAfterMinutes) {
|
|
325
|
+
const thresholdMs = escalation.autoEscalateAfterMinutes * 60_000
|
|
326
|
+
if (params.now.getTime() - startedAt.getTime() >= thresholdMs) {
|
|
327
|
+
await this.emitDeadlineEvent({
|
|
328
|
+
run: params.run,
|
|
329
|
+
nodeRun: params.nodeRun,
|
|
330
|
+
eventType: 'escalation-triggered',
|
|
331
|
+
emittedBy: 'plan-deadline-checker',
|
|
332
|
+
message: `Node "${params.nodeRun.nodeId}" exceeded its auto-escalation threshold.`,
|
|
333
|
+
detail: {
|
|
334
|
+
title: 'Execution escalation',
|
|
335
|
+
dedupeKey: `${baseKey}:auto:${escalation.autoEscalateAfterMinutes}`,
|
|
336
|
+
autoEscalateAfterMinutes: escalation.autoEscalateAfterMinutes,
|
|
337
|
+
...(escalation.escalateToAgent ? { escalateToAgent: escalation.escalateToAgent } : {}),
|
|
338
|
+
...(escalation.escalateToUser ? { escalateToUser: escalation.escalateToUser } : {}),
|
|
339
|
+
},
|
|
340
|
+
})
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
if (escalation.deadlineThresholdMinutes && params.nodeSpec.deadline) {
|
|
345
|
+
const dueAt = this.resolveDeadlineTime(params.nodeSpec.deadline, startedAt)
|
|
346
|
+
if (!dueAt) return
|
|
347
|
+
|
|
348
|
+
const thresholdMs = escalation.deadlineThresholdMinutes * 60_000
|
|
349
|
+
if (dueAt.getTime() - params.now.getTime() <= thresholdMs && dueAt.getTime() > params.now.getTime()) {
|
|
350
|
+
await this.emitDeadlineEvent({
|
|
351
|
+
run: params.run,
|
|
352
|
+
nodeRun: params.nodeRun,
|
|
353
|
+
eventType: 'escalation-triggered',
|
|
354
|
+
emittedBy: 'plan-deadline-checker',
|
|
355
|
+
message: `Node "${params.nodeRun.nodeId}" is inside its escalation threshold.`,
|
|
356
|
+
detail: {
|
|
357
|
+
title: 'Deadline escalation threshold',
|
|
358
|
+
dedupeKey: `${baseKey}:threshold:${escalation.deadlineThresholdMinutes}`,
|
|
359
|
+
deadlineThresholdMinutes: escalation.deadlineThresholdMinutes,
|
|
360
|
+
...(escalation.escalateToAgent ? { escalateToAgent: escalation.escalateToAgent } : {}),
|
|
361
|
+
...(escalation.escalateToUser ? { escalateToUser: escalation.escalateToUser } : {}),
|
|
362
|
+
},
|
|
363
|
+
})
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
private async emitDeadlineEvent(params: {
|
|
369
|
+
run: PlanRunRecord
|
|
370
|
+
nodeRun: PlanNodeRunRecord
|
|
371
|
+
eventType: Extract<PlanEventType, 'deadline-warning' | 'deadline-missed' | 'escalation-triggered'>
|
|
372
|
+
emittedBy: string
|
|
373
|
+
message: string
|
|
374
|
+
detail: Record<string, unknown>
|
|
375
|
+
}): Promise<void> {
|
|
376
|
+
const dedupeKey =
|
|
377
|
+
typeof params.detail.dedupeKey === 'string' && params.detail.dedupeKey.trim().length > 0
|
|
378
|
+
? params.detail.dedupeKey.trim()
|
|
379
|
+
: null
|
|
380
|
+
|
|
381
|
+
if (dedupeKey) {
|
|
382
|
+
const existing = await databaseService.queryMany(
|
|
383
|
+
{
|
|
384
|
+
query: `SELECT * FROM ${TABLES.PLAN_EVENT}
|
|
385
|
+
WHERE runId = $runId
|
|
386
|
+
AND nodeId = $nodeId
|
|
387
|
+
AND eventType = $eventType
|
|
388
|
+
AND detail.dedupeKey = $dedupeKey
|
|
389
|
+
LIMIT 1`,
|
|
390
|
+
bindings: {
|
|
391
|
+
runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
|
|
392
|
+
nodeId: params.nodeRun.nodeId,
|
|
393
|
+
eventType: params.eventType,
|
|
394
|
+
dedupeKey,
|
|
395
|
+
},
|
|
396
|
+
},
|
|
397
|
+
PlanEventSchema,
|
|
398
|
+
)
|
|
399
|
+
if (existing.length > 0) {
|
|
400
|
+
return
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const spec = await planRunService.getPlanSpecById(params.run.planSpecId)
|
|
405
|
+
const event = PlanEventSchema.parse(
|
|
406
|
+
await databaseService.create(
|
|
407
|
+
TABLES.PLAN_EVENT,
|
|
408
|
+
{
|
|
409
|
+
id: new RecordId(TABLES.PLAN_EVENT, Bun.randomUUIDv7()),
|
|
410
|
+
planSpecId: ensureRecordId(spec.id, TABLES.PLAN_SPEC),
|
|
411
|
+
runId: ensureRecordId(params.run.id, TABLES.PLAN_RUN),
|
|
412
|
+
nodeId: params.nodeRun.nodeId,
|
|
413
|
+
eventType: params.eventType,
|
|
414
|
+
message: params.message,
|
|
415
|
+
emittedBy: params.emittedBy,
|
|
416
|
+
detail: params.detail,
|
|
417
|
+
},
|
|
418
|
+
PlanEventSchema,
|
|
419
|
+
),
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
await planEventDeliveryService.dispatchEvent(event)
|
|
423
|
+
}
|
|
424
|
+
|
|
283
425
|
private async enqueueDeadlineCheck(scheduledFor: Date): Promise<void> {
|
|
284
426
|
const { enqueueDeadlineCheck } = await import('../queues/plan-scheduler.queue')
|
|
285
427
|
await enqueueDeadlineCheck(scheduledFor)
|