@lota-sdk/core 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/infrastructure/schema/02_execution_plan.surql +30 -0
- package/package.json +2 -2
- package/src/create-runtime.ts +29 -1
- package/src/db/tables.ts +1 -0
- package/src/queues/autonomous-job.queue.ts +31 -20
- package/src/queues/document-processor.queue.ts +13 -9
- package/src/queues/memory-consolidation.queue.ts +14 -9
- package/src/queues/queue-factory.ts +13 -9
- package/src/runtime/agent-runtime-policy.ts +2 -0
- package/src/runtime/thread-chat-helpers.ts +54 -0
- package/src/services/artifact.service.ts +371 -0
- package/src/services/autonomous-job.service.ts +16 -12
- package/src/services/execution-plan.service.ts +4 -13
- package/src/services/index.ts +1 -0
- package/src/services/ownership-dispatcher.service.ts +4 -0
- package/src/services/plan-artifact.service.ts +7 -1
- package/src/services/plan-coordination.service.ts +23 -18
- package/src/services/plan-executor.service.ts +360 -294
- package/src/services/plan-run.service.ts +4 -0
- package/src/services/plan-template.service.ts +57 -2
- package/src/services/plan-validator.service.ts +1 -1
- package/src/services/queue-job.service.ts +180 -134
- package/src/services/thread-turn-preparation.service.ts +36 -8
- package/src/services/thread-turn.ts +4 -0
- package/src/storage/generated-document-storage.service.ts +8 -0
- package/src/tools/execution-plan.tool.ts +43 -15
- package/src/workers/worker-utils.ts +10 -2
|
@@ -29,8 +29,10 @@ import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
|
29
29
|
import { databaseService } from '../db/service'
|
|
30
30
|
import type { DatabaseTransaction } from '../db/service'
|
|
31
31
|
import { TABLES } from '../db/tables'
|
|
32
|
+
import { generatedDocumentStorageService } from '../storage/generated-document-storage.service'
|
|
32
33
|
import { toDatabaseDateTime } from '../utils/date-time'
|
|
33
34
|
import { isRecord } from '../utils/string'
|
|
35
|
+
import { artifactService } from './artifact.service'
|
|
34
36
|
import { planApprovalService } from './plan-approval.service'
|
|
35
37
|
import { planArtifactService } from './plan-artifact.service'
|
|
36
38
|
import { planCheckpointService } from './plan-checkpoint.service'
|
|
@@ -109,6 +111,22 @@ function buildNodeContext(params: {
|
|
|
109
111
|
}
|
|
110
112
|
}
|
|
111
113
|
|
|
114
|
+
function buildPublishedArtifactContent(params: {
|
|
115
|
+
artifact: PlanNodeResultSubmission['artifacts'][number]
|
|
116
|
+
notes: string
|
|
117
|
+
}): string {
|
|
118
|
+
const { artifact, notes } = params
|
|
119
|
+
if (artifact.content?.trim()) {
|
|
120
|
+
return artifact.content
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (artifact.payload !== undefined) {
|
|
124
|
+
return `# ${artifact.name}\n\n\`\`\`json\n${JSON.stringify(artifact.payload, null, 2)}\n\`\`\``
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return `# ${artifact.name}\n\n${notes.trim()}`
|
|
128
|
+
}
|
|
129
|
+
|
|
112
130
|
function evaluateCondition(expression: string | undefined, context: Record<string, unknown>): boolean {
|
|
113
131
|
if (!expression?.trim()) return true
|
|
114
132
|
const normalized = expression.trim()
|
|
@@ -305,257 +323,351 @@ class PlanExecutorService {
|
|
|
305
323
|
result: params.result,
|
|
306
324
|
})
|
|
307
325
|
const emittedEvents: PlanEventRecord[] = []
|
|
326
|
+
const publishedArtifactStorageKeys: string[] = []
|
|
308
327
|
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
328
|
+
try {
|
|
329
|
+
await databaseService.withTransaction(async (tx) => {
|
|
330
|
+
const attempt = await this.createAttempt({
|
|
331
|
+
tx,
|
|
332
|
+
run,
|
|
333
|
+
nodeRun,
|
|
334
|
+
emittedBy: params.emittedBy,
|
|
335
|
+
result: params.result,
|
|
336
|
+
status: validation.blocking.length > 0 ? 'failed' : 'completed',
|
|
337
|
+
failureClass: validation.failureClass,
|
|
338
|
+
})
|
|
319
339
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
340
|
+
const issues = await this.persistValidationIssues({
|
|
341
|
+
tx,
|
|
342
|
+
run,
|
|
343
|
+
spec,
|
|
344
|
+
attemptId: attempt.id,
|
|
345
|
+
nodeId: params.nodeId,
|
|
346
|
+
issues: [...validation.blocking, ...validation.warnings],
|
|
347
|
+
})
|
|
328
348
|
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
349
|
+
const finalizedAttempt = PlanNodeAttemptSchema.parse(
|
|
350
|
+
await tx
|
|
351
|
+
.update(ensureRecordId(attempt.id, TABLES.PLAN_NODE_ATTEMPT))
|
|
352
|
+
.content({
|
|
353
|
+
runId: ensureRecordId(attempt.runId, TABLES.PLAN_RUN),
|
|
354
|
+
nodeRunId: ensureRecordId(attempt.nodeRunId, TABLES.PLAN_NODE_RUN),
|
|
355
|
+
nodeId: attempt.nodeId,
|
|
356
|
+
emittedBy: attempt.emittedBy,
|
|
357
|
+
status: attempt.status,
|
|
358
|
+
...(attempt.structuredOutput ? { structuredOutput: attempt.structuredOutput } : {}),
|
|
359
|
+
...(attempt.notes ? { notes: attempt.notes } : {}),
|
|
360
|
+
validationIssueIds: issues.map((issue) => ensureRecordId(issue.id, TABLES.PLAN_VALIDATION_ISSUE)),
|
|
361
|
+
...(attempt.failureClass ? { failureClass: attempt.failureClass } : {}),
|
|
362
|
+
})
|
|
363
|
+
.output('after'),
|
|
364
|
+
)
|
|
345
365
|
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
366
|
+
const publishedArtifacts =
|
|
367
|
+
validation.blocking.length > 0
|
|
368
|
+
? params.result.artifacts
|
|
369
|
+
: await Promise.all(
|
|
370
|
+
params.result.artifacts.map(async (artifact) => {
|
|
371
|
+
const deliverable = nodeSpec.deliverables.find((candidate) => candidate.name === artifact.name)
|
|
372
|
+
if (!deliverable?.publishAs) {
|
|
373
|
+
return artifact
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
const content = buildPublishedArtifactContent({ artifact, notes: params.result.notes })
|
|
377
|
+
const published = await artifactService.publishArtifactInTransaction(
|
|
378
|
+
{
|
|
379
|
+
organizationId: recordIdToString(run.organizationId, TABLES.ORGANIZATION),
|
|
380
|
+
authorAgentId: params.emittedBy,
|
|
381
|
+
title: artifact.name,
|
|
382
|
+
artifactKind: deliverable.publishAs.artifactKind,
|
|
383
|
+
templateId: deliverable.publishAs.templateId,
|
|
384
|
+
canonicalKey: deliverable.publishAs.canonicalKey,
|
|
385
|
+
content,
|
|
386
|
+
tags: [],
|
|
387
|
+
...(artifact.description ? { description: artifact.description } : {}),
|
|
388
|
+
sourceThreadId: recordIdToString(run.threadId, TABLES.THREAD),
|
|
389
|
+
sourcePlanRunId: recordIdToString(run.id, TABLES.PLAN_RUN),
|
|
390
|
+
sourcePlanNodeId: params.nodeId,
|
|
391
|
+
deliverableName: artifact.name,
|
|
392
|
+
},
|
|
393
|
+
tx,
|
|
394
|
+
)
|
|
395
|
+
publishedArtifactStorageKeys.push(published.storageKey)
|
|
396
|
+
|
|
397
|
+
return { ...artifact, content, publishedArtifactId: recordIdToString(published.id, TABLES.ARTIFACT) }
|
|
398
|
+
}),
|
|
399
|
+
)
|
|
353
400
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
.
|
|
357
|
-
.
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
latestStructuredOutput: params.result.structuredOutput ?? null,
|
|
362
|
-
latestNotes: params.result.notes,
|
|
363
|
-
}),
|
|
364
|
-
)
|
|
365
|
-
.output('after'),
|
|
366
|
-
)
|
|
401
|
+
const persistedArtifacts = await planArtifactService.persistArtifacts({
|
|
402
|
+
tx,
|
|
403
|
+
runId: run.id,
|
|
404
|
+
attemptId: finalizedAttempt.id,
|
|
405
|
+
nodeId: params.nodeId,
|
|
406
|
+
artifacts: publishedArtifacts,
|
|
407
|
+
})
|
|
367
408
|
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
409
|
+
let nextNodeRun = PlanNodeRunSchema.parse(
|
|
410
|
+
await tx
|
|
411
|
+
.update(ensureRecordId(nodeRun.id, TABLES.PLAN_NODE_RUN))
|
|
412
|
+
.content(
|
|
413
|
+
toNodeRunData(nodeRun, {
|
|
414
|
+
attemptCount: nodeRun.attemptCount + 1,
|
|
415
|
+
latestAttemptId: finalizedAttempt.id,
|
|
416
|
+
latestStructuredOutput: params.result.structuredOutput ?? null,
|
|
417
|
+
latestNotes: params.result.notes,
|
|
418
|
+
}),
|
|
419
|
+
)
|
|
420
|
+
.output('after'),
|
|
421
|
+
)
|
|
373
422
|
|
|
374
|
-
|
|
375
|
-
const
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
423
|
+
const nodeRuns = await planRunService.listNodeRuns(run.id)
|
|
424
|
+
const withUpdatedNodeRuns = nodeRuns.map((candidate) =>
|
|
425
|
+
candidate.nodeId === nextNodeRun.nodeId ? nextNodeRun : candidate,
|
|
426
|
+
)
|
|
427
|
+
const nextArtifacts = [...existingArtifacts, ...persistedArtifacts]
|
|
379
428
|
|
|
380
|
-
if (
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
status: 'ready',
|
|
387
|
-
retryCount: nextNodeRun.retryCount + 1,
|
|
388
|
-
failureClass: validation.failureClass,
|
|
389
|
-
blockedReason: validation.blocking[0]?.message ?? null,
|
|
390
|
-
readyAt: new Date(),
|
|
391
|
-
startedAt: null,
|
|
392
|
-
completedAt: null,
|
|
393
|
-
}),
|
|
394
|
-
)
|
|
395
|
-
.output('after'),
|
|
396
|
-
)
|
|
429
|
+
if (validation.blocking.length > 0) {
|
|
430
|
+
const shouldRetry =
|
|
431
|
+
validation.failureClass &&
|
|
432
|
+
nodeSpec.retryPolicy.maxAttempts > nextNodeRun.retryCount &&
|
|
433
|
+
(nodeSpec.retryPolicy.retryOn.length === 0 ||
|
|
434
|
+
nodeSpec.retryPolicy.retryOn.includes(validation.failureClass))
|
|
397
435
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
attemptId: finalizedAttempt.id,
|
|
416
|
-
eventType: 'node-unblocked',
|
|
417
|
-
fromStatus: nodeRun.status,
|
|
418
|
-
toStatus: nextNodeRun.status,
|
|
419
|
-
message: `Node "${nodeSpec.label}" is ready for another attempt.`,
|
|
420
|
-
detail: { retryCount: nextNodeRun.retryCount },
|
|
421
|
-
emittedBy: params.emittedBy,
|
|
422
|
-
capturedEvents: emittedEvents,
|
|
423
|
-
})
|
|
436
|
+
if (shouldRetry) {
|
|
437
|
+
nextNodeRun = PlanNodeRunSchema.parse(
|
|
438
|
+
await tx
|
|
439
|
+
.update(ensureRecordId(nextNodeRun.id, TABLES.PLAN_NODE_RUN))
|
|
440
|
+
.content(
|
|
441
|
+
toNodeRunData(nextNodeRun, {
|
|
442
|
+
status: 'ready',
|
|
443
|
+
retryCount: nextNodeRun.retryCount + 1,
|
|
444
|
+
failureClass: validation.failureClass,
|
|
445
|
+
blockedReason: validation.blocking[0]?.message ?? null,
|
|
446
|
+
readyAt: new Date(),
|
|
447
|
+
startedAt: null,
|
|
448
|
+
completedAt: null,
|
|
449
|
+
}),
|
|
450
|
+
)
|
|
451
|
+
.output('after'),
|
|
452
|
+
)
|
|
424
453
|
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
454
|
+
await this.emitEvent({
|
|
455
|
+
tx,
|
|
456
|
+
run,
|
|
457
|
+
spec,
|
|
458
|
+
nodeId: nextNodeRun.nodeId,
|
|
459
|
+
attemptId: finalizedAttempt.id,
|
|
460
|
+
eventType: 'validation-reported',
|
|
461
|
+
message: `Validation failed for node "${nodeSpec.label}", scheduling retry.`,
|
|
462
|
+
detail: { issues: validation.blocking.map((issue) => issue.code) },
|
|
463
|
+
emittedBy: params.emittedBy,
|
|
464
|
+
capturedEvents: emittedEvents,
|
|
465
|
+
})
|
|
466
|
+
await this.emitEvent({
|
|
467
|
+
tx,
|
|
468
|
+
run,
|
|
469
|
+
spec,
|
|
470
|
+
nodeId: nextNodeRun.nodeId,
|
|
471
|
+
attemptId: finalizedAttempt.id,
|
|
472
|
+
eventType: 'node-unblocked',
|
|
473
|
+
fromStatus: nodeRun.status,
|
|
474
|
+
toStatus: nextNodeRun.status,
|
|
475
|
+
message: `Node "${nodeSpec.label}" is ready for another attempt.`,
|
|
476
|
+
detail: { retryCount: nextNodeRun.retryCount },
|
|
477
|
+
emittedBy: params.emittedBy,
|
|
478
|
+
capturedEvents: emittedEvents,
|
|
479
|
+
})
|
|
437
480
|
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
481
|
+
const synced = await this.syncRunGraph({
|
|
482
|
+
tx,
|
|
483
|
+
run,
|
|
484
|
+
spec,
|
|
485
|
+
nodeSpecs,
|
|
486
|
+
nodeRuns: withUpdatedNodeRuns.map((candidate) =>
|
|
487
|
+
candidate.nodeId === nextNodeRun.nodeId ? nextNodeRun : candidate,
|
|
488
|
+
),
|
|
489
|
+
artifacts: nextArtifacts,
|
|
490
|
+
emittedBy: params.emittedBy,
|
|
491
|
+
capturedEvents: emittedEvents,
|
|
492
|
+
})
|
|
448
493
|
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
494
|
+
const checkpoint = await this.saveCheckpoint({
|
|
495
|
+
tx,
|
|
496
|
+
run: synced.run,
|
|
497
|
+
spec,
|
|
498
|
+
nodeRuns: synced.nodeRuns,
|
|
499
|
+
artifacts: synced.artifacts,
|
|
500
|
+
sequence: (latestCheckpoint?.sequence ?? 0) + 1,
|
|
501
|
+
reason: 'node-result-retry',
|
|
502
|
+
capturedEvents: emittedEvents,
|
|
503
|
+
})
|
|
452
504
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
await tx
|
|
457
|
-
.update(ensureRecordId(nextNodeRun.id, TABLES.PLAN_NODE_RUN))
|
|
458
|
-
.content(
|
|
459
|
-
toNodeRunData(nextNodeRun, {
|
|
460
|
-
status: 'awaiting-human',
|
|
461
|
-
retryCount: nextNodeRun.retryCount + 1,
|
|
462
|
-
failureClass: validation.failureClass,
|
|
463
|
-
blockedReason: validation.blocking[0]?.message ?? null,
|
|
464
|
-
startedAt: nextNodeRun.startedAt ?? new Date(),
|
|
465
|
-
}),
|
|
466
|
-
)
|
|
467
|
-
.output('after'),
|
|
468
|
-
)
|
|
505
|
+
await this.attachCheckpoint(tx, synced.run, checkpoint)
|
|
506
|
+
return
|
|
507
|
+
}
|
|
469
508
|
|
|
470
|
-
const
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
509
|
+
const failureAction = this.resolveFailureAction(nodeSpec, validation.failureClass)
|
|
510
|
+
if (failureAction === 'human-review') {
|
|
511
|
+
nextNodeRun = PlanNodeRunSchema.parse(
|
|
512
|
+
await tx
|
|
513
|
+
.update(ensureRecordId(nextNodeRun.id, TABLES.PLAN_NODE_RUN))
|
|
514
|
+
.content(
|
|
515
|
+
toNodeRunData(nextNodeRun, {
|
|
516
|
+
status: 'awaiting-human',
|
|
517
|
+
retryCount: nextNodeRun.retryCount + 1,
|
|
518
|
+
failureClass: validation.failureClass,
|
|
519
|
+
blockedReason: validation.blocking[0]?.message ?? null,
|
|
520
|
+
startedAt: nextNodeRun.startedAt ?? new Date(),
|
|
521
|
+
}),
|
|
522
|
+
)
|
|
523
|
+
.output('after'),
|
|
524
|
+
)
|
|
484
525
|
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
526
|
+
const approval = await planApprovalService.createPendingApproval({
|
|
527
|
+
tx,
|
|
528
|
+
runId: run.id,
|
|
529
|
+
nodeRunId: nextNodeRun.id,
|
|
530
|
+
nodeId: nextNodeRun.nodeId,
|
|
531
|
+
requestedBy: params.emittedBy,
|
|
532
|
+
presented: {
|
|
533
|
+
nodeId: nodeSpec.nodeId,
|
|
534
|
+
label: nodeSpec.label,
|
|
535
|
+
objective: nodeSpec.objective,
|
|
536
|
+
instructions: nodeSpec.instructions,
|
|
537
|
+
validationIssues: validation.blocking,
|
|
538
|
+
},
|
|
539
|
+
})
|
|
492
540
|
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
eventType: 'approval-requested',
|
|
501
|
-
fromStatus: run.status,
|
|
502
|
-
toStatus: failedRun.status,
|
|
503
|
-
message: `Node "${nodeSpec.label}" requires human review before continuing.`,
|
|
504
|
-
detail: { issues: validation.blocking.map((issue) => issue.code) },
|
|
505
|
-
emittedBy: params.emittedBy,
|
|
506
|
-
capturedEvents: emittedEvents,
|
|
507
|
-
})
|
|
541
|
+
const failedRun = await this.replaceRun(tx, run, {
|
|
542
|
+
status: 'awaiting-human',
|
|
543
|
+
currentNodeId: nextNodeRun.nodeId,
|
|
544
|
+
waitingNodeId: nextNodeRun.nodeId,
|
|
545
|
+
readyNodeIds: [],
|
|
546
|
+
failureCount: run.failureCount + 1,
|
|
547
|
+
})
|
|
508
548
|
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
549
|
+
await this.emitEvent({
|
|
550
|
+
tx,
|
|
551
|
+
run: failedRun,
|
|
552
|
+
spec,
|
|
553
|
+
nodeId: nextNodeRun.nodeId,
|
|
554
|
+
attemptId: finalizedAttempt.id,
|
|
555
|
+
approvalId: approval.id,
|
|
556
|
+
eventType: 'approval-requested',
|
|
557
|
+
fromStatus: run.status,
|
|
558
|
+
toStatus: failedRun.status,
|
|
559
|
+
message: `Node "${nodeSpec.label}" requires human review before continuing.`,
|
|
560
|
+
detail: { issues: validation.blocking.map((issue) => issue.code) },
|
|
561
|
+
emittedBy: params.emittedBy,
|
|
562
|
+
capturedEvents: emittedEvents,
|
|
563
|
+
})
|
|
521
564
|
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
565
|
+
const checkpoint = await this.saveCheckpoint({
|
|
566
|
+
tx,
|
|
567
|
+
run: failedRun,
|
|
568
|
+
spec,
|
|
569
|
+
nodeRuns: withUpdatedNodeRuns.map((candidate) =>
|
|
570
|
+
candidate.nodeId === nextNodeRun.nodeId ? nextNodeRun : candidate,
|
|
571
|
+
),
|
|
572
|
+
artifacts: nextArtifacts,
|
|
573
|
+
sequence: (latestCheckpoint?.sequence ?? 0) + 1,
|
|
574
|
+
reason: 'node-result-human-review',
|
|
575
|
+
capturedEvents: emittedEvents,
|
|
576
|
+
})
|
|
577
|
+
|
|
578
|
+
await this.attachCheckpoint(tx, failedRun, checkpoint)
|
|
579
|
+
return
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
if (failureAction === 'replan') {
|
|
583
|
+
nextNodeRun = PlanNodeRunSchema.parse(
|
|
584
|
+
await tx
|
|
585
|
+
.update(ensureRecordId(nextNodeRun.id, TABLES.PLAN_NODE_RUN))
|
|
586
|
+
.content(
|
|
587
|
+
toNodeRunData(nextNodeRun, {
|
|
588
|
+
status: 'blocked',
|
|
589
|
+
retryCount: nextNodeRun.retryCount + 1,
|
|
590
|
+
failureClass: validation.failureClass,
|
|
591
|
+
blockedReason: validation.blocking[0]?.message ?? null,
|
|
592
|
+
}),
|
|
593
|
+
)
|
|
594
|
+
.output('after'),
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
const blockedRun = await this.replaceRun(tx, run, {
|
|
598
|
+
status: 'blocked',
|
|
599
|
+
currentNodeId: nextNodeRun.nodeId,
|
|
600
|
+
waitingNodeId: null,
|
|
601
|
+
readyNodeIds: [],
|
|
602
|
+
failureCount: run.failureCount + 1,
|
|
603
|
+
})
|
|
604
|
+
|
|
605
|
+
await this.emitEvent({
|
|
606
|
+
tx,
|
|
607
|
+
run: blockedRun,
|
|
608
|
+
spec,
|
|
609
|
+
nodeId: nextNodeRun.nodeId,
|
|
610
|
+
attemptId: finalizedAttempt.id,
|
|
611
|
+
eventType: 'node-blocked',
|
|
612
|
+
fromStatus: nodeRun.status,
|
|
613
|
+
toStatus: nextNodeRun.status,
|
|
614
|
+
message: `Node "${nodeSpec.label}" failed validation and requires replanning.`,
|
|
615
|
+
detail: { failureClass: validation.failureClass, issues: validation.blocking.map((issue) => issue.code) },
|
|
616
|
+
emittedBy: params.emittedBy,
|
|
617
|
+
capturedEvents: emittedEvents,
|
|
618
|
+
})
|
|
619
|
+
|
|
620
|
+
const checkpoint = await this.saveCheckpoint({
|
|
621
|
+
tx,
|
|
622
|
+
run: blockedRun,
|
|
623
|
+
spec,
|
|
624
|
+
nodeRuns: withUpdatedNodeRuns.map((candidate) =>
|
|
625
|
+
candidate.nodeId === nextNodeRun.nodeId ? nextNodeRun : candidate,
|
|
626
|
+
),
|
|
627
|
+
artifacts: nextArtifacts,
|
|
628
|
+
sequence: (latestCheckpoint?.sequence ?? 0) + 1,
|
|
629
|
+
reason: 'node-result-replan',
|
|
630
|
+
capturedEvents: emittedEvents,
|
|
631
|
+
})
|
|
632
|
+
|
|
633
|
+
await this.attachCheckpoint(tx, blockedRun, checkpoint)
|
|
634
|
+
return
|
|
635
|
+
}
|
|
525
636
|
|
|
526
|
-
if (failureAction === 'replan') {
|
|
527
637
|
nextNodeRun = PlanNodeRunSchema.parse(
|
|
528
638
|
await tx
|
|
529
639
|
.update(ensureRecordId(nextNodeRun.id, TABLES.PLAN_NODE_RUN))
|
|
530
640
|
.content(
|
|
531
641
|
toNodeRunData(nextNodeRun, {
|
|
532
|
-
status: '
|
|
642
|
+
status: 'failed',
|
|
533
643
|
retryCount: nextNodeRun.retryCount + 1,
|
|
534
644
|
failureClass: validation.failureClass,
|
|
535
645
|
blockedReason: validation.blocking[0]?.message ?? null,
|
|
646
|
+
completedAt: new Date(),
|
|
536
647
|
}),
|
|
537
648
|
)
|
|
538
649
|
.output('after'),
|
|
539
650
|
)
|
|
540
651
|
|
|
541
|
-
const
|
|
542
|
-
status: '
|
|
543
|
-
currentNodeId:
|
|
652
|
+
const failedRun = await this.replaceRun(tx, run, {
|
|
653
|
+
status: 'failed',
|
|
654
|
+
currentNodeId: null,
|
|
544
655
|
waitingNodeId: null,
|
|
545
656
|
readyNodeIds: [],
|
|
546
657
|
failureCount: run.failureCount + 1,
|
|
658
|
+
completedAt: new Date(),
|
|
547
659
|
})
|
|
548
660
|
|
|
549
661
|
await this.emitEvent({
|
|
550
662
|
tx,
|
|
551
|
-
run:
|
|
663
|
+
run: failedRun,
|
|
552
664
|
spec,
|
|
553
665
|
nodeId: nextNodeRun.nodeId,
|
|
554
666
|
attemptId: finalizedAttempt.id,
|
|
555
|
-
eventType: 'node-
|
|
667
|
+
eventType: 'node-failed',
|
|
556
668
|
fromStatus: nodeRun.status,
|
|
557
669
|
toStatus: nextNodeRun.status,
|
|
558
|
-
message: `Node "${nodeSpec.label}" failed validation and
|
|
670
|
+
message: `Node "${nodeSpec.label}" failed validation and the run has been aborted.`,
|
|
559
671
|
detail: { failureClass: validation.failureClass, issues: validation.blocking.map((issue) => issue.code) },
|
|
560
672
|
emittedBy: params.emittedBy,
|
|
561
673
|
capturedEvents: emittedEvents,
|
|
@@ -563,18 +675,18 @@ class PlanExecutorService {
|
|
|
563
675
|
|
|
564
676
|
const checkpoint = await this.saveCheckpoint({
|
|
565
677
|
tx,
|
|
566
|
-
run:
|
|
678
|
+
run: failedRun,
|
|
567
679
|
spec,
|
|
568
680
|
nodeRuns: withUpdatedNodeRuns.map((candidate) =>
|
|
569
681
|
candidate.nodeId === nextNodeRun.nodeId ? nextNodeRun : candidate,
|
|
570
682
|
),
|
|
571
683
|
artifacts: nextArtifacts,
|
|
572
684
|
sequence: (latestCheckpoint?.sequence ?? 0) + 1,
|
|
573
|
-
reason: 'node-result-
|
|
685
|
+
reason: 'node-result-failed',
|
|
574
686
|
capturedEvents: emittedEvents,
|
|
575
687
|
})
|
|
576
688
|
|
|
577
|
-
await this.attachCheckpoint(tx,
|
|
689
|
+
await this.attachCheckpoint(tx, failedRun, checkpoint)
|
|
578
690
|
return
|
|
579
691
|
}
|
|
580
692
|
|
|
@@ -583,118 +695,70 @@ class PlanExecutorService {
|
|
|
583
695
|
.update(ensureRecordId(nextNodeRun.id, TABLES.PLAN_NODE_RUN))
|
|
584
696
|
.content(
|
|
585
697
|
toNodeRunData(nextNodeRun, {
|
|
586
|
-
status: '
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
698
|
+
status: validation.warnings.length > 0 ? 'partial' : 'completed',
|
|
699
|
+
latestAttemptId: finalizedAttempt.id,
|
|
700
|
+
latestStructuredOutput: params.result.structuredOutput ?? null,
|
|
701
|
+
latestNotes: params.result.notes,
|
|
702
|
+
blockedReason: null,
|
|
703
|
+
failureClass: null,
|
|
590
704
|
completedAt: new Date(),
|
|
591
705
|
}),
|
|
592
706
|
)
|
|
593
707
|
.output('after'),
|
|
594
708
|
)
|
|
595
709
|
|
|
596
|
-
const failedRun = await this.replaceRun(tx, run, {
|
|
597
|
-
status: 'failed',
|
|
598
|
-
currentNodeId: null,
|
|
599
|
-
waitingNodeId: null,
|
|
600
|
-
readyNodeIds: [],
|
|
601
|
-
failureCount: run.failureCount + 1,
|
|
602
|
-
completedAt: new Date(),
|
|
603
|
-
})
|
|
604
|
-
|
|
605
710
|
await this.emitEvent({
|
|
606
711
|
tx,
|
|
607
|
-
run
|
|
712
|
+
run,
|
|
608
713
|
spec,
|
|
609
714
|
nodeId: nextNodeRun.nodeId,
|
|
610
715
|
attemptId: finalizedAttempt.id,
|
|
611
|
-
eventType: 'node-
|
|
716
|
+
eventType: validation.warnings.length > 0 ? 'node-partial' : 'node-completed',
|
|
612
717
|
fromStatus: nodeRun.status,
|
|
613
718
|
toStatus: nextNodeRun.status,
|
|
614
|
-
message:
|
|
615
|
-
|
|
719
|
+
message:
|
|
720
|
+
validation.warnings.length > 0
|
|
721
|
+
? `Node "${nodeSpec.label}" completed with warnings.`
|
|
722
|
+
: `Node "${nodeSpec.label}" completed successfully.`,
|
|
723
|
+
detail: { warningCount: validation.warnings.length },
|
|
616
724
|
emittedBy: params.emittedBy,
|
|
617
725
|
capturedEvents: emittedEvents,
|
|
618
726
|
})
|
|
619
727
|
|
|
620
|
-
const
|
|
728
|
+
const synced = await this.syncRunGraph({
|
|
621
729
|
tx,
|
|
622
|
-
run
|
|
730
|
+
run,
|
|
623
731
|
spec,
|
|
732
|
+
nodeSpecs,
|
|
624
733
|
nodeRuns: withUpdatedNodeRuns.map((candidate) =>
|
|
625
734
|
candidate.nodeId === nextNodeRun.nodeId ? nextNodeRun : candidate,
|
|
626
735
|
),
|
|
627
736
|
artifacts: nextArtifacts,
|
|
628
|
-
|
|
629
|
-
reason: 'node-result-failed',
|
|
737
|
+
emittedBy: params.emittedBy,
|
|
630
738
|
capturedEvents: emittedEvents,
|
|
631
739
|
})
|
|
632
740
|
|
|
633
|
-
await this.
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
latestAttemptId: finalizedAttempt.id,
|
|
644
|
-
latestStructuredOutput: params.result.structuredOutput ?? null,
|
|
645
|
-
latestNotes: params.result.notes,
|
|
646
|
-
blockedReason: null,
|
|
647
|
-
failureClass: null,
|
|
648
|
-
completedAt: new Date(),
|
|
649
|
-
}),
|
|
650
|
-
)
|
|
651
|
-
.output('after'),
|
|
652
|
-
)
|
|
741
|
+
const checkpoint = await this.saveCheckpoint({
|
|
742
|
+
tx,
|
|
743
|
+
run: synced.run,
|
|
744
|
+
spec,
|
|
745
|
+
nodeRuns: synced.nodeRuns,
|
|
746
|
+
artifacts: synced.artifacts,
|
|
747
|
+
sequence: (latestCheckpoint?.sequence ?? 0) + 1,
|
|
748
|
+
reason: 'node-result-complete',
|
|
749
|
+
capturedEvents: emittedEvents,
|
|
750
|
+
})
|
|
653
751
|
|
|
654
|
-
|
|
655
|
-
tx,
|
|
656
|
-
run,
|
|
657
|
-
spec,
|
|
658
|
-
nodeId: nextNodeRun.nodeId,
|
|
659
|
-
attemptId: finalizedAttempt.id,
|
|
660
|
-
eventType: validation.warnings.length > 0 ? 'node-partial' : 'node-completed',
|
|
661
|
-
fromStatus: nodeRun.status,
|
|
662
|
-
toStatus: nextNodeRun.status,
|
|
663
|
-
message:
|
|
664
|
-
validation.warnings.length > 0
|
|
665
|
-
? `Node "${nodeSpec.label}" completed with warnings.`
|
|
666
|
-
: `Node "${nodeSpec.label}" completed successfully.`,
|
|
667
|
-
detail: { warningCount: validation.warnings.length },
|
|
668
|
-
emittedBy: params.emittedBy,
|
|
669
|
-
capturedEvents: emittedEvents,
|
|
752
|
+
await this.attachCheckpoint(tx, synced.run, checkpoint)
|
|
670
753
|
})
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
spec,
|
|
676
|
-
nodeSpecs,
|
|
677
|
-
nodeRuns: withUpdatedNodeRuns.map((candidate) =>
|
|
678
|
-
candidate.nodeId === nextNodeRun.nodeId ? nextNodeRun : candidate,
|
|
754
|
+
} catch (error) {
|
|
755
|
+
await Promise.allSettled(
|
|
756
|
+
publishedArtifactStorageKeys.map((storageKey) =>
|
|
757
|
+
generatedDocumentStorageService.deleteTextArtifact(storageKey),
|
|
679
758
|
),
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
})
|
|
684
|
-
|
|
685
|
-
const checkpoint = await this.saveCheckpoint({
|
|
686
|
-
tx,
|
|
687
|
-
run: synced.run,
|
|
688
|
-
spec,
|
|
689
|
-
nodeRuns: synced.nodeRuns,
|
|
690
|
-
artifacts: synced.artifacts,
|
|
691
|
-
sequence: (latestCheckpoint?.sequence ?? 0) + 1,
|
|
692
|
-
reason: 'node-result-complete',
|
|
693
|
-
capturedEvents: emittedEvents,
|
|
694
|
-
})
|
|
695
|
-
|
|
696
|
-
await this.attachCheckpoint(tx, synced.run, checkpoint)
|
|
697
|
-
})
|
|
759
|
+
)
|
|
760
|
+
throw error
|
|
761
|
+
}
|
|
698
762
|
|
|
699
763
|
const orgId = recordIdToString(run.organizationId, TABLES.ORGANIZATION)
|
|
700
764
|
const runIdStr = recordIdToString(run.id, TABLES.PLAN_RUN)
|
|
@@ -1287,7 +1351,9 @@ class PlanExecutorService {
|
|
|
1287
1351
|
eventType: 'run-status-changed',
|
|
1288
1352
|
fromStatus: params.run.status,
|
|
1289
1353
|
toStatus: currentRun.status,
|
|
1290
|
-
message: `Run blocked: unresolved cross-plan dependencies (${unresolved
|
|
1354
|
+
message: `Run blocked: unresolved cross-plan dependencies (${unresolved
|
|
1355
|
+
.map((d) => d.sourcePlanSpecId)
|
|
1356
|
+
.join(', ')}).`,
|
|
1291
1357
|
emittedBy: params.emittedBy,
|
|
1292
1358
|
capturedEvents: params.capturedEvents,
|
|
1293
1359
|
})
|