@telora/factory 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit.d.ts +69 -0
- package/dist/audit.d.ts.map +1 -0
- package/dist/audit.js +376 -0
- package/dist/audit.js.map +1 -0
- package/dist/builder-completion.d.ts +35 -0
- package/dist/builder-completion.d.ts.map +1 -0
- package/dist/builder-completion.js +375 -0
- package/dist/builder-completion.js.map +1 -0
- package/dist/builder-spawner.d.ts +40 -0
- package/dist/builder-spawner.d.ts.map +1 -0
- package/dist/builder-spawner.js +493 -0
- package/dist/builder-spawner.js.map +1 -0
- package/dist/completion-gate.d.ts +52 -0
- package/dist/completion-gate.d.ts.map +1 -0
- package/dist/completion-gate.js +336 -0
- package/dist/completion-gate.js.map +1 -0
- package/dist/completion-report.d.ts +36 -0
- package/dist/completion-report.d.ts.map +1 -0
- package/dist/completion-report.js +348 -0
- package/dist/completion-report.js.map +1 -0
- package/dist/completion.d.ts +58 -0
- package/dist/completion.d.ts.map +1 -0
- package/dist/completion.js +287 -0
- package/dist/completion.js.map +1 -0
- package/dist/config.d.ts +16 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +57 -0
- package/dist/config.js.map +1 -0
- package/dist/context-manager.d.ts +152 -0
- package/dist/context-manager.d.ts.map +1 -0
- package/dist/context-manager.js +421 -0
- package/dist/context-manager.js.map +1 -0
- package/dist/crash-detection.d.ts +70 -0
- package/dist/crash-detection.d.ts.map +1 -0
- package/dist/crash-detection.js +123 -0
- package/dist/crash-detection.js.map +1 -0
- package/dist/crash-recovery.d.ts +83 -0
- package/dist/crash-recovery.d.ts.map +1 -0
- package/dist/crash-recovery.js +522 -0
- package/dist/crash-recovery.js.map +1 -0
- package/dist/crash-resolution.d.ts +34 -0
- package/dist/crash-resolution.d.ts.map +1 -0
- package/dist/crash-resolution.js +382 -0
- package/dist/crash-resolution.js.map +1 -0
- package/dist/escalation.d.ts +150 -0
- package/dist/escalation.d.ts.map +1 -0
- package/dist/escalation.js +352 -0
- package/dist/escalation.js.map +1 -0
- package/dist/execution-target.d.ts +31 -0
- package/dist/execution-target.d.ts.map +1 -0
- package/dist/execution-target.js +71 -0
- package/dist/execution-target.js.map +1 -0
- package/dist/execution-unit-init.d.ts +28 -0
- package/dist/execution-unit-init.d.ts.map +1 -0
- package/dist/execution-unit-init.js +115 -0
- package/dist/execution-unit-init.js.map +1 -0
- package/dist/execution.d.ts +17 -0
- package/dist/execution.d.ts.map +1 -0
- package/dist/execution.js +20 -0
- package/dist/execution.js.map +1 -0
- package/dist/factory-engine.d.ts +100 -0
- package/dist/factory-engine.d.ts.map +1 -0
- package/dist/factory-engine.js +243 -0
- package/dist/factory-engine.js.map +1 -0
- package/dist/gap-detection.d.ts +43 -0
- package/dist/gap-detection.d.ts.map +1 -0
- package/dist/gap-detection.js +149 -0
- package/dist/gap-detection.js.map +1 -0
- package/dist/gate-context.d.ts +23 -0
- package/dist/gate-context.d.ts.map +1 -0
- package/dist/gate-context.js +63 -0
- package/dist/gate-context.js.map +1 -0
- package/dist/gate-engine.d.ts +55 -0
- package/dist/gate-engine.d.ts.map +1 -0
- package/dist/gate-engine.js +191 -0
- package/dist/gate-engine.js.map +1 -0
- package/dist/gates/adversarial.d.ts +59 -0
- package/dist/gates/adversarial.d.ts.map +1 -0
- package/dist/gates/adversarial.js +426 -0
- package/dist/gates/adversarial.js.map +1 -0
- package/dist/gates/adversary-spawner.d.ts +35 -0
- package/dist/gates/adversary-spawner.d.ts.map +1 -0
- package/dist/gates/adversary-spawner.js +286 -0
- package/dist/gates/adversary-spawner.js.map +1 -0
- package/dist/gates/adversary-test-dir.d.ts +41 -0
- package/dist/gates/adversary-test-dir.d.ts.map +1 -0
- package/dist/gates/adversary-test-dir.js +150 -0
- package/dist/gates/adversary-test-dir.js.map +1 -0
- package/dist/gates/behavioral-parser.d.ts +32 -0
- package/dist/gates/behavioral-parser.d.ts.map +1 -0
- package/dist/gates/behavioral-parser.js +190 -0
- package/dist/gates/behavioral-parser.js.map +1 -0
- package/dist/gates/behavioral-runner.d.ts +36 -0
- package/dist/gates/behavioral-runner.d.ts.map +1 -0
- package/dist/gates/behavioral-runner.js +306 -0
- package/dist/gates/behavioral-runner.js.map +1 -0
- package/dist/gates/behavioral.d.ts +37 -0
- package/dist/gates/behavioral.d.ts.map +1 -0
- package/dist/gates/behavioral.js +485 -0
- package/dist/gates/behavioral.js.map +1 -0
- package/dist/gates/deterministic.d.ts +24 -0
- package/dist/gates/deterministic.d.ts.map +1 -0
- package/dist/gates/deterministic.js +186 -0
- package/dist/gates/deterministic.js.map +1 -0
- package/dist/git-factory.d.ts +59 -0
- package/dist/git-factory.d.ts.map +1 -0
- package/dist/git-factory.js +102 -0
- package/dist/git-factory.js.map +1 -0
- package/dist/guard-evaluation.d.ts +48 -0
- package/dist/guard-evaluation.d.ts.map +1 -0
- package/dist/guard-evaluation.js +416 -0
- package/dist/guard-evaluation.js.map +1 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +39 -0
- package/dist/index.js.map +1 -0
- package/dist/instance-completion.d.ts +34 -0
- package/dist/instance-completion.d.ts.map +1 -0
- package/dist/instance-completion.js +366 -0
- package/dist/instance-completion.js.map +1 -0
- package/dist/instance-lifecycle.d.ts +15 -0
- package/dist/instance-lifecycle.d.ts.map +1 -0
- package/dist/instance-lifecycle.js +18 -0
- package/dist/instance-lifecycle.js.map +1 -0
- package/dist/instance-phase-dispatch.d.ts +75 -0
- package/dist/instance-phase-dispatch.d.ts.map +1 -0
- package/dist/instance-phase-dispatch.js +674 -0
- package/dist/instance-phase-dispatch.js.map +1 -0
- package/dist/instance-poll-loop.d.ts +43 -0
- package/dist/instance-poll-loop.d.ts.map +1 -0
- package/dist/instance-poll-loop.js +360 -0
- package/dist/instance-poll-loop.js.map +1 -0
- package/dist/instance-state-machine.d.ts +52 -0
- package/dist/instance-state-machine.d.ts.map +1 -0
- package/dist/instance-state-machine.js +235 -0
- package/dist/instance-state-machine.js.map +1 -0
- package/dist/log-manager.d.ts +28 -0
- package/dist/log-manager.d.ts.map +1 -0
- package/dist/log-manager.js +71 -0
- package/dist/log-manager.js.map +1 -0
- package/dist/pipeline-evaluator.d.ts +61 -0
- package/dist/pipeline-evaluator.d.ts.map +1 -0
- package/dist/pipeline-evaluator.js +107 -0
- package/dist/pipeline-evaluator.js.map +1 -0
- package/dist/pipeline-metrics.d.ts +52 -0
- package/dist/pipeline-metrics.d.ts.map +1 -0
- package/dist/pipeline-metrics.js +40 -0
- package/dist/pipeline-metrics.js.map +1 -0
- package/dist/pipeline-traversal.d.ts +43 -0
- package/dist/pipeline-traversal.d.ts.map +1 -0
- package/dist/pipeline-traversal.js +68 -0
- package/dist/pipeline-traversal.js.map +1 -0
- package/dist/plan-parser.d.ts +76 -0
- package/dist/plan-parser.d.ts.map +1 -0
- package/dist/plan-parser.js +223 -0
- package/dist/plan-parser.js.map +1 -0
- package/dist/planning-phase.d.ts +52 -0
- package/dist/planning-phase.d.ts.map +1 -0
- package/dist/planning-phase.js +444 -0
- package/dist/planning-phase.js.map +1 -0
- package/dist/planning-prompt.d.ts +64 -0
- package/dist/planning-prompt.d.ts.map +1 -0
- package/dist/planning-prompt.js +251 -0
- package/dist/planning-prompt.js.map +1 -0
- package/dist/planning.d.ts +16 -0
- package/dist/planning.d.ts.map +1 -0
- package/dist/planning.js +17 -0
- package/dist/planning.js.map +1 -0
- package/dist/process-runner.d.ts +41 -0
- package/dist/process-runner.d.ts.map +1 -0
- package/dist/process-runner.js +81 -0
- package/dist/process-runner.js.map +1 -0
- package/dist/product-config.d.ts +34 -0
- package/dist/product-config.d.ts.map +1 -0
- package/dist/product-config.js +43 -0
- package/dist/product-config.js.map +1 -0
- package/dist/queries/cycle-evaluations.d.ts +23 -0
- package/dist/queries/cycle-evaluations.d.ts.map +1 -0
- package/dist/queries/cycle-evaluations.js +37 -0
- package/dist/queries/cycle-evaluations.js.map +1 -0
- package/dist/queries/escalations.d.ts +30 -0
- package/dist/queries/escalations.d.ts.map +1 -0
- package/dist/queries/escalations.js +42 -0
- package/dist/queries/escalations.js.map +1 -0
- package/dist/queries/execution-units.d.ts +76 -0
- package/dist/queries/execution-units.d.ts.map +1 -0
- package/dist/queries/execution-units.js +109 -0
- package/dist/queries/execution-units.js.map +1 -0
- package/dist/queries/gate-results.d.ts +32 -0
- package/dist/queries/gate-results.d.ts.map +1 -0
- package/dist/queries/gate-results.js +44 -0
- package/dist/queries/gate-results.js.map +1 -0
- package/dist/queries/instances.d.ts +51 -0
- package/dist/queries/instances.d.ts.map +1 -0
- package/dist/queries/instances.js +77 -0
- package/dist/queries/instances.js.map +1 -0
- package/dist/queries/sessions.d.ts +50 -0
- package/dist/queries/sessions.d.ts.map +1 -0
- package/dist/queries/sessions.js +81 -0
- package/dist/queries/sessions.js.map +1 -0
- package/dist/queries/shared.d.ts +38 -0
- package/dist/queries/shared.d.ts.map +1 -0
- package/dist/queries/shared.js +119 -0
- package/dist/queries/shared.js.map +1 -0
- package/dist/queries/specs.d.ts +12 -0
- package/dist/queries/specs.d.ts.map +1 -0
- package/dist/queries/specs.js +21 -0
- package/dist/queries/specs.js.map +1 -0
- package/dist/queries/strategies.d.ts +14 -0
- package/dist/queries/strategies.d.ts.map +1 -0
- package/dist/queries/strategies.js +18 -0
- package/dist/queries/strategies.js.map +1 -0
- package/dist/queries/work-units.d.ts +42 -0
- package/dist/queries/work-units.d.ts.map +1 -0
- package/dist/queries/work-units.js +57 -0
- package/dist/queries/work-units.js.map +1 -0
- package/dist/queries/workflows.d.ts +29 -0
- package/dist/queries/workflows.d.ts.map +1 -0
- package/dist/queries/workflows.js +103 -0
- package/dist/queries/workflows.js.map +1 -0
- package/dist/remediation-units.d.ts +40 -0
- package/dist/remediation-units.d.ts.map +1 -0
- package/dist/remediation-units.js +263 -0
- package/dist/remediation-units.js.map +1 -0
- package/dist/replanning.d.ts +72 -0
- package/dist/replanning.d.ts.map +1 -0
- package/dist/replanning.js +403 -0
- package/dist/replanning.js.map +1 -0
- package/dist/resource-limits.d.ts +62 -0
- package/dist/resource-limits.d.ts.map +1 -0
- package/dist/resource-limits.js +322 -0
- package/dist/resource-limits.js.map +1 -0
- package/dist/scheduler.d.ts +98 -0
- package/dist/scheduler.d.ts.map +1 -0
- package/dist/scheduler.js +203 -0
- package/dist/scheduler.js.map +1 -0
- package/dist/session-adapter.d.ts +89 -0
- package/dist/session-adapter.d.ts.map +1 -0
- package/dist/session-adapter.js +108 -0
- package/dist/session-adapter.js.map +1 -0
- package/dist/sop-generator.d.ts +29 -0
- package/dist/sop-generator.d.ts.map +1 -0
- package/dist/sop-generator.js +235 -0
- package/dist/sop-generator.js.map +1 -0
- package/dist/spec-profiles.d.ts +41 -0
- package/dist/spec-profiles.d.ts.map +1 -0
- package/dist/spec-profiles.js +131 -0
- package/dist/spec-profiles.js.map +1 -0
- package/dist/strategy-design-graph.d.ts +23 -0
- package/dist/strategy-design-graph.d.ts.map +1 -0
- package/dist/strategy-design-graph.js +205 -0
- package/dist/strategy-design-graph.js.map +1 -0
- package/dist/strategy-design-prompt.d.ts +28 -0
- package/dist/strategy-design-prompt.d.ts.map +1 -0
- package/dist/strategy-design-prompt.js +108 -0
- package/dist/strategy-design-prompt.js.map +1 -0
- package/dist/strategy-design-schema.d.ts +767 -0
- package/dist/strategy-design-schema.d.ts.map +1 -0
- package/dist/strategy-design-schema.js +126 -0
- package/dist/strategy-design-schema.js.map +1 -0
- package/dist/strategy-design.d.ts +69 -0
- package/dist/strategy-design.d.ts.map +1 -0
- package/dist/strategy-design.js +411 -0
- package/dist/strategy-design.js.map +1 -0
- package/dist/strategy-gating.d.ts +31 -0
- package/dist/strategy-gating.d.ts.map +1 -0
- package/dist/strategy-gating.js +276 -0
- package/dist/strategy-gating.js.map +1 -0
- package/dist/team-prompt-builder.d.ts +47 -0
- package/dist/team-prompt-builder.d.ts.map +1 -0
- package/dist/team-prompt-builder.js +362 -0
- package/dist/team-prompt-builder.js.map +1 -0
- package/dist/trace-engine.d.ts +40 -0
- package/dist/trace-engine.d.ts.map +1 -0
- package/dist/trace-engine.js +344 -0
- package/dist/trace-engine.js.map +1 -0
- package/dist/types.d.ts +612 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +9 -0
- package/dist/types.js.map +1 -0
- package/dist/unit-session-lifecycle.d.ts +78 -0
- package/dist/unit-session-lifecycle.d.ts.map +1 -0
- package/dist/unit-session-lifecycle.js +141 -0
- package/dist/unit-session-lifecycle.js.map +1 -0
- package/dist/unit-session.d.ts +30 -0
- package/dist/unit-session.d.ts.map +1 -0
- package/dist/unit-session.js +370 -0
- package/dist/unit-session.js.map +1 -0
- package/dist/watchdogs.d.ts +33 -0
- package/dist/watchdogs.d.ts.map +1 -0
- package/dist/watchdogs.js +170 -0
- package/dist/watchdogs.js.map +1 -0
- package/dist/work-unit-scheduler.d.ts +34 -0
- package/dist/work-unit-scheduler.d.ts.map +1 -0
- package/dist/work-unit-scheduler.js +91 -0
- package/dist/work-unit-scheduler.js.map +1 -0
- package/dist/workflow-transition.d.ts +90 -0
- package/dist/workflow-transition.d.ts.map +1 -0
- package/dist/workflow-transition.js +340 -0
- package/dist/workflow-transition.js.map +1 -0
- package/package.json +65 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-entity recovery actions for factory crash recovery.
|
|
3
|
+
*
|
|
4
|
+
* Performs recovery actions for each type of orphaned entity found during
|
|
5
|
+
* crash detection:
|
|
6
|
+
* - WIP-commit uncommitted changes in orphaned worktrees
|
|
7
|
+
* - Reset factory_sessions stuck in 'running' to 'failed'
|
|
8
|
+
* - Reset factory_instances stuck in 'building'/'gating' to 'paused'
|
|
9
|
+
* - Create escalations for paused instances
|
|
10
|
+
* - Reset orphaned execution units to 'idle'
|
|
11
|
+
*
|
|
12
|
+
* Runs after config init and before the poll loop starts, so the factory
|
|
13
|
+
* starts in a clean, known state.
|
|
14
|
+
*/
|
|
15
|
+
import type { FactoryConfig } from './types.js';
|
|
16
|
+
import { type RecoveryResult } from './crash-detection.js';
|
|
17
|
+
/**
|
|
18
|
+
* Run full crash recovery on factory orchestrator startup.
|
|
19
|
+
*
|
|
20
|
+
* Should be called after config init and API client setup, but before
|
|
21
|
+
* the poll loop starts. Safe to call multiple times -- idempotent by
|
|
22
|
+
* design. A second run with no crashed state finds nothing to recover.
|
|
23
|
+
*
|
|
24
|
+
* Recovery steps:
|
|
25
|
+
* 1. Scan for orphaned worktrees and WIP-commit uncommitted changes.
|
|
26
|
+
* 2. Reset factory_sessions stuck in 'running' to 'failed'.
|
|
27
|
+
* 3. Reset factory_instances stuck in 'building'/'gating' to 'paused'.
|
|
28
|
+
* 4. Create escalations for paused instances.
|
|
29
|
+
*
|
|
30
|
+
* @param config - Factory configuration.
|
|
31
|
+
* @returns A RecoveryResult summarizing all actions taken.
|
|
32
|
+
*/
|
|
33
|
+
export declare function recoverFromCrash(config: FactoryConfig): Promise<RecoveryResult>;
|
|
34
|
+
//# sourceMappingURL=crash-resolution.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crash-resolution.d.ts","sourceRoot":"","sources":["../src/crash-resolution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAmB,MAAM,YAAY,CAAC;AAMjE,OAAO,EAIL,KAAK,cAAc,EACpB,MAAM,sBAAsB,CAAC;AAoU9B;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CA4FrF"}
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-entity recovery actions for factory crash recovery.
|
|
3
|
+
*
|
|
4
|
+
* Performs recovery actions for each type of orphaned entity found during
|
|
5
|
+
* crash detection:
|
|
6
|
+
* - WIP-commit uncommitted changes in orphaned worktrees
|
|
7
|
+
* - Reset factory_sessions stuck in 'running' to 'failed'
|
|
8
|
+
* - Reset factory_instances stuck in 'building'/'gating' to 'paused'
|
|
9
|
+
* - Create escalations for paused instances
|
|
10
|
+
* - Reset orphaned execution units to 'idle'
|
|
11
|
+
*
|
|
12
|
+
* Runs after config init and before the poll loop starts, so the factory
|
|
13
|
+
* starts in a clean, known state.
|
|
14
|
+
*/
|
|
15
|
+
import { commitWip } from './git-factory.js';
|
|
16
|
+
import { updateInstanceStatus, getActiveInstances } from './queries/instances.js';
|
|
17
|
+
import { updateFactorySession } from './queries/sessions.js';
|
|
18
|
+
import { createFactoryEscalation } from './queries/escalations.js';
|
|
19
|
+
import { getExecutionUnitsByInstance, updateExecutionUnit } from './queries/execution-units.js';
|
|
20
|
+
import { scanFactoryWorktrees, } from './crash-detection.js';
|
|
21
|
+
// ============================================================================
|
|
22
|
+
// Recovery: orphaned worktrees
|
|
23
|
+
// ============================================================================
|
|
24
|
+
/**
|
|
25
|
+
* WIP-commit any uncommitted changes in orphaned factory worktrees.
|
|
26
|
+
*
|
|
27
|
+
* This preserves in-flight work that was not committed before the crash.
|
|
28
|
+
* The worktrees are left in place -- they will be reused when the instance
|
|
29
|
+
* resumes (after being reset to 'paused').
|
|
30
|
+
*/
|
|
31
|
+
function recoverOrphanedWorktrees(orphans) {
|
|
32
|
+
let wipCommitsCreated = 0;
|
|
33
|
+
const actions = [];
|
|
34
|
+
for (const orphan of orphans) {
|
|
35
|
+
const label = orphan.branchName ?? 'unknown-branch';
|
|
36
|
+
try {
|
|
37
|
+
const sha = commitWip(orphan.worktreePath, `crash-recovery: ${label}`);
|
|
38
|
+
if (sha) {
|
|
39
|
+
wipCommitsCreated++;
|
|
40
|
+
actions.push({
|
|
41
|
+
type: 'worktree_wip_committed',
|
|
42
|
+
entityId: orphan.worktreePath,
|
|
43
|
+
description: `WIP commit ${sha.slice(0, 8)} created for orphaned worktree (branch: ${label})`,
|
|
44
|
+
});
|
|
45
|
+
console.log(`[crash-recovery] WIP commit ${sha.slice(0, 8)} for orphaned worktree: ${orphan.worktreePath}`);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
console.log(`[crash-recovery] Orphaned worktree has no uncommitted changes: ${orphan.worktreePath}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
console.warn(`[crash-recovery] Failed to WIP-commit orphaned worktree ${orphan.worktreePath}: ` +
|
|
53
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return { wipCommitsCreated, actions };
|
|
57
|
+
}
|
|
58
|
+
// ============================================================================
|
|
59
|
+
// Recovery: stale sessions
|
|
60
|
+
// ============================================================================
|
|
61
|
+
/**
|
|
62
|
+
* Reset factory sessions stuck in 'running' or 'starting' status to 'failed'.
|
|
63
|
+
*
|
|
64
|
+
* After a crash, any sessions that were 'running' are no longer actually
|
|
65
|
+
* running -- their processes died with the orchestrator. We query the API
|
|
66
|
+
* for all instances that might have active sessions and reset them.
|
|
67
|
+
*/
|
|
68
|
+
async function recoverStaleSessions(config) {
|
|
69
|
+
const sessionsReset = 0;
|
|
70
|
+
const actions = [];
|
|
71
|
+
const errors = [];
|
|
72
|
+
// We need to find sessions stuck in 'running' status. Since we don't have
|
|
73
|
+
// a direct "get all running sessions" query, we use getActiveSessionsForInstance
|
|
74
|
+
// for each instance that might have stale sessions. We get the instances from
|
|
75
|
+
// the pending query (which returns instances that aren't in terminal states).
|
|
76
|
+
//
|
|
77
|
+
// However, getPendingInstances only returns 'pending' status. For crash
|
|
78
|
+
// recovery we need instances in 'building' and 'gating' states too.
|
|
79
|
+
// Those will be handled via the instance recovery below, which also
|
|
80
|
+
// resets their sessions. But we still want a catch-all for any sessions
|
|
81
|
+
// that might be orphaned.
|
|
82
|
+
//
|
|
83
|
+
// The approach: reset sessions for instances we know about from worktree
|
|
84
|
+
// scanning. The instance recovery step will handle the rest.
|
|
85
|
+
// Note: Session reset for specific instances happens in recoverStuckInstances.
|
|
86
|
+
// This function serves as documentation of the strategy. The actual per-instance
|
|
87
|
+
// session reset is co-located with the instance status reset for atomicity.
|
|
88
|
+
void config; // Config may be used in future for direct session queries
|
|
89
|
+
return { sessionsReset, actions, errors };
|
|
90
|
+
}
|
|
91
|
+
// ============================================================================
|
|
92
|
+
// Recovery: stuck instances
|
|
93
|
+
// ============================================================================
|
|
94
|
+
/**
|
|
95
|
+
* Find and recover factory instances stuck in 'building' or 'gating' status.
|
|
96
|
+
*
|
|
97
|
+
* After an unclean shutdown, these instances had active processing that
|
|
98
|
+
* was interrupted. We:
|
|
99
|
+
* 1. Reset their status to 'paused'.
|
|
100
|
+
* 2. Mark their active sessions as 'failed'.
|
|
101
|
+
* 3. Create an escalation to notify the human operator.
|
|
102
|
+
*/
|
|
103
|
+
async function recoverStuckInstances(config) {
|
|
104
|
+
let instancesPaused = 0;
|
|
105
|
+
let sessionsReset = 0;
|
|
106
|
+
let escalationsCreated = 0;
|
|
107
|
+
const actions = [];
|
|
108
|
+
const errors = [];
|
|
109
|
+
// Query non-terminal instances (designing, planning, building, paused) and
|
|
110
|
+
// filter for those stuck in active processing states after the crash.
|
|
111
|
+
let instances;
|
|
112
|
+
try {
|
|
113
|
+
instances = await getActiveInstances(config.productId);
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
const msg = `Failed to query instances for crash recovery: ${err instanceof Error ? err.message : String(err)}`;
|
|
117
|
+
console.warn(`[crash-recovery] ${msg}`);
|
|
118
|
+
errors.push(msg);
|
|
119
|
+
return { instancesPaused, sessionsReset, escalationsCreated, actions, errors };
|
|
120
|
+
}
|
|
121
|
+
// Filter for instances that are stuck in active processing states
|
|
122
|
+
// Note: 'gating' was removed as an instance-level status -- gates run
|
|
123
|
+
// inline per work unit during the 'building' phase.
|
|
124
|
+
const stuckStatuses = new Set(['building']);
|
|
125
|
+
const stuckInstances = instances.filter((inst) => stuckStatuses.has(inst.status));
|
|
126
|
+
if (stuckInstances.length === 0) {
|
|
127
|
+
console.log('[crash-recovery] No stuck instances found in building status');
|
|
128
|
+
return { instancesPaused, sessionsReset, escalationsCreated, actions, errors };
|
|
129
|
+
}
|
|
130
|
+
console.log(`[crash-recovery] Found ${stuckInstances.length} instance(s) stuck in building/gating status`);
|
|
131
|
+
for (const instance of stuckInstances) {
|
|
132
|
+
// Step 1: Reset active sessions to 'failed'
|
|
133
|
+
try {
|
|
134
|
+
const { getActiveSessionsForInstance } = await import('./queries/sessions.js');
|
|
135
|
+
const activeSessions = await getActiveSessionsForInstance(instance.id);
|
|
136
|
+
for (const session of activeSessions) {
|
|
137
|
+
try {
|
|
138
|
+
await updateFactorySession(session.id, {
|
|
139
|
+
status: 'failed',
|
|
140
|
+
endedAt: new Date().toISOString(),
|
|
141
|
+
});
|
|
142
|
+
sessionsReset++;
|
|
143
|
+
actions.push({
|
|
144
|
+
type: 'session_reset',
|
|
145
|
+
entityId: session.id,
|
|
146
|
+
description: `Session reset from '${session.status}' to 'failed' (instance ${instance.id}, crash recovery)`,
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
const msg = `Failed to reset session ${session.id}: ${err instanceof Error ? err.message : String(err)}`;
|
|
151
|
+
console.warn(`[crash-recovery] ${msg}`);
|
|
152
|
+
errors.push(msg);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
catch (err) {
|
|
157
|
+
const msg = `Failed to query sessions for instance ${instance.id}: ${err instanceof Error ? err.message : String(err)}`;
|
|
158
|
+
console.warn(`[crash-recovery] ${msg}`);
|
|
159
|
+
errors.push(msg);
|
|
160
|
+
}
|
|
161
|
+
// Step 2: Reset instance status to 'paused'
|
|
162
|
+
try {
|
|
163
|
+
await updateInstanceStatus(instance.id, 'paused', {
|
|
164
|
+
escalationReason: 'Unclean shutdown recovery',
|
|
165
|
+
escalationMessage: `Factory orchestrator crashed or was killed while instance was in '${instance.status}' status. ` +
|
|
166
|
+
'Instance has been paused for manual review. Active sessions have been marked as failed. ' +
|
|
167
|
+
'Resume or cancel this instance from the control panel.',
|
|
168
|
+
escalatedAt: new Date().toISOString(),
|
|
169
|
+
});
|
|
170
|
+
instancesPaused++;
|
|
171
|
+
actions.push({
|
|
172
|
+
type: 'instance_paused',
|
|
173
|
+
entityId: instance.id,
|
|
174
|
+
description: `Instance reset from '${instance.status}' to 'paused' (crash recovery)`,
|
|
175
|
+
});
|
|
176
|
+
console.log(`[crash-recovery] Instance ${instance.id} reset: ${instance.status} -> paused`);
|
|
177
|
+
}
|
|
178
|
+
catch (err) {
|
|
179
|
+
const msg = `Failed to pause instance ${instance.id}: ${err instanceof Error ? err.message : String(err)}`;
|
|
180
|
+
console.error(`[crash-recovery] ${msg}`);
|
|
181
|
+
errors.push(msg);
|
|
182
|
+
continue; // Skip escalation if we couldn't even pause the instance
|
|
183
|
+
}
|
|
184
|
+
// Step 3: Create escalation
|
|
185
|
+
try {
|
|
186
|
+
await createFactoryEscalation(instance.id, 'Unclean shutdown: factory orchestrator crashed', `Instance was in '${instance.status}' status when the factory orchestrator shut down unexpectedly. ` +
|
|
187
|
+
`The instance has been paused and its active sessions marked as failed. ` +
|
|
188
|
+
`Please review the instance state and either resume or cancel it.`);
|
|
189
|
+
escalationsCreated++;
|
|
190
|
+
actions.push({
|
|
191
|
+
type: 'escalation_created',
|
|
192
|
+
entityId: instance.id,
|
|
193
|
+
description: `Escalation created for instance paused during crash recovery (was '${instance.status}')`,
|
|
194
|
+
});
|
|
195
|
+
console.log(`[crash-recovery] Created escalation for instance ${instance.id}`);
|
|
196
|
+
}
|
|
197
|
+
catch (err) {
|
|
198
|
+
const msg = `Failed to create escalation for instance ${instance.id}: ${err instanceof Error ? err.message : String(err)}`;
|
|
199
|
+
console.warn(`[crash-recovery] ${msg}`);
|
|
200
|
+
errors.push(msg);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return { instancesPaused, sessionsReset, escalationsCreated, actions, errors };
|
|
204
|
+
}
|
|
205
|
+
// ============================================================================
|
|
206
|
+
// Recovery: orphaned execution units
|
|
207
|
+
// ============================================================================
|
|
208
|
+
/**
|
|
209
|
+
* Reset execution units stuck in 'assigned' or 'running' status to 'idle'.
|
|
210
|
+
*
|
|
211
|
+
* After a crash, these units had active sessions that died with the daemon.
|
|
212
|
+
* We query ALL active (non-terminal) instances and reset their orphaned units.
|
|
213
|
+
* This runs BEFORE instance adoption so the scheduler sees clean state.
|
|
214
|
+
*/
|
|
215
|
+
async function recoverOrphanedExecutionUnits(config) {
|
|
216
|
+
let executionUnitsReset = 0;
|
|
217
|
+
const actions = [];
|
|
218
|
+
const errors = [];
|
|
219
|
+
let instances;
|
|
220
|
+
try {
|
|
221
|
+
instances = await getActiveInstances(config.productId);
|
|
222
|
+
}
|
|
223
|
+
catch (err) {
|
|
224
|
+
const msg = `Failed to query instances for execution unit recovery: ${err instanceof Error ? err.message : String(err)}`;
|
|
225
|
+
console.warn(`[crash-recovery] ${msg}`);
|
|
226
|
+
errors.push(msg);
|
|
227
|
+
return { executionUnitsReset, actions, errors };
|
|
228
|
+
}
|
|
229
|
+
// Only building instances have execution units
|
|
230
|
+
const buildingInstances = instances.filter((inst) => inst.status === 'building');
|
|
231
|
+
if (buildingInstances.length === 0) {
|
|
232
|
+
console.log('[crash-recovery] No building instances found -- skipping execution unit recovery');
|
|
233
|
+
return { executionUnitsReset, actions, errors };
|
|
234
|
+
}
|
|
235
|
+
for (const instance of buildingInstances) {
|
|
236
|
+
let units;
|
|
237
|
+
try {
|
|
238
|
+
units = await getExecutionUnitsByInstance(instance.id);
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
const msg = `Failed to query execution units for instance ${instance.id}: ${err instanceof Error ? err.message : String(err)}`;
|
|
242
|
+
console.warn(`[crash-recovery] ${msg}`);
|
|
243
|
+
errors.push(msg);
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
const orphanedUnits = units.filter((u) => u.status === 'assigned' || u.status === 'running');
|
|
247
|
+
for (const unit of orphanedUnits) {
|
|
248
|
+
try {
|
|
249
|
+
await updateExecutionUnit(unit.id, {
|
|
250
|
+
status: 'idle',
|
|
251
|
+
assignedStrategyId: null,
|
|
252
|
+
claudeSessionId: null,
|
|
253
|
+
assignedAt: null,
|
|
254
|
+
});
|
|
255
|
+
executionUnitsReset++;
|
|
256
|
+
actions.push({
|
|
257
|
+
type: 'execution_unit_reset',
|
|
258
|
+
entityId: unit.id,
|
|
259
|
+
description: `Execution unit reset from '${unit.status}' to 'idle' ` +
|
|
260
|
+
`(instance ${instance.id}, slot ${unit.slotIndex}, crash recovery)`,
|
|
261
|
+
});
|
|
262
|
+
console.log(`[crash-recovery] Reset execution unit ${unit.id} (slot ${unit.slotIndex}) ` +
|
|
263
|
+
`from '${unit.status}' to 'idle' (instance ${instance.id})`);
|
|
264
|
+
}
|
|
265
|
+
catch (err) {
|
|
266
|
+
const msg = `Failed to reset execution unit ${unit.id}: ${err instanceof Error ? err.message : String(err)}`;
|
|
267
|
+
console.warn(`[crash-recovery] ${msg}`);
|
|
268
|
+
errors.push(msg);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (executionUnitsReset > 0) {
|
|
273
|
+
console.log(`[crash-recovery] Reset ${executionUnitsReset} orphaned execution unit(s) to idle`);
|
|
274
|
+
}
|
|
275
|
+
else {
|
|
276
|
+
console.log('[crash-recovery] No orphaned execution units found');
|
|
277
|
+
}
|
|
278
|
+
return { executionUnitsReset, actions, errors };
|
|
279
|
+
}
|
|
280
|
+
// ============================================================================
|
|
281
|
+
// Main recovery orchestrator
|
|
282
|
+
// ============================================================================
|
|
283
|
+
/**
|
|
284
|
+
* Run full crash recovery on factory orchestrator startup.
|
|
285
|
+
*
|
|
286
|
+
* Should be called after config init and API client setup, but before
|
|
287
|
+
* the poll loop starts. Safe to call multiple times -- idempotent by
|
|
288
|
+
* design. A second run with no crashed state finds nothing to recover.
|
|
289
|
+
*
|
|
290
|
+
* Recovery steps:
|
|
291
|
+
* 1. Scan for orphaned worktrees and WIP-commit uncommitted changes.
|
|
292
|
+
* 2. Reset factory_sessions stuck in 'running' to 'failed'.
|
|
293
|
+
* 3. Reset factory_instances stuck in 'building'/'gating' to 'paused'.
|
|
294
|
+
* 4. Create escalations for paused instances.
|
|
295
|
+
*
|
|
296
|
+
* @param config - Factory configuration.
|
|
297
|
+
* @returns A RecoveryResult summarizing all actions taken.
|
|
298
|
+
*/
|
|
299
|
+
export async function recoverFromCrash(config) {
|
|
300
|
+
console.log('[crash-recovery] Starting factory crash recovery scan...');
|
|
301
|
+
const result = {
|
|
302
|
+
orphanedWorktreesFound: 0,
|
|
303
|
+
wipCommitsCreated: 0,
|
|
304
|
+
sessionsReset: 0,
|
|
305
|
+
instancesPaused: 0,
|
|
306
|
+
escalationsCreated: 0,
|
|
307
|
+
executionUnitsReset: 0,
|
|
308
|
+
actions: [],
|
|
309
|
+
errors: [],
|
|
310
|
+
};
|
|
311
|
+
// Step 1: Scan for orphaned worktrees and WIP-commit uncommitted changes
|
|
312
|
+
try {
|
|
313
|
+
const orphans = scanFactoryWorktrees(config);
|
|
314
|
+
result.orphanedWorktreesFound = orphans.length;
|
|
315
|
+
if (orphans.length > 0) {
|
|
316
|
+
console.log(`[crash-recovery] Found ${orphans.length} worktree(s) in factory-worktrees directory`);
|
|
317
|
+
const worktreeResult = recoverOrphanedWorktrees(orphans);
|
|
318
|
+
result.wipCommitsCreated = worktreeResult.wipCommitsCreated;
|
|
319
|
+
result.actions.push(...worktreeResult.actions);
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
console.log('[crash-recovery] No orphaned factory worktrees found');
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
catch (err) {
|
|
326
|
+
const msg = `Failed during worktree scan: ${err instanceof Error ? err.message : String(err)}`;
|
|
327
|
+
console.error(`[crash-recovery] ${msg}`);
|
|
328
|
+
result.errors.push(msg);
|
|
329
|
+
}
|
|
330
|
+
// Step 2: Reset stale sessions (catch-all for sessions not tied to instances)
|
|
331
|
+
try {
|
|
332
|
+
const sessionResult = await recoverStaleSessions(config);
|
|
333
|
+
result.sessionsReset += sessionResult.sessionsReset;
|
|
334
|
+
result.actions.push(...sessionResult.actions);
|
|
335
|
+
result.errors.push(...sessionResult.errors);
|
|
336
|
+
}
|
|
337
|
+
catch (err) {
|
|
338
|
+
const msg = `Failed during stale session recovery: ${err instanceof Error ? err.message : String(err)}`;
|
|
339
|
+
console.error(`[crash-recovery] ${msg}`);
|
|
340
|
+
result.errors.push(msg);
|
|
341
|
+
}
|
|
342
|
+
// Step 3: Reset orphaned execution units (assigned/running -> idle)
|
|
343
|
+
// This runs BEFORE instance adoption so the scheduler sees clean state.
|
|
344
|
+
try {
|
|
345
|
+
const euResult = await recoverOrphanedExecutionUnits(config);
|
|
346
|
+
result.executionUnitsReset = euResult.executionUnitsReset;
|
|
347
|
+
result.actions.push(...euResult.actions);
|
|
348
|
+
result.errors.push(...euResult.errors);
|
|
349
|
+
}
|
|
350
|
+
catch (err) {
|
|
351
|
+
const msg = `Failed during execution unit recovery: ${err instanceof Error ? err.message : String(err)}`;
|
|
352
|
+
console.error(`[crash-recovery] ${msg}`);
|
|
353
|
+
result.errors.push(msg);
|
|
354
|
+
}
|
|
355
|
+
// Step 4: Reset stuck instances (building/gating -> paused) and their sessions
|
|
356
|
+
try {
|
|
357
|
+
const instanceResult = await recoverStuckInstances(config);
|
|
358
|
+
result.instancesPaused = instanceResult.instancesPaused;
|
|
359
|
+
result.sessionsReset += instanceResult.sessionsReset;
|
|
360
|
+
result.escalationsCreated = instanceResult.escalationsCreated;
|
|
361
|
+
result.actions.push(...instanceResult.actions);
|
|
362
|
+
result.errors.push(...instanceResult.errors);
|
|
363
|
+
}
|
|
364
|
+
catch (err) {
|
|
365
|
+
const msg = `Failed during stuck instance recovery: ${err instanceof Error ? err.message : String(err)}`;
|
|
366
|
+
console.error(`[crash-recovery] ${msg}`);
|
|
367
|
+
result.errors.push(msg);
|
|
368
|
+
}
|
|
369
|
+
// Summary
|
|
370
|
+
console.log(`[crash-recovery] Factory recovery complete: ` +
|
|
371
|
+
`${result.orphanedWorktreesFound} worktree(s) found, ` +
|
|
372
|
+
`${result.wipCommitsCreated} WIP commit(s), ` +
|
|
373
|
+
`${result.sessionsReset} session(s) reset, ` +
|
|
374
|
+
`${result.executionUnitsReset} execution unit(s) reset, ` +
|
|
375
|
+
`${result.instancesPaused} instance(s) paused, ` +
|
|
376
|
+
`${result.escalationsCreated} escalation(s) created`);
|
|
377
|
+
if (result.errors.length > 0) {
|
|
378
|
+
console.warn(`[crash-recovery] ${result.errors.length} non-fatal error(s) during recovery`);
|
|
379
|
+
}
|
|
380
|
+
return result;
|
|
381
|
+
}
|
|
382
|
+
//# sourceMappingURL=crash-resolution.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crash-resolution.js","sourceRoot":"","sources":["../src/crash-resolution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACnE,OAAO,EAAE,2BAA2B,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAChG,OAAO,EACL,oBAAoB,GAIrB,MAAM,sBAAsB,CAAC;AAE9B,+EAA+E;AAC/E,+BAA+B;AAC/B,+EAA+E;AAE/E;;;;;;GAMG;AACH,SAAS,wBAAwB,CAC/B,OAA2B;IAE3B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,IAAI,gBAAgB,CAAC;QAEpD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,mBAAmB,KAAK,EAAE,CAAC,CAAC;YACvE,IAAI,GAAG,EAAE,CAAC;gBACR,iBAAiB,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,wBAAwB;oBAC9B,QAAQ,EAAE,MAAM,CAAC,YAAY;oBAC7B,WAAW,EAAE,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,2CAA2C,KAAK,GAAG;iBAC9F,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CACT,+BAA+B,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,2BAA2B,MAAM,CAAC,YAAY,EAAE,CAC/F,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CACT,kEAAkE,MAAM,CAAC,YAAY,EAAE,CACxF,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CACV,2DAA2D,MAAM,CAAC,YAAY,IAAI;gBAClF,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC;AACxC,CAAC;AAED,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E;;;;;;GAMG;AACH,KAAK,UAAU,oBAAoB,CACjC,MAAqB;IAErB,MAAM,aAAa,GAAG,CAAC,CAAC;IACxB,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,0EAA0E;IAC1E,iFAAiF;IACjF,8EAA8E;IAC9E,8EAA8E;IAC9E,EAAE;IACF,wEAAwE;IACxE,oEAAoE;IACpE,oEAAoE;IACpE,wEAAwE;IACxE,0BAA0B;IAC1B,EAAE;IACF,yEAAyE;IACzE,6DAA6D;IAE7D,+EAA+E;IAC/E,iFAAiF;IACjF,4EAA4E;IAE5E,KAAK,MAAM,CAAC,CAAC,0DAA0D;IAEvE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC5C,CAAC;AAED,+EAA+E;AAC/E,4BAA4B;AAC5B,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,KAAK,UAAU,qBAAqB,CAClC,MAAqB;IAQrB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,2EAA2E;IAC3E,sEAAsE;IAEtE,IAAI,SAA4B,CAAC;IACjC,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,iDAAiD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAChH,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACjF,CAAC;IAED,kEAAkE;IAClE,sEAAsE;IACtE,oDAAoD;IACpD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAElF,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACjF,CAAC;IAED,OAAO,CAAC,GAAG,CACT,0BAA0B,cAAc,CAAC,MAAM,8CAA8C,CAC9F,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACtC,4CAA4C;QAC5C,IAAI,CAAC;YACH,MAAM,EAAE,4BAA4B,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YAC/E,MAAM,cAAc,GAAG,MAAM,4BAA4B,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAEvE,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,MAAM,oBAAoB,CAAC,OAAO,CAAC,EAAE,EAAE;wBACrC,MAAM,EAAE,QAAQ;wBAChB,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBAClC,CAAC,CAAC;oBACH,aAAa,EAAE,CAAC;oBAChB,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,eAAe;wBACrB,QAAQ,EAAE,OAAO,CAAC,EAAE;wBACpB,WAAW,EAAE,uBAAuB,OAAO,CAAC,MAAM,2BAA2B,QAAQ,CAAC,EAAE,mBAAmB;qBAC5G,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,GAAG,GAAG,2BAA2B,OAAO,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzG,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;oBACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,yCAAyC,QAAQ,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACxH,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QAED,4CAA4C;QAC5C,IAAI,CAAC;YACH,MAAM,oBAAoB,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE;gBAChD,gBAAgB,EAAE,2BAA2B;gBAC7C,iBAAiB,EACf,qEAAqE,QAAQ,CAAC,MAAM,YAAY;oBAChG,0FAA0F;oBAC1F,wDAAwD;gBAC1D,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACtC,CAAC,CAAC;YACH,eAAe,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,iBAAiB;gBACvB,QAAQ,EAAE,QAAQ,CAAC,EAAE;gBACrB,WAAW,EAAE,wBAAwB,QAAQ,CAAC,MAAM,gCAAgC;aACrF,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CACT,6BAA6B,QAAQ,CAAC,EAAE,WAAW,QAAQ,CAAC,MAAM,YAAY,CAC/E,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,4BAA4B,QAAQ,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3G,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,SAAS,CAAC,yDAAyD;QACrE,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC;YACH,MAAM,uBAAuB,CAC3B,QAAQ,CAAC,EAAE,EACX,gDAAgD,EAChD,oBAAoB,QAAQ,CAAC,MAAM,iEAAiE;gBACpG,yEAAyE;gBACzE,kEAAkE,CACnE,CAAC;YACF,kBAAkB,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,oBAAoB;gBAC1B,QAAQ,EAAE,QAAQ,CAAC,EAAE;gBACrB,WAAW,EAAE,sEAAsE,QAAQ,CAAC,MAAM,IAAI;aACvG,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CACT,oDAAoD,QAAQ,CAAC,EAAE,EAAE,CAClE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,4CAA4C,QAAQ,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3H,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACjF,CAAC;AAED,+EAA+E;AAC/E,qCAAqC;AACrC,+EAA+E;AAE/E;;;;;;GAMG;AACH,KAAK,UAAU,6BAA6B,CAC1C,MAAqB;IAErB,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,SAA4B,CAAC;IACjC,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,0DAA0D,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACzH,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClD,CAAC;IAED,+CAA+C;IAC/C,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;IAEjF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,kFAAkF,CAAC,CAAC;QAChG,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClD,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;QACzC,IAAI,KAAK,CAAC;QACV,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,2BAA2B,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,gDAAgD,QAAQ,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/H,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,SAAS;QACX,CAAC;QAED,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CACzD,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;oBACjC,MAAM,EAAE,MAAM;oBACd,kBAAkB,EAAE,IAAI;oBACxB,eAAe,EAAE,IAAI;oBACrB,UAAU,EAAE,IAAI;iBACjB,CAAC,CAAC;gBACH,mBAAmB,EAAE,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,sBAAsB;oBAC5B,QAAQ,EAAE,IAAI,CAAC,EAAE;oBACjB,WAAW,EAAE,8BAA8B,IAAI,CAAC,MAAM,cAAc;wBAClE,aAAa,QAAQ,CAAC,EAAE,UAAU,IAAI,CAAC,SAAS,mBAAmB;iBACtE,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CACT,yCAAyC,IAAI,CAAC,EAAE,UAAU,IAAI,CAAC,SAAS,IAAI;oBAC5E,SAAS,IAAI,CAAC,MAAM,yBAAyB,QAAQ,CAAC,EAAE,GAAG,CAC5D,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,kCAAkC,IAAI,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7G,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CACT,0BAA0B,mBAAmB,qCAAqC,CACnF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAClD,CAAC;AAED,+EAA+E;AAC/E,6BAA6B;AAC7B,+EAA+E;AAE/E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAqB;IAC1D,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IAExE,MAAM,MAAM,GAAmB;QAC7B,sBAAsB,EAAE,CAAC;QACzB,iBAAiB,EAAE,CAAC;QACpB,aAAa,EAAE,CAAC;QAChB,eAAe,EAAE,CAAC;QAClB,kBAAkB,EAAE,CAAC;QACrB,mBAAmB,EAAE,CAAC;QACtB,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,yEAAyE;IACzE,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,CAAC,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;QAE/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CACT,0BAA0B,OAAO,CAAC,MAAM,6CAA6C,CACtF,CAAC;YACF,MAAM,cAAc,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;YACzD,MAAM,CAAC,iBAAiB,GAAG,cAAc,CAAC,iBAAiB,CAAC;YAC5D,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/F,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,8EAA8E;IAC9E,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,CAAC,aAAa,IAAI,aAAa,CAAC,aAAa,CAAC;QACpD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,yCAAyC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACxG,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,oEAAoE;IACpE,wEAAwE;IACxE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,6BAA6B,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,CAAC,mBAAmB,GAAG,QAAQ,CAAC,mBAAmB,CAAC;QAC1D,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,0CAA0C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACzG,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,+EAA+E;IAC/E,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC3D,MAAM,CAAC,eAAe,GAAG,cAAc,CAAC,eAAe,CAAC;QACxD,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC,aAAa,CAAC;QACrD,MAAM,CAAC,kBAAkB,GAAG,cAAc,CAAC,kBAAkB,CAAC;QAC9D,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,0CAA0C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACzG,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,UAAU;IACV,OAAO,CAAC,GAAG,CACT,8CAA8C;QAC9C,GAAG,MAAM,CAAC,sBAAsB,sBAAsB;QACtD,GAAG,MAAM,CAAC,iBAAiB,kBAAkB;QAC7C,GAAG,MAAM,CAAC,aAAa,qBAAqB;QAC5C,GAAG,MAAM,CAAC,mBAAmB,4BAA4B;QACzD,GAAG,MAAM,CAAC,eAAe,uBAAuB;QAChD,GAAG,MAAM,CAAC,kBAAkB,wBAAwB,CACrD,CAAC;IAEF,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,IAAI,CACV,oBAAoB,MAAM,CAAC,MAAM,CAAC,MAAM,qCAAqC,CAC9E,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Factory instance escalation and stuck detection.
|
|
3
|
+
*
|
|
4
|
+
* Handles escalation when resource limits are hit or the factory gets stuck.
|
|
5
|
+
* Pauses the instance and signals the human via the agent_escalations table.
|
|
6
|
+
*
|
|
7
|
+
* Stuck detection identifies when the same gate is failing N consecutive
|
|
8
|
+
* times with the same error pattern, indicating the builder is unable to
|
|
9
|
+
* make progress.
|
|
10
|
+
*
|
|
11
|
+
* Resume support: when a human resolves the escalation, the instance can
|
|
12
|
+
* transition from 'paused' back to 'building' via resumeInstance().
|
|
13
|
+
*/
|
|
14
|
+
import type { FactoryBlueprint, FactoryInstanceState, FactoryGateResult } from './types.js';
|
|
15
|
+
/** Configuration for escalation behavior. */
|
|
16
|
+
export interface EscalationConfig {
|
|
17
|
+
/**
|
|
18
|
+
* Number of consecutive gate failures with the same error pattern
|
|
19
|
+
* before triggering a stuck escalation. Defaults to 3.
|
|
20
|
+
*/
|
|
21
|
+
stuckThreshold: number;
|
|
22
|
+
}
|
|
23
|
+
/** Result of a stuck detection check for a strategy. */
|
|
24
|
+
export interface StuckDetectionResult {
|
|
25
|
+
/** Whether the strategy appears to be stuck. */
|
|
26
|
+
isStuck: boolean;
|
|
27
|
+
/** The strategy ID. */
|
|
28
|
+
strategyId: string;
|
|
29
|
+
/** The repeated gate type (if stuck). */
|
|
30
|
+
gateType: string | null;
|
|
31
|
+
/** The repeated error pattern (if stuck). */
|
|
32
|
+
errorPattern: string | null;
|
|
33
|
+
/** Number of consecutive failures with the same pattern. */
|
|
34
|
+
consecutiveFailures: number;
|
|
35
|
+
}
|
|
36
|
+
/** Trigger fired when resource limits (token budget, wall clock, etc.) are exceeded. */
|
|
37
|
+
export declare const TRIGGER_RESOURCE_LIMIT_EXCEEDED = "resource_limit_exceeded";
|
|
38
|
+
/** Trigger fired when a builder appears stuck (same gate failure repeated). */
|
|
39
|
+
export declare const TRIGGER_BUILDER_STUCK = "builder_stuck";
|
|
40
|
+
/** Trigger fired when a work unit exhausts all gate retry iterations. */
|
|
41
|
+
export declare const TRIGGER_GATE_FAILURE_EXHAUSTED = "gate_failure_exhausted";
|
|
42
|
+
/** Trigger fired when strategy-level gate evaluation fails. */
|
|
43
|
+
export declare const TRIGGER_STRATEGY_GATE_FAILED = "strategy_gate_failed";
|
|
44
|
+
/**
|
|
45
|
+
* Escalate a factory instance.
|
|
46
|
+
*
|
|
47
|
+
* Sets the instance status to 'paused', records the escalation reason
|
|
48
|
+
* and message on the instance record, and creates an escalation entry
|
|
49
|
+
* in the agent_escalations table to signal the human.
|
|
50
|
+
*
|
|
51
|
+
* @param instanceId - The factory instance UUID.
|
|
52
|
+
* @param reason - Short reason for the escalation (e.g. "token_budget_exceeded").
|
|
53
|
+
* @param message - Detailed human-readable message describing the situation.
|
|
54
|
+
* @param config - Optional escalation configuration overrides.
|
|
55
|
+
* @param sessionId - Optional session ID to link the escalation to.
|
|
56
|
+
*/
|
|
57
|
+
export declare function escalateInstance(instanceId: string, reason: string, message: string, config?: Partial<EscalationConfig>, sessionId?: string | null): Promise<void>;
|
|
58
|
+
/**
|
|
59
|
+
* Escalate a factory instance using its in-memory state.
|
|
60
|
+
*
|
|
61
|
+
* Attempts a workflow-governed transition from the current status to
|
|
62
|
+
* 'paused'. If the workflow allows it (or no workflow exists), updates
|
|
63
|
+
* in-memory state and persists the escalation.
|
|
64
|
+
*/
|
|
65
|
+
export declare function escalateInstanceState(state: FactoryInstanceState, reason: string, message: string, config?: Partial<EscalationConfig>, sessionId?: string | null): Promise<void>;
|
|
66
|
+
/**
|
|
67
|
+
* Resume a paused factory instance.
|
|
68
|
+
*
|
|
69
|
+
* Attempts a workflow-governed transition from 'paused' back to 'building'
|
|
70
|
+
* and clears the escalation fields. This is called when a human resolves
|
|
71
|
+
* the escalation and wants the factory to continue.
|
|
72
|
+
*
|
|
73
|
+
* @param instanceId - The factory instance UUID.
|
|
74
|
+
* @param state - The in-memory instance state (if available).
|
|
75
|
+
*/
|
|
76
|
+
export declare function resumeInstance(instanceId: string, state?: FactoryInstanceState): Promise<void>;
|
|
77
|
+
/**
|
|
78
|
+
* Check whether a given trigger name is configured in the blueprint's
|
|
79
|
+
* escalation triggers list.
|
|
80
|
+
*
|
|
81
|
+
* If the trigger is in the list, the scenario should cause a pause+escalate.
|
|
82
|
+
* If not, the scenario should cause an immediate failure.
|
|
83
|
+
*
|
|
84
|
+
* @param blueprint - The factory blueprint with escalationTriggers config.
|
|
85
|
+
* @param triggerName - The trigger name to check (e.g. TRIGGER_RESOURCE_LIMIT_EXCEEDED).
|
|
86
|
+
* @returns true if the trigger should cause an escalation.
|
|
87
|
+
*/
|
|
88
|
+
export declare function shouldEscalate(blueprint: FactoryBlueprint, triggerName: string): boolean;
|
|
89
|
+
/**
|
|
90
|
+
* Route an escalation-worthy event to either pause+escalate or immediate failure.
|
|
91
|
+
*
|
|
92
|
+
* If the trigger name is in the blueprint's escalationTriggers list, pauses
|
|
93
|
+
* the instance and creates an escalation. Otherwise, calls the onFail callback
|
|
94
|
+
* to fail the instance immediately.
|
|
95
|
+
*
|
|
96
|
+
* @param state - The in-memory instance state.
|
|
97
|
+
* @param triggerName - The trigger category (e.g. TRIGGER_RESOURCE_LIMIT_EXCEEDED).
|
|
98
|
+
* @param reason - Short reason for the escalation.
|
|
99
|
+
* @param message - Detailed human-readable message.
|
|
100
|
+
* @param onFail - Callback invoked when the trigger is not configured for escalation.
|
|
101
|
+
* @param config - Optional escalation configuration overrides.
|
|
102
|
+
*/
|
|
103
|
+
export declare function escalateOrFail(state: FactoryInstanceState, triggerName: string, reason: string, message: string, onFail: () => void | Promise<void>, config?: Partial<EscalationConfig>, sessionId?: string | null): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* Detect whether any strategies in an instance appear to be stuck.
|
|
106
|
+
*
|
|
107
|
+
* A strategy is considered "stuck" when the same gate has failed N
|
|
108
|
+
* consecutive times (where N is the stuckThreshold) with the same
|
|
109
|
+
* error pattern. This indicates the builder is unable to make progress
|
|
110
|
+
* despite retries.
|
|
111
|
+
*
|
|
112
|
+
* Gate results are grouped by strategy_id and analyzed per-strategy.
|
|
113
|
+
*
|
|
114
|
+
* @param instanceState - The in-memory state of the factory instance.
|
|
115
|
+
* @param config - Optional escalation configuration overrides.
|
|
116
|
+
* @returns Array of stuck detection results (one per stuck strategy).
|
|
117
|
+
*/
|
|
118
|
+
export declare function detectStuckStrategies(instanceState: FactoryInstanceState, config?: Partial<EscalationConfig>): Promise<StuckDetectionResult[]>;
|
|
119
|
+
/**
|
|
120
|
+
* Check stuck detection and escalate or fail based on blueprint triggers.
|
|
121
|
+
*
|
|
122
|
+
* Convenience function that combines detectStuckStrategies() with
|
|
123
|
+
* escalateOrFail(). Returns true if the instance was escalated or failed.
|
|
124
|
+
*
|
|
125
|
+
* @param onFail - Callback invoked when TRIGGER_BUILDER_STUCK is not in
|
|
126
|
+
* escalationTriggers and the instance should fail instead.
|
|
127
|
+
*/
|
|
128
|
+
export declare function checkAndEscalateIfStuck(state: FactoryInstanceState, onFail: () => void | Promise<void>, config?: Partial<EscalationConfig>): Promise<boolean>;
|
|
129
|
+
/**
|
|
130
|
+
* Analyze gate results for a single strategy to detect stuck patterns.
|
|
131
|
+
*
|
|
132
|
+
* Looks at the most recent gate results in reverse chronological order.
|
|
133
|
+
* If N consecutive failed results have the same gate type and normalized
|
|
134
|
+
* error output, the strategy is considered stuck.
|
|
135
|
+
*/
|
|
136
|
+
export declare function analyzeGateResultsForStuck(strategyId: string, gateResults: FactoryGateResult[], threshold: number): StuckDetectionResult;
|
|
137
|
+
/**
|
|
138
|
+
* Normalize an error output string for pattern comparison.
|
|
139
|
+
*
|
|
140
|
+
* Applies the following normalizations:
|
|
141
|
+
* - Trims whitespace
|
|
142
|
+
* - Collapses multiple whitespace to single space
|
|
143
|
+
* - Truncates to ERROR_PATTERN_MAX_LENGTH characters
|
|
144
|
+
* - Returns empty string for null/undefined input
|
|
145
|
+
*
|
|
146
|
+
* This allows comparing error outputs that differ only in timestamps,
|
|
147
|
+
* line numbers, or minor formatting variations.
|
|
148
|
+
*/
|
|
149
|
+
export declare function normalizeErrorPattern(output: string | null): string;
|
|
150
|
+
//# sourceMappingURL=escalation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"escalation.d.ts","sourceRoot":"","sources":["../src/escalation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EACV,gBAAgB,EAChB,oBAAoB,EACpB,iBAAiB,EAElB,MAAM,YAAY,CAAC;AAUpB,6CAA6C;AAC7C,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,wDAAwD;AACxD,MAAM,WAAW,oBAAoB;IACnC,gDAAgD;IAChD,OAAO,EAAE,OAAO,CAAC;IACjB,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,4DAA4D;IAC5D,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAMD,wFAAwF;AACxF,eAAO,MAAM,+BAA+B,4BAA4B,CAAC;AAEzE,+EAA+E;AAC/E,eAAO,MAAM,qBAAqB,kBAAkB,CAAC;AAErD,yEAAyE;AACzE,eAAO,MAAM,8BAA8B,2BAA2B,CAAC;AAEvE,+DAA+D;AAC/D,eAAO,MAAM,4BAA4B,yBAAyB,CAAC;AAmBnE;;;;;;;;;;;;GAYG;AACH,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,EAClC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GACxB,OAAO,CAAC,IAAI,CAAC,CAwBf;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,oBAAoB,EAC3B,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,EAClC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GACxB,OAAO,CAAC,IAAI,CAAC,CAwBf;AAMD;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAClC,UAAU,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,oBAAoB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA0Cf;AA6BD;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAExF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,oBAAoB,EAC3B,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAClC,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,EAClC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GACxB,OAAO,CAAC,IAAI,CAAC,CASf;AAMD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,qBAAqB,CACzC,aAAa,EAAE,oBAAoB,EACnC,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GACjC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CA8BjC;AAED;;;;;;;;GAQG;AACH,wBAAsB,uBAAuB,CAC3C,KAAK,EAAE,oBAAoB,EAC3B,MAAM,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAClC,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GACjC,OAAO,CAAC,OAAO,CAAC,CAyBlB;AAMD;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,iBAAiB,EAAE,EAChC,SAAS,EAAE,MAAM,GAChB,oBAAoB,CA+DtB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CASnE"}
|