@telora/factory 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit.d.ts +69 -0
- package/dist/audit.d.ts.map +1 -0
- package/dist/audit.js +376 -0
- package/dist/audit.js.map +1 -0
- package/dist/builder-completion.d.ts +35 -0
- package/dist/builder-completion.d.ts.map +1 -0
- package/dist/builder-completion.js +375 -0
- package/dist/builder-completion.js.map +1 -0
- package/dist/builder-spawner.d.ts +40 -0
- package/dist/builder-spawner.d.ts.map +1 -0
- package/dist/builder-spawner.js +493 -0
- package/dist/builder-spawner.js.map +1 -0
- package/dist/completion-gate.d.ts +52 -0
- package/dist/completion-gate.d.ts.map +1 -0
- package/dist/completion-gate.js +336 -0
- package/dist/completion-gate.js.map +1 -0
- package/dist/completion-report.d.ts +36 -0
- package/dist/completion-report.d.ts.map +1 -0
- package/dist/completion-report.js +348 -0
- package/dist/completion-report.js.map +1 -0
- package/dist/completion.d.ts +58 -0
- package/dist/completion.d.ts.map +1 -0
- package/dist/completion.js +287 -0
- package/dist/completion.js.map +1 -0
- package/dist/config.d.ts +16 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +57 -0
- package/dist/config.js.map +1 -0
- package/dist/context-manager.d.ts +152 -0
- package/dist/context-manager.d.ts.map +1 -0
- package/dist/context-manager.js +421 -0
- package/dist/context-manager.js.map +1 -0
- package/dist/crash-detection.d.ts +70 -0
- package/dist/crash-detection.d.ts.map +1 -0
- package/dist/crash-detection.js +123 -0
- package/dist/crash-detection.js.map +1 -0
- package/dist/crash-recovery.d.ts +83 -0
- package/dist/crash-recovery.d.ts.map +1 -0
- package/dist/crash-recovery.js +522 -0
- package/dist/crash-recovery.js.map +1 -0
- package/dist/crash-resolution.d.ts +34 -0
- package/dist/crash-resolution.d.ts.map +1 -0
- package/dist/crash-resolution.js +382 -0
- package/dist/crash-resolution.js.map +1 -0
- package/dist/escalation.d.ts +150 -0
- package/dist/escalation.d.ts.map +1 -0
- package/dist/escalation.js +352 -0
- package/dist/escalation.js.map +1 -0
- package/dist/execution-target.d.ts +31 -0
- package/dist/execution-target.d.ts.map +1 -0
- package/dist/execution-target.js +71 -0
- package/dist/execution-target.js.map +1 -0
- package/dist/execution-unit-init.d.ts +28 -0
- package/dist/execution-unit-init.d.ts.map +1 -0
- package/dist/execution-unit-init.js +115 -0
- package/dist/execution-unit-init.js.map +1 -0
- package/dist/execution.d.ts +17 -0
- package/dist/execution.d.ts.map +1 -0
- package/dist/execution.js +20 -0
- package/dist/execution.js.map +1 -0
- package/dist/factory-engine.d.ts +100 -0
- package/dist/factory-engine.d.ts.map +1 -0
- package/dist/factory-engine.js +243 -0
- package/dist/factory-engine.js.map +1 -0
- package/dist/gap-detection.d.ts +43 -0
- package/dist/gap-detection.d.ts.map +1 -0
- package/dist/gap-detection.js +149 -0
- package/dist/gap-detection.js.map +1 -0
- package/dist/gate-context.d.ts +23 -0
- package/dist/gate-context.d.ts.map +1 -0
- package/dist/gate-context.js +63 -0
- package/dist/gate-context.js.map +1 -0
- package/dist/gate-engine.d.ts +55 -0
- package/dist/gate-engine.d.ts.map +1 -0
- package/dist/gate-engine.js +191 -0
- package/dist/gate-engine.js.map +1 -0
- package/dist/gates/adversarial.d.ts +59 -0
- package/dist/gates/adversarial.d.ts.map +1 -0
- package/dist/gates/adversarial.js +426 -0
- package/dist/gates/adversarial.js.map +1 -0
- package/dist/gates/adversary-spawner.d.ts +35 -0
- package/dist/gates/adversary-spawner.d.ts.map +1 -0
- package/dist/gates/adversary-spawner.js +286 -0
- package/dist/gates/adversary-spawner.js.map +1 -0
- package/dist/gates/adversary-test-dir.d.ts +41 -0
- package/dist/gates/adversary-test-dir.d.ts.map +1 -0
- package/dist/gates/adversary-test-dir.js +150 -0
- package/dist/gates/adversary-test-dir.js.map +1 -0
- package/dist/gates/behavioral-parser.d.ts +32 -0
- package/dist/gates/behavioral-parser.d.ts.map +1 -0
- package/dist/gates/behavioral-parser.js +190 -0
- package/dist/gates/behavioral-parser.js.map +1 -0
- package/dist/gates/behavioral-runner.d.ts +36 -0
- package/dist/gates/behavioral-runner.d.ts.map +1 -0
- package/dist/gates/behavioral-runner.js +306 -0
- package/dist/gates/behavioral-runner.js.map +1 -0
- package/dist/gates/behavioral.d.ts +37 -0
- package/dist/gates/behavioral.d.ts.map +1 -0
- package/dist/gates/behavioral.js +485 -0
- package/dist/gates/behavioral.js.map +1 -0
- package/dist/gates/deterministic.d.ts +24 -0
- package/dist/gates/deterministic.d.ts.map +1 -0
- package/dist/gates/deterministic.js +186 -0
- package/dist/gates/deterministic.js.map +1 -0
- package/dist/git-factory.d.ts +59 -0
- package/dist/git-factory.d.ts.map +1 -0
- package/dist/git-factory.js +102 -0
- package/dist/git-factory.js.map +1 -0
- package/dist/guard-evaluation.d.ts +48 -0
- package/dist/guard-evaluation.d.ts.map +1 -0
- package/dist/guard-evaluation.js +416 -0
- package/dist/guard-evaluation.js.map +1 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +39 -0
- package/dist/index.js.map +1 -0
- package/dist/instance-completion.d.ts +34 -0
- package/dist/instance-completion.d.ts.map +1 -0
- package/dist/instance-completion.js +366 -0
- package/dist/instance-completion.js.map +1 -0
- package/dist/instance-lifecycle.d.ts +15 -0
- package/dist/instance-lifecycle.d.ts.map +1 -0
- package/dist/instance-lifecycle.js +18 -0
- package/dist/instance-lifecycle.js.map +1 -0
- package/dist/instance-phase-dispatch.d.ts +75 -0
- package/dist/instance-phase-dispatch.d.ts.map +1 -0
- package/dist/instance-phase-dispatch.js +674 -0
- package/dist/instance-phase-dispatch.js.map +1 -0
- package/dist/instance-poll-loop.d.ts +43 -0
- package/dist/instance-poll-loop.d.ts.map +1 -0
- package/dist/instance-poll-loop.js +360 -0
- package/dist/instance-poll-loop.js.map +1 -0
- package/dist/instance-state-machine.d.ts +52 -0
- package/dist/instance-state-machine.d.ts.map +1 -0
- package/dist/instance-state-machine.js +235 -0
- package/dist/instance-state-machine.js.map +1 -0
- package/dist/log-manager.d.ts +28 -0
- package/dist/log-manager.d.ts.map +1 -0
- package/dist/log-manager.js +71 -0
- package/dist/log-manager.js.map +1 -0
- package/dist/pipeline-evaluator.d.ts +61 -0
- package/dist/pipeline-evaluator.d.ts.map +1 -0
- package/dist/pipeline-evaluator.js +107 -0
- package/dist/pipeline-evaluator.js.map +1 -0
- package/dist/pipeline-metrics.d.ts +52 -0
- package/dist/pipeline-metrics.d.ts.map +1 -0
- package/dist/pipeline-metrics.js +40 -0
- package/dist/pipeline-metrics.js.map +1 -0
- package/dist/pipeline-traversal.d.ts +43 -0
- package/dist/pipeline-traversal.d.ts.map +1 -0
- package/dist/pipeline-traversal.js +68 -0
- package/dist/pipeline-traversal.js.map +1 -0
- package/dist/plan-parser.d.ts +76 -0
- package/dist/plan-parser.d.ts.map +1 -0
- package/dist/plan-parser.js +223 -0
- package/dist/plan-parser.js.map +1 -0
- package/dist/planning-phase.d.ts +52 -0
- package/dist/planning-phase.d.ts.map +1 -0
- package/dist/planning-phase.js +444 -0
- package/dist/planning-phase.js.map +1 -0
- package/dist/planning-prompt.d.ts +64 -0
- package/dist/planning-prompt.d.ts.map +1 -0
- package/dist/planning-prompt.js +251 -0
- package/dist/planning-prompt.js.map +1 -0
- package/dist/planning.d.ts +16 -0
- package/dist/planning.d.ts.map +1 -0
- package/dist/planning.js +17 -0
- package/dist/planning.js.map +1 -0
- package/dist/process-runner.d.ts +41 -0
- package/dist/process-runner.d.ts.map +1 -0
- package/dist/process-runner.js +81 -0
- package/dist/process-runner.js.map +1 -0
- package/dist/product-config.d.ts +34 -0
- package/dist/product-config.d.ts.map +1 -0
- package/dist/product-config.js +43 -0
- package/dist/product-config.js.map +1 -0
- package/dist/queries/cycle-evaluations.d.ts +23 -0
- package/dist/queries/cycle-evaluations.d.ts.map +1 -0
- package/dist/queries/cycle-evaluations.js +37 -0
- package/dist/queries/cycle-evaluations.js.map +1 -0
- package/dist/queries/escalations.d.ts +30 -0
- package/dist/queries/escalations.d.ts.map +1 -0
- package/dist/queries/escalations.js +42 -0
- package/dist/queries/escalations.js.map +1 -0
- package/dist/queries/execution-units.d.ts +76 -0
- package/dist/queries/execution-units.d.ts.map +1 -0
- package/dist/queries/execution-units.js +109 -0
- package/dist/queries/execution-units.js.map +1 -0
- package/dist/queries/gate-results.d.ts +32 -0
- package/dist/queries/gate-results.d.ts.map +1 -0
- package/dist/queries/gate-results.js +44 -0
- package/dist/queries/gate-results.js.map +1 -0
- package/dist/queries/instances.d.ts +51 -0
- package/dist/queries/instances.d.ts.map +1 -0
- package/dist/queries/instances.js +77 -0
- package/dist/queries/instances.js.map +1 -0
- package/dist/queries/sessions.d.ts +50 -0
- package/dist/queries/sessions.d.ts.map +1 -0
- package/dist/queries/sessions.js +81 -0
- package/dist/queries/sessions.js.map +1 -0
- package/dist/queries/shared.d.ts +38 -0
- package/dist/queries/shared.d.ts.map +1 -0
- package/dist/queries/shared.js +119 -0
- package/dist/queries/shared.js.map +1 -0
- package/dist/queries/specs.d.ts +12 -0
- package/dist/queries/specs.d.ts.map +1 -0
- package/dist/queries/specs.js +21 -0
- package/dist/queries/specs.js.map +1 -0
- package/dist/queries/strategies.d.ts +14 -0
- package/dist/queries/strategies.d.ts.map +1 -0
- package/dist/queries/strategies.js +18 -0
- package/dist/queries/strategies.js.map +1 -0
- package/dist/queries/work-units.d.ts +42 -0
- package/dist/queries/work-units.d.ts.map +1 -0
- package/dist/queries/work-units.js +57 -0
- package/dist/queries/work-units.js.map +1 -0
- package/dist/queries/workflows.d.ts +29 -0
- package/dist/queries/workflows.d.ts.map +1 -0
- package/dist/queries/workflows.js +103 -0
- package/dist/queries/workflows.js.map +1 -0
- package/dist/remediation-units.d.ts +40 -0
- package/dist/remediation-units.d.ts.map +1 -0
- package/dist/remediation-units.js +263 -0
- package/dist/remediation-units.js.map +1 -0
- package/dist/replanning.d.ts +72 -0
- package/dist/replanning.d.ts.map +1 -0
- package/dist/replanning.js +403 -0
- package/dist/replanning.js.map +1 -0
- package/dist/resource-limits.d.ts +62 -0
- package/dist/resource-limits.d.ts.map +1 -0
- package/dist/resource-limits.js +322 -0
- package/dist/resource-limits.js.map +1 -0
- package/dist/scheduler.d.ts +98 -0
- package/dist/scheduler.d.ts.map +1 -0
- package/dist/scheduler.js +203 -0
- package/dist/scheduler.js.map +1 -0
- package/dist/session-adapter.d.ts +89 -0
- package/dist/session-adapter.d.ts.map +1 -0
- package/dist/session-adapter.js +108 -0
- package/dist/session-adapter.js.map +1 -0
- package/dist/sop-generator.d.ts +29 -0
- package/dist/sop-generator.d.ts.map +1 -0
- package/dist/sop-generator.js +235 -0
- package/dist/sop-generator.js.map +1 -0
- package/dist/spec-profiles.d.ts +41 -0
- package/dist/spec-profiles.d.ts.map +1 -0
- package/dist/spec-profiles.js +131 -0
- package/dist/spec-profiles.js.map +1 -0
- package/dist/strategy-design-graph.d.ts +23 -0
- package/dist/strategy-design-graph.d.ts.map +1 -0
- package/dist/strategy-design-graph.js +205 -0
- package/dist/strategy-design-graph.js.map +1 -0
- package/dist/strategy-design-prompt.d.ts +28 -0
- package/dist/strategy-design-prompt.d.ts.map +1 -0
- package/dist/strategy-design-prompt.js +108 -0
- package/dist/strategy-design-prompt.js.map +1 -0
- package/dist/strategy-design-schema.d.ts +767 -0
- package/dist/strategy-design-schema.d.ts.map +1 -0
- package/dist/strategy-design-schema.js +126 -0
- package/dist/strategy-design-schema.js.map +1 -0
- package/dist/strategy-design.d.ts +69 -0
- package/dist/strategy-design.d.ts.map +1 -0
- package/dist/strategy-design.js +411 -0
- package/dist/strategy-design.js.map +1 -0
- package/dist/strategy-gating.d.ts +31 -0
- package/dist/strategy-gating.d.ts.map +1 -0
- package/dist/strategy-gating.js +276 -0
- package/dist/strategy-gating.js.map +1 -0
- package/dist/team-prompt-builder.d.ts +47 -0
- package/dist/team-prompt-builder.d.ts.map +1 -0
- package/dist/team-prompt-builder.js +362 -0
- package/dist/team-prompt-builder.js.map +1 -0
- package/dist/trace-engine.d.ts +40 -0
- package/dist/trace-engine.d.ts.map +1 -0
- package/dist/trace-engine.js +344 -0
- package/dist/trace-engine.js.map +1 -0
- package/dist/types.d.ts +612 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +9 -0
- package/dist/types.js.map +1 -0
- package/dist/unit-session-lifecycle.d.ts +78 -0
- package/dist/unit-session-lifecycle.d.ts.map +1 -0
- package/dist/unit-session-lifecycle.js +141 -0
- package/dist/unit-session-lifecycle.js.map +1 -0
- package/dist/unit-session.d.ts +30 -0
- package/dist/unit-session.d.ts.map +1 -0
- package/dist/unit-session.js +370 -0
- package/dist/unit-session.js.map +1 -0
- package/dist/watchdogs.d.ts +33 -0
- package/dist/watchdogs.d.ts.map +1 -0
- package/dist/watchdogs.js +170 -0
- package/dist/watchdogs.js.map +1 -0
- package/dist/work-unit-scheduler.d.ts +34 -0
- package/dist/work-unit-scheduler.d.ts.map +1 -0
- package/dist/work-unit-scheduler.js +91 -0
- package/dist/work-unit-scheduler.js.map +1 -0
- package/dist/workflow-transition.d.ts +90 -0
- package/dist/workflow-transition.d.ts.map +1 -0
- package/dist/workflow-transition.js +340 -0
- package/dist/workflow-transition.js.map +1 -0
- package/package.json +65 -0
|
@@ -0,0 +1,674 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Factory instance phase dispatch -- orchestrates designing, planning,
|
|
3
|
+
* building, and review phases.
|
|
4
|
+
*
|
|
5
|
+
* Contains the phase handler functions that drive instances through
|
|
6
|
+
* their lifecycle stages. Imports from instance-state-machine (leaf)
|
|
7
|
+
* and instance-completion, but NOT from instance-poll-loop.
|
|
8
|
+
*/
|
|
9
|
+
import { updateInstanceStatus } from './queries/instances.js';
|
|
10
|
+
import { getInstanceWorkUnits, updateWorkUnitStatus, createWorkUnits } from './queries/work-units.js';
|
|
11
|
+
import { deleteExecutionUnitsByInstance } from './queries/execution-units.js';
|
|
12
|
+
import { createCycleEvaluation } from './queries/cycle-evaluations.js';
|
|
13
|
+
import { createFactoryEscalation } from './queries/escalations.js';
|
|
14
|
+
import { updateStrategyExecutionStatus } from './queries/strategies.js';
|
|
15
|
+
import { createFactorySession, updateFactorySession } from './queries/sessions.js';
|
|
16
|
+
import { runPlanningPhase, generateBranchName } from './planning.js';
|
|
17
|
+
import { runStrategyDesignPhase } from './strategy-design.js';
|
|
18
|
+
import { createFactoryWorktree } from './git-factory.js';
|
|
19
|
+
import { configForFactoryProduct, findFactoryProduct } from './product-config.js';
|
|
20
|
+
import { runSchedulerCycle, createSchedulerDeps } from './scheduler.js';
|
|
21
|
+
import { spawnUnitSession, terminateAllUnits, terminateUnitSession } from './unit-session.js';
|
|
22
|
+
import { checkResourceLimits } from './resource-limits.js';
|
|
23
|
+
import { checkAndEscalateIfStuck, escalateInstanceState, } from './escalation.js';
|
|
24
|
+
import { runCompletionGate } from './completion-gate.js';
|
|
25
|
+
import { advancePipeline, getNodeLabel } from './pipeline-traversal.js';
|
|
26
|
+
import { collectMetrics } from './pipeline-metrics.js';
|
|
27
|
+
import { randomUUID } from 'node:crypto';
|
|
28
|
+
import { transitionInstanceStatus, handleInstanceFailure, } from './instance-state-machine.js';
|
|
29
|
+
import { runCompletionSequence, detectStrategyCompletion, checkInstanceCompletion, } from './instance-completion.js';
|
|
30
|
+
// ============================================================================
|
|
31
|
+
// Module-level config reference
|
|
32
|
+
// ============================================================================
|
|
33
|
+
/** Module-level config, set by setPhaseDispatchConfig. */
|
|
34
|
+
let config = null;
|
|
35
|
+
/** Optional resource governor for global concurrency control across engines. */
|
|
36
|
+
let governor = null;
|
|
37
|
+
/**
|
|
38
|
+
* Set the module-level config for phase dispatch functions.
|
|
39
|
+
* Called by the poll loop when starting.
|
|
40
|
+
*/
|
|
41
|
+
export function setPhaseDispatchConfig(factoryConfig) {
|
|
42
|
+
config = factoryConfig;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Set the module-level governor for phase dispatch functions.
|
|
46
|
+
* Called by the poll loop when the governor is set.
|
|
47
|
+
*/
|
|
48
|
+
export function setPhaseDispatchGovernor(gov) {
|
|
49
|
+
governor = gov;
|
|
50
|
+
}
|
|
51
|
+
// ============================================================================
|
|
52
|
+
// Helpers
|
|
53
|
+
// ============================================================================
|
|
54
|
+
export function getCurrentNodeTemplateType(state) {
|
|
55
|
+
if (!state.pipelineGraph || !state.currentPipelineNodeId)
|
|
56
|
+
return null;
|
|
57
|
+
const node = state.pipelineGraph.nodes.find((n) => n.id === state.currentPipelineNodeId);
|
|
58
|
+
return node?.templateType ?? null;
|
|
59
|
+
}
|
|
60
|
+
// ============================================================================
|
|
61
|
+
// Phase handlers
|
|
62
|
+
// ============================================================================
|
|
63
|
+
/**
|
|
64
|
+
* Run the designing phase for an instance.
|
|
65
|
+
*
|
|
66
|
+
* Creates a git worktree, spawns Claude Code to decompose the specification
|
|
67
|
+
* into bounded strategies, validates and persists them, then transitions
|
|
68
|
+
* to "planning".
|
|
69
|
+
*/
|
|
70
|
+
export async function runDesigning(state) {
|
|
71
|
+
if (!config) {
|
|
72
|
+
handleInstanceFailure(state, 'Config not initialized -- cannot run designing');
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
// Review nodes skip design/planning -- they run the completion gate directly
|
|
76
|
+
// and use gap results to drive the pipeline advance decision.
|
|
77
|
+
// Terminal nodes mean the pipeline is done -- complete the instance directly.
|
|
78
|
+
const nodeTemplateType = getCurrentNodeTemplateType(state);
|
|
79
|
+
if (nodeTemplateType === 'review') {
|
|
80
|
+
await runReviewGateAndAdvance(state);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
if (nodeTemplateType === 'terminal') {
|
|
84
|
+
await runCompletionSequence(state, config, governor);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
console.log(`[factory] Designing phase started for instance ${state.instanceId}`);
|
|
88
|
+
// Step 1: Create git worktree (shared with subsequent planning phase)
|
|
89
|
+
// Use product-scoped config so worktrees land in the correct product repo
|
|
90
|
+
if (!state.worktreePath) {
|
|
91
|
+
const productEntry = findFactoryProduct(config, state.productId);
|
|
92
|
+
const productConfig = productEntry ? configForFactoryProduct(config, productEntry) : config;
|
|
93
|
+
const branchName = generateBranchName(state.blueprint.name, state.instanceId);
|
|
94
|
+
try {
|
|
95
|
+
const worktreePath = createFactoryWorktree(productConfig.repoPath, productConfig.factoryWorktreeDir, branchName);
|
|
96
|
+
console.log(`[factory] Worktree created for designing: ${worktreePath} (branch: ${branchName})`);
|
|
97
|
+
await updateInstanceStatus(state.instanceId, 'designing', {
|
|
98
|
+
branchName,
|
|
99
|
+
worktreePath,
|
|
100
|
+
});
|
|
101
|
+
state.branchName = branchName;
|
|
102
|
+
state.worktreePath = worktreePath;
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
await handleInstanceFailure(state, `Failed to create worktree: ${err instanceof Error ? err.message : String(err)}`);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// Step 2: Run strategy design phase (audit -> prompt -> generate -> validate -> persist)
|
|
110
|
+
// Track the designing phase as a planner session for telemetry
|
|
111
|
+
const designSession = await createFactorySession({
|
|
112
|
+
instanceId: state.instanceId,
|
|
113
|
+
sessionType: 'planner',
|
|
114
|
+
});
|
|
115
|
+
await updateFactorySession(designSession.id, {
|
|
116
|
+
status: 'running',
|
|
117
|
+
startedAt: new Date().toISOString(),
|
|
118
|
+
});
|
|
119
|
+
const result = await runStrategyDesignPhase(state, config, governor);
|
|
120
|
+
// Build session update fields including token/cost data (TEL-6)
|
|
121
|
+
const designSessionFields = {
|
|
122
|
+
status: result.success ? 'completed' : 'failed',
|
|
123
|
+
endedAt: new Date().toISOString(),
|
|
124
|
+
};
|
|
125
|
+
if (result.tokenUsage) {
|
|
126
|
+
const sessionTokens = result.tokenUsage.inputTokens + result.tokenUsage.outputTokens;
|
|
127
|
+
designSessionFields.tokenCount = sessionTokens;
|
|
128
|
+
if (result.tokenUsage.totalCostUsd !== null) {
|
|
129
|
+
designSessionFields.costEstimate = result.tokenUsage.totalCostUsd;
|
|
130
|
+
}
|
|
131
|
+
// Aggregate into instance-level token counter
|
|
132
|
+
state.tokensUsed += sessionTokens;
|
|
133
|
+
await updateInstanceStatus(state.instanceId, state.status, {
|
|
134
|
+
tokensUsed: state.tokensUsed,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
await updateFactorySession(designSession.id, designSessionFields);
|
|
138
|
+
if (!result.success) {
|
|
139
|
+
await transitionInstanceStatus(state, 'failed', `Designing failed: ${result.error ?? 'unknown'}`);
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
// Step 3: Transition to planning
|
|
143
|
+
await transitionInstanceStatus(state, 'planning');
|
|
144
|
+
// Clear review gap context after it has been injected into the design prompt
|
|
145
|
+
state.reviewGaps = undefined;
|
|
146
|
+
// Kick off planning immediately
|
|
147
|
+
runPlanning(state).catch((error) => {
|
|
148
|
+
console.error(`[factory] Planning failed for instance ${state.instanceId}:`, error.message);
|
|
149
|
+
handleInstanceFailure(state, `Planning failed: ${error.message}`);
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Execute the Review pipeline node.
|
|
154
|
+
*
|
|
155
|
+
* Review nodes skip the full design/plan/build cycle. Instead they:
|
|
156
|
+
* 1. Run the completion gate against all work produced so far
|
|
157
|
+
* 2. Use gap counts to populate pipeline edge metrics
|
|
158
|
+
* 3. Advance the pipeline (back to Engineering if gaps, to Done if clean)
|
|
159
|
+
* 4. Store gap details in state.reviewGaps so the next Engineering cycle
|
|
160
|
+
* can constrain its scope to closing those gaps only
|
|
161
|
+
*/
|
|
162
|
+
export async function runReviewGateAndAdvance(state) {
|
|
163
|
+
if (!config) {
|
|
164
|
+
handleInstanceFailure(state, 'Config not initialized -- cannot run review gate');
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
// Run the completion gate to get actual gap metrics (only when enabled)
|
|
168
|
+
let gateResult = null;
|
|
169
|
+
if (state.blueprint.completionGateEnabled) {
|
|
170
|
+
console.log(`[factory] Review node: running completion gate for instance ${state.instanceId}`);
|
|
171
|
+
try {
|
|
172
|
+
const workUnits = await getInstanceWorkUnits(state.instanceId);
|
|
173
|
+
gateResult = await runCompletionGate(state, config, workUnits, governor);
|
|
174
|
+
if (gateResult) {
|
|
175
|
+
console.log(`[factory] Review gate: ${gateResult.passed ? 'passed' : 'failed'}, ` +
|
|
176
|
+
`${gateResult.gaps.length} gap(s) found (confidence: ${gateResult.confidence})`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
catch (err) {
|
|
180
|
+
console.warn(`[factory] Review gate error (non-fatal, proceeding with gap_count=0): ` +
|
|
181
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
182
|
+
}
|
|
183
|
+
// Persist the gate evaluation
|
|
184
|
+
if (gateResult) {
|
|
185
|
+
try {
|
|
186
|
+
state.completionGateIterations++;
|
|
187
|
+
await createCycleEvaluation(state.instanceId, state.completionGateIterations, gateResult);
|
|
188
|
+
}
|
|
189
|
+
catch (persistErr) {
|
|
190
|
+
console.warn(`[factory] Failed to persist review cycle evaluation (non-fatal): ` +
|
|
191
|
+
`${persistErr instanceof Error ? persistErr.message : String(persistErr)}`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
console.log(`[factory] Review node: completion gate disabled on blueprint -- skipping AI review for instance ${state.instanceId}`);
|
|
197
|
+
}
|
|
198
|
+
// Compute gap_count (critical gaps only) for pipeline edge evaluation
|
|
199
|
+
const criticalGaps = gateResult ? gateResult.gaps.filter((g) => g.severity === 'critical') : [];
|
|
200
|
+
const gapCount = criticalGaps.length;
|
|
201
|
+
// Assemble pipeline advance metrics
|
|
202
|
+
const workUnitsForMetrics = await getInstanceWorkUnits(state.instanceId).catch(() => []);
|
|
203
|
+
const completedUnits = workUnitsForMetrics.filter((u) => u.status === 'completed');
|
|
204
|
+
const wallClockMs = state.startedAt ? Date.now() - state.startedAt.getTime() : 0;
|
|
205
|
+
const metrics = collectMetrics({
|
|
206
|
+
cycleCount: state.completionGateIterations,
|
|
207
|
+
deliveryCompletionPct: workUnitsForMetrics.length > 0
|
|
208
|
+
? (completedUnits.length / workUnitsForMetrics.length) * 100
|
|
209
|
+
: 0,
|
|
210
|
+
wallClockMinutes: wallClockMs / 60_000,
|
|
211
|
+
gapCount,
|
|
212
|
+
gapSeverity: criticalGaps.length > 0 ? 'critical' : gateResult?.gaps.length ? 'important' : 'low',
|
|
213
|
+
});
|
|
214
|
+
const advanceResult = advancePipeline(state, metrics);
|
|
215
|
+
switch (advanceResult.outcome) {
|
|
216
|
+
case 'advanced': {
|
|
217
|
+
const nextNodeId = advanceResult.nextNodeId;
|
|
218
|
+
const nextNodeTemplateType = state.pipelineGraph.nodes.find((n) => n.id === nextNodeId)?.templateType;
|
|
219
|
+
// If the next node is terminal, treat it as completion rather than
|
|
220
|
+
// starting a new design/build cycle.
|
|
221
|
+
if (nextNodeTemplateType === 'terminal') {
|
|
222
|
+
const nodeLabel = getNodeLabel(state.pipelineGraph, nextNodeId);
|
|
223
|
+
console.log(`[factory] Review: pipeline advanced to terminal node "${nodeLabel}" -- ` +
|
|
224
|
+
`completing instance ${state.instanceId}`);
|
|
225
|
+
await runCompletionSequence(state, config, governor);
|
|
226
|
+
break;
|
|
227
|
+
}
|
|
228
|
+
// Store gap context so the next Engineering designing phase can constrain scope
|
|
229
|
+
if (gateResult && gateResult.gaps.length > 0) {
|
|
230
|
+
state.reviewGaps = gateResult.gaps.map((g) => ({
|
|
231
|
+
area: g.area,
|
|
232
|
+
description: g.description,
|
|
233
|
+
severity: g.severity,
|
|
234
|
+
suggestedFix: g.suggestedFix,
|
|
235
|
+
}));
|
|
236
|
+
console.log(`[factory] Review found ${gateResult.gaps.length} gap(s) -- ` +
|
|
237
|
+
`advancing to Engineering with gap constraints`);
|
|
238
|
+
}
|
|
239
|
+
// Advance to next pipeline node (typically Engineering)
|
|
240
|
+
terminateAllUnits(state);
|
|
241
|
+
await deleteExecutionUnitsByInstance(state.instanceId);
|
|
242
|
+
state.currentPipelineNodeId = nextNodeId;
|
|
243
|
+
state.executionUnits.clear();
|
|
244
|
+
state.completionGateIterations = 0;
|
|
245
|
+
const nodeLabel = getNodeLabel(state.pipelineGraph, nextNodeId);
|
|
246
|
+
// Engineering nodes skip strategy redesign -- go directly to planning
|
|
247
|
+
if (nextNodeTemplateType === 'engineering') {
|
|
248
|
+
state.status = 'planning';
|
|
249
|
+
await updateInstanceStatus(state.instanceId, 'planning', {
|
|
250
|
+
currentPipelineNodeId: nextNodeId,
|
|
251
|
+
});
|
|
252
|
+
console.log(`[factory] Review: pipeline advanced to engineering node "${nodeLabel}" ` +
|
|
253
|
+
`(${nextNodeId}) -- skipping design, starting planning for instance ${state.instanceId}`);
|
|
254
|
+
runPlanning(state).catch((error) => {
|
|
255
|
+
console.error(`[factory] Planning failed after review advance for instance ${state.instanceId}:`, error.message);
|
|
256
|
+
handleInstanceFailure(state, `Planning failed after review: ${error.message}`);
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
else {
|
|
260
|
+
state.status = 'designing';
|
|
261
|
+
await updateInstanceStatus(state.instanceId, 'designing', {
|
|
262
|
+
currentPipelineNodeId: nextNodeId,
|
|
263
|
+
});
|
|
264
|
+
console.log(`[factory] Review: pipeline advanced to node "${nodeLabel}" ` +
|
|
265
|
+
`(${nextNodeId}) for instance ${state.instanceId}`);
|
|
266
|
+
runDesigning(state).catch((error) => {
|
|
267
|
+
console.error(`[factory] Designing failed after review advance for instance ${state.instanceId}:`, error.message);
|
|
268
|
+
handleInstanceFailure(state, `Designing failed after review: ${error.message}`);
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
break;
|
|
272
|
+
}
|
|
273
|
+
case 'completed': {
|
|
274
|
+
// Pipeline reached terminal node -- finalize instance
|
|
275
|
+
console.log(`[factory] Review: pipeline complete at terminal node for instance ${state.instanceId}`);
|
|
276
|
+
state.status = 'completed';
|
|
277
|
+
await runCompletionSequence(state, config, governor);
|
|
278
|
+
break;
|
|
279
|
+
}
|
|
280
|
+
case 'no_match': {
|
|
281
|
+
// No edge conditions matched -- escalate
|
|
282
|
+
terminateAllUnits(state);
|
|
283
|
+
const nodeLabel = getNodeLabel(state.pipelineGraph, state.currentPipelineNodeId);
|
|
284
|
+
console.warn(`[factory] Review: pipeline stuck at "${nodeLabel}" for instance ${state.instanceId}. Escalating.`);
|
|
285
|
+
await createFactoryEscalation(state.instanceId, `Pipeline stuck at Review node "${nodeLabel}": no outgoing edge conditions matched`, `Review pipeline edge conditions for node "${nodeLabel}" and either add a fallback edge or manually advance the instance.`);
|
|
286
|
+
state.status = 'paused';
|
|
287
|
+
await updateInstanceStatus(state.instanceId, 'paused');
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
default:
|
|
291
|
+
break;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Run the planning phase for an instance.
|
|
296
|
+
*
|
|
297
|
+
* Creates a git worktree, spawns Claude Code to generate a work plan,
|
|
298
|
+
* validates and persists work units, then transitions to "building".
|
|
299
|
+
*/
|
|
300
|
+
export async function runPlanning(state) {
|
|
301
|
+
if (!config) {
|
|
302
|
+
handleInstanceFailure(state, 'Config not initialized -- cannot run planning');
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
console.log(`[factory] Planning phase started for instance ${state.instanceId}`);
|
|
306
|
+
// Track the planning phase as a planner session for telemetry
|
|
307
|
+
const planSession = await createFactorySession({
|
|
308
|
+
instanceId: state.instanceId,
|
|
309
|
+
sessionType: 'planner',
|
|
310
|
+
});
|
|
311
|
+
await updateFactorySession(planSession.id, {
|
|
312
|
+
status: 'running',
|
|
313
|
+
startedAt: new Date().toISOString(),
|
|
314
|
+
});
|
|
315
|
+
const result = await runPlanningPhase(state, config, governor);
|
|
316
|
+
// Build session update fields including token/cost data (TEL-6)
|
|
317
|
+
const planSessionFields = {
|
|
318
|
+
status: result.success ? 'completed' : 'failed',
|
|
319
|
+
endedAt: new Date().toISOString(),
|
|
320
|
+
};
|
|
321
|
+
if (result.tokenUsage) {
|
|
322
|
+
const sessionTokens = result.tokenUsage.inputTokens + result.tokenUsage.outputTokens;
|
|
323
|
+
planSessionFields.tokenCount = sessionTokens;
|
|
324
|
+
if (result.tokenUsage.totalCostUsd !== null) {
|
|
325
|
+
planSessionFields.costEstimate = result.tokenUsage.totalCostUsd;
|
|
326
|
+
}
|
|
327
|
+
// Aggregate into instance-level token counter
|
|
328
|
+
state.tokensUsed += sessionTokens;
|
|
329
|
+
await updateInstanceStatus(state.instanceId, state.status, {
|
|
330
|
+
tokensUsed: state.tokensUsed,
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
await updateFactorySession(planSession.id, planSessionFields);
|
|
334
|
+
if (!result.success) {
|
|
335
|
+
await transitionInstanceStatus(state, 'failed', `Planning failed: ${result.error ?? 'unknown'}`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
// ============================================================================
|
|
339
|
+
// Building phase monitoring
|
|
340
|
+
// ============================================================================
|
|
341
|
+
/**
|
|
342
|
+
* Detect in_progress or gate_fix_pending strategies that have no execution
|
|
343
|
+
* unit assigned (orphaned after a session crash). Reset them to pending
|
|
344
|
+
* and their in_progress work units to ready so the scheduler re-assigns them.
|
|
345
|
+
*/
|
|
346
|
+
async function resetOrphanedStrategies(state) {
|
|
347
|
+
let strategies;
|
|
348
|
+
try {
|
|
349
|
+
const { getStrategiesByFactoryInstance } = await import('./strategy-design.js');
|
|
350
|
+
strategies = await getStrategiesByFactoryInstance(state.instanceId);
|
|
351
|
+
}
|
|
352
|
+
catch {
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
// Build set of strategy IDs that have a live execution unit
|
|
356
|
+
const activeStrategyIds = new Set();
|
|
357
|
+
for (const unit of state.executionUnits.values()) {
|
|
358
|
+
if (unit.assignedStrategyId && (unit.status === 'assigned' || unit.status === 'running')) {
|
|
359
|
+
activeStrategyIds.add(unit.assignedStrategyId);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
const orphanStatuses = new Set(['in_progress', 'gate_fix_pending']);
|
|
363
|
+
for (const strategy of strategies) {
|
|
364
|
+
if (!strategy.executionStatus || !orphanStatuses.has(strategy.executionStatus))
|
|
365
|
+
continue;
|
|
366
|
+
if (activeStrategyIds.has(strategy.id))
|
|
367
|
+
continue;
|
|
368
|
+
// Orphaned: strategy is in_progress/gate_fix_pending but no unit is working on it
|
|
369
|
+
console.warn(`[factory] Orphaned strategy "${strategy.name}" (${strategy.id.slice(0, 8)}) ` +
|
|
370
|
+
`is ${strategy.executionStatus} with no live session -- resetting to pending`);
|
|
371
|
+
await updateStrategyExecutionStatus(strategy.id, 'pending', { gateFixStartedAt: null });
|
|
372
|
+
// Reset in_progress work units to ready so the new session gets them dispatched
|
|
373
|
+
const workUnits = await getInstanceWorkUnits(state.instanceId);
|
|
374
|
+
const orphanedUnits = workUnits.filter((wu) => wu.strategyId === strategy.id &&
|
|
375
|
+
wu.status === 'in_progress' &&
|
|
376
|
+
!wu.infrastructureOwned);
|
|
377
|
+
for (const wu of orphanedUnits) {
|
|
378
|
+
await updateWorkUnitStatus(wu.id, 'ready');
|
|
379
|
+
}
|
|
380
|
+
if (orphanedUnits.length > 0) {
|
|
381
|
+
console.log(`[factory] Reset ${orphanedUnits.length} in_progress work unit(s) to ready ` +
|
|
382
|
+
`for strategy "${strategy.name}"`);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Spawn unit sessions for a list of scheduler assignments.
|
|
388
|
+
*
|
|
389
|
+
* Extracted to avoid duplication between the regular scheduler cycle and
|
|
390
|
+
* the post-gate follow-up cycle in checkBuilding.
|
|
391
|
+
*/
|
|
392
|
+
async function spawnAssignedSessions(assignments, state) {
|
|
393
|
+
if (!config)
|
|
394
|
+
return;
|
|
395
|
+
const { getStrategiesByFactoryInstance } = await import('./strategy-design.js');
|
|
396
|
+
const strategies = await getStrategiesByFactoryInstance(state.instanceId);
|
|
397
|
+
const strategiesById = new Map(strategies.map((s) => [s.id, s]));
|
|
398
|
+
for (const assignment of assignments) {
|
|
399
|
+
const unitState = state.executionUnits.get(assignment.unitId);
|
|
400
|
+
if (!unitState) {
|
|
401
|
+
console.warn(`[factory] Assigned unit ${assignment.unitId} not found in state -- skipping spawn`);
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
const dbStrategy = strategiesById.get(assignment.strategyId);
|
|
405
|
+
if (!dbStrategy) {
|
|
406
|
+
console.warn(`[factory] Assigned strategy ${assignment.strategyId} not found -- skipping spawn`);
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
// Update in-memory unit state to reflect the assignment
|
|
410
|
+
unitState.assignedStrategyId = assignment.strategyId;
|
|
411
|
+
unitState.assignedAt = new Date();
|
|
412
|
+
unitState.status = 'assigned';
|
|
413
|
+
const strategyCtx = {
|
|
414
|
+
strategyId: dbStrategy.id,
|
|
415
|
+
strategyName: dbStrategy.name,
|
|
416
|
+
dependsOn: dbStrategy.dependsOn,
|
|
417
|
+
};
|
|
418
|
+
try {
|
|
419
|
+
const adapter = await spawnUnitSession(unitState, strategyCtx, state, config, governor);
|
|
420
|
+
// Seed an infrastructure-owned gate verification work unit for this strategy
|
|
421
|
+
// before dispatching agent work. This unit is resolved by the engine after gates run.
|
|
422
|
+
await seedGateWorkUnit(state.instanceId, assignment.strategyId);
|
|
423
|
+
// Dispatch strategy work units to the session
|
|
424
|
+
const workUnits = await getInstanceWorkUnits(state.instanceId);
|
|
425
|
+
const strategyWorkUnits = workUnits.filter((wu) => wu.strategyId === assignment.strategyId &&
|
|
426
|
+
(wu.status === 'ready' || wu.status === 'pending') &&
|
|
427
|
+
!wu.infrastructureOwned);
|
|
428
|
+
if (strategyWorkUnits.length > 0) {
|
|
429
|
+
adapter.dispatchWork(strategyWorkUnits, state);
|
|
430
|
+
// Track dispatch time and dispatched unit IDs for timeout watchdog
|
|
431
|
+
unitState.dispatchedAt = new Date();
|
|
432
|
+
unitState.dispatchedWorkUnitIds = new Set(strategyWorkUnits.map((wu) => wu.id));
|
|
433
|
+
// Mark work units as in_progress
|
|
434
|
+
for (const wu of strategyWorkUnits) {
|
|
435
|
+
updateWorkUnitStatus(wu.id, 'in_progress').catch((err) => {
|
|
436
|
+
console.error(`[factory] Failed to mark work unit ${wu.id} as in_progress:`, err.message);
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
catch (err) {
|
|
442
|
+
console.error(`[factory] Failed to spawn session for unit ${unitState.slotIndex} ` +
|
|
443
|
+
`(strategy "${dbStrategy.name}"): ${err.message}`);
|
|
444
|
+
// Reset unit back to idle so the scheduler can retry
|
|
445
|
+
unitState.status = 'idle';
|
|
446
|
+
unitState.assignedStrategyId = null;
|
|
447
|
+
unitState.assignedAt = null;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* Check on an instance in the building phase.
|
|
453
|
+
*
|
|
454
|
+
* Runs resource limit checks, the DAG-aware scheduler to assign strategies
|
|
455
|
+
* to execution units, spawns sessions for newly assigned units, monitors
|
|
456
|
+
* running unit health, evaluates strategy-level gates, and checks for
|
|
457
|
+
* instance completion.
|
|
458
|
+
*/
|
|
459
|
+
export async function checkBuilding(state) {
|
|
460
|
+
if (!config)
|
|
461
|
+
return;
|
|
462
|
+
// Update wall_clock_elapsed_seconds on every poll cycle (TEL-6).
|
|
463
|
+
const elapsedSeconds = Math.floor((Date.now() - state.startedAt.getTime()) / 1000);
|
|
464
|
+
await updateInstanceStatus(state.instanceId, 'building', {
|
|
465
|
+
wallClockElapsedSeconds: elapsedSeconds,
|
|
466
|
+
});
|
|
467
|
+
// Check resource limits first -- always pause (never fail directly)
|
|
468
|
+
const limitResult = await checkResourceLimits(state);
|
|
469
|
+
if (limitResult.exceeded) {
|
|
470
|
+
terminateAllUnits(state);
|
|
471
|
+
await escalateInstanceState(state, 'resource_limit_exceeded', limitResult.reason);
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
// Check for stuck work units -- escalate or fail based on blueprint triggers
|
|
475
|
+
const wasEscalated = await checkAndEscalateIfStuck(state, () => transitionInstanceStatus(state, 'failed', 'Builder stuck: repeated gate failures with same error pattern'));
|
|
476
|
+
if (wasEscalated)
|
|
477
|
+
return;
|
|
478
|
+
// --- Orphan detection: reset in_progress strategies with no live session ---
|
|
479
|
+
// When a session dies mid-work the strategy stays in_progress but the unit
|
|
480
|
+
// is released to idle. Reset the strategy to pending (and its work units to
|
|
481
|
+
// ready) so the scheduler can re-assign it.
|
|
482
|
+
await resetOrphanedStrategies(state);
|
|
483
|
+
// --- Scheduler: assign ready strategies to idle execution units ---
|
|
484
|
+
const deps = createSchedulerDeps();
|
|
485
|
+
const cycleResult = await runSchedulerCycle(state.instanceId, deps);
|
|
486
|
+
if (cycleResult.assignments.length > 0) {
|
|
487
|
+
await spawnAssignedSessions(cycleResult.assignments, state);
|
|
488
|
+
}
|
|
489
|
+
// --- Health monitoring: detect orphaned running units ---
|
|
490
|
+
for (const unit of state.executionUnits.values()) {
|
|
491
|
+
if (unit.status === 'running' && !unit.pid) {
|
|
492
|
+
// Unit is marked running but has no process handle (orphaned state)
|
|
493
|
+
console.warn(`[factory] Unit ${unit.slotIndex} is running with no pid -- resetting to idle`);
|
|
494
|
+
unit.status = 'idle';
|
|
495
|
+
unit.assignedStrategyId = null;
|
|
496
|
+
unit.assignedAt = null;
|
|
497
|
+
unit.stdin = null;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
// --- Operational watchdogs: assignment, dispatch, and signal silence ---
|
|
501
|
+
try {
|
|
502
|
+
const { runWatchdogs } = await import('./watchdogs.js');
|
|
503
|
+
await runWatchdogs(state);
|
|
504
|
+
}
|
|
505
|
+
catch (err) {
|
|
506
|
+
console.warn(`[factory] Watchdog check error (non-fatal): ${err.message}`);
|
|
507
|
+
}
|
|
508
|
+
// --- Gate fix timeout enforcement ---
|
|
509
|
+
await checkGateFixTimeouts(state);
|
|
510
|
+
// --- Per-strategy completion detection ---
|
|
511
|
+
// Check if all work units for any in_progress strategy are terminal
|
|
512
|
+
await detectStrategyCompletion(state);
|
|
513
|
+
// --- Strategy-level gate evaluation ---
|
|
514
|
+
let gatesPassedThisCycle = false;
|
|
515
|
+
try {
|
|
516
|
+
const { evaluateStrategyGates } = await import('./strategy-gating.js');
|
|
517
|
+
gatesPassedThisCycle = await evaluateStrategyGates(state, config, governor);
|
|
518
|
+
}
|
|
519
|
+
catch (err) {
|
|
520
|
+
console.warn(`[factory] Strategy gate evaluation error (non-fatal): ${err.message}`);
|
|
521
|
+
}
|
|
522
|
+
// If a strategy just passed gates, it eagerly released its unit back to idle.
|
|
523
|
+
// Run a second scheduler cycle immediately to assign any newly eligible
|
|
524
|
+
// downstream strategies without waiting for the next 30s poll.
|
|
525
|
+
if (gatesPassedThisCycle) {
|
|
526
|
+
const followUpDeps = createSchedulerDeps();
|
|
527
|
+
const followUpResult = await runSchedulerCycle(state.instanceId, followUpDeps);
|
|
528
|
+
if (followUpResult.assignments.length > 0) {
|
|
529
|
+
console.log(`[factory] Post-gate scheduler: assigning ${followUpResult.assignments.length} ` +
|
|
530
|
+
`downstream strategy/ies immediately`);
|
|
531
|
+
await spawnAssignedSessions(followUpResult.assignments, state);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
// --- Cycle summary ---
|
|
535
|
+
logCycleSummary(state);
|
|
536
|
+
// --- Instance completion check ---
|
|
537
|
+
await checkInstanceCompletion(state, config, runDesigning, runPlanning);
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Log a compact cycle summary showing execution unit and strategy states.
|
|
541
|
+
*/
|
|
542
|
+
export function logCycleSummary(state) {
|
|
543
|
+
const units = [...state.executionUnits.values()];
|
|
544
|
+
const idle = units.filter((u) => u.status === 'idle').length;
|
|
545
|
+
const assigned = units.filter((u) => u.status === 'assigned').length;
|
|
546
|
+
const running = units.filter((u) => u.status === 'running').length;
|
|
547
|
+
const terminated = units.filter((u) => u.status === 'terminated').length;
|
|
548
|
+
const assignments = units
|
|
549
|
+
.filter((u) => u.assignedStrategyId)
|
|
550
|
+
.map((u) => `slot${u.slotIndex}=${u.assignedStrategyId.slice(0, 8)}`)
|
|
551
|
+
.join(', ');
|
|
552
|
+
console.log(`[factory] Cycle: units=${idle}i/${assigned}a/${running}r/${terminated}t` +
|
|
553
|
+
(assignments ? ` | ${assignments}` : '') +
|
|
554
|
+
` | tokens=${state.tokensUsed}`);
|
|
555
|
+
}
|
|
556
|
+
// ============================================================================
|
|
557
|
+
// Infrastructure-owned gate work unit seeding
|
|
558
|
+
// ============================================================================
|
|
559
|
+
/**
|
|
560
|
+
* Seed an infrastructure-owned "Gate verification" work unit for a strategy.
|
|
561
|
+
*
|
|
562
|
+
* Called before the first agent prompt is dispatched for a strategy. The unit
|
|
563
|
+
* is created with `infrastructureOwned: true` so agents cannot claim it.
|
|
564
|
+
* It will be resolved to completed or failed by the engine after gate
|
|
565
|
+
* evaluation runs (see resolveGateWorkUnit).
|
|
566
|
+
*
|
|
567
|
+
* The cycle number is determined by looking at existing work units for the
|
|
568
|
+
* strategy and using the max cycle number found (or 1 if none exist).
|
|
569
|
+
*/
|
|
570
|
+
async function seedGateWorkUnit(instanceId, strategyId) {
|
|
571
|
+
try {
|
|
572
|
+
// Determine cycle number from existing work units for this strategy
|
|
573
|
+
const existingUnits = await getInstanceWorkUnits(instanceId);
|
|
574
|
+
const strategyUnits = existingUnits.filter((u) => u.strategyId === strategyId);
|
|
575
|
+
const cycleNumber = strategyUnits.length > 0
|
|
576
|
+
? Math.max(...strategyUnits.map((u) => u.cycleNumber))
|
|
577
|
+
: 1;
|
|
578
|
+
// Check if a gate verification unit already exists for this strategy and cycle
|
|
579
|
+
const existing = strategyUnits.find((u) => u.infrastructureOwned && u.cycleNumber === cycleNumber);
|
|
580
|
+
if (existing) {
|
|
581
|
+
console.log(`[factory] Gate verification unit already exists for strategy ${strategyId.slice(0, 8)} ` +
|
|
582
|
+
`cycle ${cycleNumber} -- skipping seed`);
|
|
583
|
+
return;
|
|
584
|
+
}
|
|
585
|
+
await createWorkUnits(instanceId, [{
|
|
586
|
+
id: randomUUID(),
|
|
587
|
+
title: 'Gate verification',
|
|
588
|
+
description: 'Infrastructure-managed gate verification. This work unit is automatically ' +
|
|
589
|
+
'created and resolved by the factory engine. Do not modify.',
|
|
590
|
+
sortOrder: 9999,
|
|
591
|
+
blockedBy: [],
|
|
592
|
+
cycleNumber,
|
|
593
|
+
strategyId,
|
|
594
|
+
infrastructureOwned: true,
|
|
595
|
+
}]);
|
|
596
|
+
console.log(`[factory] Seeded gate verification unit for strategy ${strategyId.slice(0, 8)} ` +
|
|
597
|
+
`(cycle ${cycleNumber})`);
|
|
598
|
+
}
|
|
599
|
+
catch (err) {
|
|
600
|
+
// Non-fatal -- gate seeding failure should not block agent execution
|
|
601
|
+
console.warn(`[factory] Failed to seed gate verification unit for strategy ${strategyId.slice(0, 8)}: ` +
|
|
602
|
+
`${err.message}`);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
// ============================================================================
|
|
606
|
+
// Gate fix timeout enforcement
|
|
607
|
+
// ============================================================================
|
|
608
|
+
/**
|
|
609
|
+
* Check for strategies stuck in gate_fix_pending past their timeout.
|
|
610
|
+
*
|
|
611
|
+
* For each gate_fix_pending strategy with a configured gate_fix_timeout_ms,
|
|
612
|
+
* checks whether gate_fix_started_at + timeout has elapsed. On timeout,
|
|
613
|
+
* transitions the strategy to failed, creates an escalation, and terminates
|
|
614
|
+
* the unit session.
|
|
615
|
+
*
|
|
616
|
+
* Also warns about orphaned gate_fix_pending strategies that have no live
|
|
617
|
+
* unit session (the scheduler may reassign, so we don't auto-fail).
|
|
618
|
+
*/
|
|
619
|
+
/** @internal Exported for testing. */
|
|
620
|
+
export async function checkGateFixTimeouts(state) {
|
|
621
|
+
const timeoutMs = state.blueprint.gateFixTimeoutMs;
|
|
622
|
+
let strategies;
|
|
623
|
+
try {
|
|
624
|
+
const { getStrategiesByFactoryInstance } = await import('./strategy-design.js');
|
|
625
|
+
strategies = await getStrategiesByFactoryInstance(state.instanceId);
|
|
626
|
+
}
|
|
627
|
+
catch {
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
const gateFixStrategies = strategies.filter((s) => s.executionStatus === 'gate_fix_pending');
|
|
631
|
+
if (gateFixStrategies.length === 0)
|
|
632
|
+
return;
|
|
633
|
+
const now = Date.now();
|
|
634
|
+
for (const strategy of gateFixStrategies) {
|
|
635
|
+
// Detect orphaned gate_fix_pending (no live unit session)
|
|
636
|
+
const hasLiveUnit = [...state.executionUnits.values()].some((u) => u.assignedStrategyId === strategy.id && u.status === 'running');
|
|
637
|
+
if (!hasLiveUnit) {
|
|
638
|
+
console.warn(`[factory] Strategy "${strategy.name}" is gate_fix_pending but has no live unit session ` +
|
|
639
|
+
`-- scheduler may reassign`);
|
|
640
|
+
}
|
|
641
|
+
// Timeout enforcement (only if blueprint configures it)
|
|
642
|
+
if (timeoutMs && strategy.gateFixStartedAt) {
|
|
643
|
+
const startedAt = new Date(strategy.gateFixStartedAt).getTime();
|
|
644
|
+
const elapsed = now - startedAt;
|
|
645
|
+
if (elapsed >= timeoutMs) {
|
|
646
|
+
console.log(`[factory] Strategy "${strategy.name}" failed: gate fix timeout ` +
|
|
647
|
+
`(${timeoutMs}ms) exceeded (elapsed: ${elapsed}ms)`);
|
|
648
|
+
try {
|
|
649
|
+
await updateStrategyExecutionStatus(strategy.id, 'failed', { gateFixStartedAt: null });
|
|
650
|
+
// Retrieve last gate feedback for context
|
|
651
|
+
const lastFeedback = state.gateFailureFeedback.get(strategy.id) ?? 'no feedback available';
|
|
652
|
+
const summary = `Strategy "${strategy.name}" exceeded gate fix timeout (${timeoutMs}ms, ` +
|
|
653
|
+
`elapsed: ${elapsed}ms). Last gate feedback: ${lastFeedback.slice(0, 500)}`;
|
|
654
|
+
await createFactoryEscalation(state.instanceId, 'gate_fix_timeout', summary);
|
|
655
|
+
}
|
|
656
|
+
catch (err) {
|
|
657
|
+
console.error(`[factory] Failed to fail timed-out strategy ${strategy.id}:`, err.message);
|
|
658
|
+
}
|
|
659
|
+
// Terminate the unit session
|
|
660
|
+
for (const unit of state.executionUnits.values()) {
|
|
661
|
+
if (unit.assignedStrategyId === strategy.id) {
|
|
662
|
+
terminateUnitSession(unit);
|
|
663
|
+
break;
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
// ============================================================================
|
|
671
|
+
// Resume recovery (re-export from state machine for poll loop use)
|
|
672
|
+
// ============================================================================
|
|
673
|
+
export { recoverResumedInstance } from './instance-state-machine.js';
|
|
674
|
+
//# sourceMappingURL=instance-phase-dispatch.js.map
|