kantban-cli 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +3 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +54 -0
- package/dist/client.js.map +1 -1
- package/dist/commands/cron.js +2 -2
- package/dist/commands/cron.js.map +1 -1
- package/dist/commands/pipeline-init.d.ts +2 -0
- package/dist/commands/pipeline-init.d.ts.map +1 -0
- package/dist/commands/pipeline-init.js +100 -0
- package/dist/commands/pipeline-init.js.map +1 -0
- package/dist/commands/pipeline.d.ts.map +1 -1
- package/dist/commands/pipeline.js +637 -44
- package/dist/commands/pipeline.js.map +1 -1
- package/dist/lib/advisor.d.ts +108 -0
- package/dist/lib/advisor.d.ts.map +1 -0
- package/dist/lib/advisor.js +139 -0
- package/dist/lib/advisor.js.map +1 -0
- package/dist/lib/checkpoint.d.ts +15 -0
- package/dist/lib/checkpoint.d.ts.map +1 -0
- package/dist/lib/checkpoint.js +49 -0
- package/dist/lib/checkpoint.js.map +1 -0
- package/dist/lib/constraint-evaluator.d.ts +40 -0
- package/dist/lib/constraint-evaluator.d.ts.map +1 -0
- package/dist/lib/constraint-evaluator.js +189 -0
- package/dist/lib/constraint-evaluator.js.map +1 -0
- package/dist/lib/cost-tracker.d.ts +46 -0
- package/dist/lib/cost-tracker.d.ts.map +1 -0
- package/dist/lib/cost-tracker.js +120 -0
- package/dist/lib/cost-tracker.js.map +1 -0
- package/dist/lib/evaluator.d.ts +17 -0
- package/dist/lib/evaluator.d.ts.map +1 -0
- package/dist/lib/evaluator.js +71 -0
- package/dist/lib/evaluator.js.map +1 -0
- package/dist/lib/event-emitter.d.ts +28 -0
- package/dist/lib/event-emitter.d.ts.map +1 -0
- package/dist/lib/event-emitter.js +100 -0
- package/dist/lib/event-emitter.js.map +1 -0
- package/dist/lib/gate-config.d.ts +7 -0
- package/dist/lib/gate-config.d.ts.map +1 -0
- package/dist/lib/gate-config.js +68 -0
- package/dist/lib/gate-config.js.map +1 -0
- package/dist/lib/gate-proxy-server.d.ts +16 -0
- package/dist/lib/gate-proxy-server.d.ts.map +1 -0
- package/dist/lib/gate-proxy-server.js +385 -0
- package/dist/lib/gate-proxy-server.js.map +1 -0
- package/dist/lib/gate-proxy.d.ts +46 -0
- package/dist/lib/gate-proxy.d.ts.map +1 -0
- package/dist/lib/gate-proxy.js +104 -0
- package/dist/lib/gate-proxy.js.map +1 -0
- package/dist/lib/gate-runner.d.ts +13 -0
- package/dist/lib/gate-runner.d.ts.map +1 -0
- package/dist/lib/gate-runner.js +104 -0
- package/dist/lib/gate-runner.js.map +1 -0
- package/dist/lib/gate-snapshot.d.ts +12 -0
- package/dist/lib/gate-snapshot.d.ts.map +1 -0
- package/dist/lib/gate-snapshot.js +49 -0
- package/dist/lib/gate-snapshot.js.map +1 -0
- package/dist/lib/light-call.d.ts +37 -0
- package/dist/lib/light-call.d.ts.map +1 -0
- package/dist/lib/light-call.js +62 -0
- package/dist/lib/light-call.js.map +1 -0
- package/dist/lib/logger.d.ts +2 -0
- package/dist/lib/logger.d.ts.map +1 -1
- package/dist/lib/logger.js +55 -9
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/mcp-config.d.ts +15 -0
- package/dist/lib/mcp-config.d.ts.map +1 -1
- package/dist/lib/mcp-config.js +70 -6
- package/dist/lib/mcp-config.js.map +1 -1
- package/dist/lib/orchestrator.d.ts +220 -6
- package/dist/lib/orchestrator.d.ts.map +1 -1
- package/dist/lib/orchestrator.js +1265 -58
- package/dist/lib/orchestrator.js.map +1 -1
- package/dist/lib/parse-utils.d.ts +6 -0
- package/dist/lib/parse-utils.d.ts.map +1 -0
- package/dist/lib/parse-utils.js +64 -0
- package/dist/lib/parse-utils.js.map +1 -0
- package/dist/lib/prompt-composer.d.ts +30 -1
- package/dist/lib/prompt-composer.d.ts.map +1 -1
- package/dist/lib/prompt-composer.js +162 -27
- package/dist/lib/prompt-composer.js.map +1 -1
- package/dist/lib/ralph-loop.d.ts +78 -4
- package/dist/lib/ralph-loop.d.ts.map +1 -1
- package/dist/lib/ralph-loop.js +249 -40
- package/dist/lib/ralph-loop.js.map +1 -1
- package/dist/lib/reaper.d.ts +14 -0
- package/dist/lib/reaper.d.ts.map +1 -0
- package/dist/lib/reaper.js +114 -0
- package/dist/lib/reaper.js.map +1 -0
- package/dist/lib/replanner.d.ts +49 -0
- package/dist/lib/replanner.d.ts.map +1 -0
- package/dist/lib/replanner.js +61 -0
- package/dist/lib/replanner.js.map +1 -0
- package/dist/lib/run-memory.d.ts +37 -0
- package/dist/lib/run-memory.d.ts.map +1 -0
- package/dist/lib/run-memory.js +115 -0
- package/dist/lib/run-memory.js.map +1 -0
- package/dist/lib/stream-parser.d.ts +20 -0
- package/dist/lib/stream-parser.d.ts.map +1 -0
- package/dist/lib/stream-parser.js +65 -0
- package/dist/lib/stream-parser.js.map +1 -0
- package/dist/lib/stuck-detector.d.ts +47 -0
- package/dist/lib/stuck-detector.d.ts.map +1 -0
- package/dist/lib/stuck-detector.js +105 -0
- package/dist/lib/stuck-detector.js.map +1 -0
- package/dist/lib/tool-profiles.d.ts +19 -0
- package/dist/lib/tool-profiles.d.ts.map +1 -0
- package/dist/lib/tool-profiles.js +22 -0
- package/dist/lib/tool-profiles.js.map +1 -0
- package/dist/lib/worktree.d.ts +12 -0
- package/dist/lib/worktree.d.ts.map +1 -0
- package/dist/lib/worktree.js +29 -0
- package/dist/lib/worktree.js.map +1 -0
- package/dist/lib/ws-client.d.ts +1 -1
- package/dist/lib/ws-client.d.ts.map +1 -1
- package/dist/lib/ws-client.js +5 -2
- package/dist/lib/ws-client.js.map +1 -1
- package/package.json +3 -1
package/dist/lib/orchestrator.js
CHANGED
|
@@ -1,3 +1,19 @@
|
|
|
1
|
+
import { resolveToolRestrictions } from './tool-profiles.js';
|
|
2
|
+
import { generateWorktreeName } from './worktree.js';
|
|
3
|
+
import { evaluateConstraints } from './constraint-evaluator.js';
|
|
4
|
+
import { classifyTrajectory } from './stuck-detector.js';
|
|
5
|
+
import { readCheckpoint } from './checkpoint.js';
|
|
6
|
+
import { parseVerdict, resolveVerdictAction } from './evaluator.js';
|
|
7
|
+
import { shouldFireReplanner } from './replanner.js';
|
|
8
|
+
export function classifyTier(input) {
|
|
9
|
+
if (input.invocationTier === 'light')
|
|
10
|
+
return 'light';
|
|
11
|
+
if (input.invocationTier === 'heavy')
|
|
12
|
+
return 'heavy';
|
|
13
|
+
if (!input.hasPromptDocument)
|
|
14
|
+
return 'light';
|
|
15
|
+
return 'heavy';
|
|
16
|
+
}
|
|
1
17
|
/**
|
|
2
18
|
* PipelineOrchestrator — coordination layer for Ralph Loops.
|
|
3
19
|
*
|
|
@@ -23,12 +39,55 @@ export class PipelineOrchestrator {
|
|
|
23
39
|
deferredTickets = new Map(); // ticketId → columnId
|
|
24
40
|
/** Ticket IDs currently in the spawning process (prevents double-spawn race) */
|
|
25
41
|
spawning = new Set();
|
|
42
|
+
/** Ticket IDs currently in onLoopComplete (prevents double-spawn during async advisor recovery) */
|
|
43
|
+
completing = new Set();
|
|
26
44
|
/** Per-column reservation count for in-flight spawns (prevents concurrency overshoot) */
|
|
27
45
|
columnReservations = new Map();
|
|
46
|
+
/** Cached board scope for constraint evaluation — refreshed each scan cycle */
|
|
47
|
+
cachedBoardScope = null;
|
|
48
|
+
/** Last time a loop was spawned per column — for column.last_fired_at subject */
|
|
49
|
+
lastFiredAt = new Map();
|
|
50
|
+
/** Column IDs currently blocked by firing constraints (prevents redundant re-evaluation within a scan) */
|
|
51
|
+
blockedColumns = new Set();
|
|
52
|
+
/** Per-ticket advisor invocation count for the current column transit */
|
|
53
|
+
advisorBudget = new Map();
|
|
54
|
+
/** Per-ticket model override (set by RETRY_DIFFERENT_MODEL, consumed by startTrackedLoop) */
|
|
55
|
+
ticketModelOverrides = new Map();
|
|
56
|
+
/** Stable session ID for this orchestrator instance (pipeline run) */
|
|
57
|
+
sessionId;
|
|
58
|
+
/** Replanner invocation count for the current pipeline run */
|
|
59
|
+
replannerInvocations = 0;
|
|
60
|
+
/** Maximum replanner invocations before auto-pause */
|
|
61
|
+
replannerMaxInvocations = 3;
|
|
62
|
+
/** When true, pipeline is paused — scanAndSpawn and spawnOrQueue will skip */
|
|
63
|
+
pipelinePaused = false;
|
|
64
|
+
/** Per-column consecutive scope refresh failure counts — auto-pauses after 3 for any column */
|
|
65
|
+
columnScopeRefreshFailures = new Map();
|
|
66
|
+
/** Consecutive board scope refresh failure count — auto-pauses after 3 */
|
|
67
|
+
boardScopeRefreshFailures = 0;
|
|
68
|
+
/** Timestamp when the pipeline started (for duration-based replanner triggers) */
|
|
69
|
+
pipelineStartTime = Date.now();
|
|
28
70
|
constructor(boardId, projectId, deps) {
|
|
29
71
|
this.boardId = boardId;
|
|
30
72
|
this.projectId = projectId;
|
|
31
73
|
this.deps = deps;
|
|
74
|
+
this.sessionId = crypto.randomUUID();
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Execute an async action with logged error handling.
|
|
78
|
+
* Returns true if the action succeeded, false if it threw.
|
|
79
|
+
* Non-blocking — callers can ignore the return value for fire-and-forget.
|
|
80
|
+
*/
|
|
81
|
+
async safeAction(ticketId, label, fn) {
|
|
82
|
+
try {
|
|
83
|
+
await fn();
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
88
|
+
console.error(` [error] ${label} failed for ${ticketId}: ${msg}`);
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
32
91
|
}
|
|
33
92
|
/** Returns the IDs of all discovered pipeline columns. */
|
|
34
93
|
get pipelineColumnIds() {
|
|
@@ -46,6 +105,14 @@ export class PipelineOrchestrator {
|
|
|
46
105
|
}
|
|
47
106
|
return this.deferredTickets.size > 0 || this.spawning.size > 0;
|
|
48
107
|
}
|
|
108
|
+
/** Returns true if any tickets are queued (waiting for a slot) or spawning — excludes deferred tickets. */
|
|
109
|
+
get hasActiveQueuedWork() {
|
|
110
|
+
for (const [, queue] of this.loopQueues) {
|
|
111
|
+
if (queue.length > 0)
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
return this.spawning.size > 0;
|
|
115
|
+
}
|
|
49
116
|
/** Returns the number of queued (waiting) tickets for a column. */
|
|
50
117
|
queuedCount(columnId) {
|
|
51
118
|
return this.loopQueues.get(columnId)?.length ?? 0;
|
|
@@ -56,8 +123,9 @@ export class PipelineOrchestrator {
|
|
|
56
123
|
*/
|
|
57
124
|
async initialize() {
|
|
58
125
|
const boardScope = await this.deps.fetchBoardScope(this.boardId);
|
|
59
|
-
|
|
60
|
-
|
|
126
|
+
this.cachedBoardScope = boardScope;
|
|
127
|
+
// Identify pipeline columns: (has_prompt=true OR type=evaluator) AND type !== 'done'
|
|
128
|
+
const pipelineCols = boardScope.columns.filter((col) => (col.has_prompt || col.type === 'evaluator') && col.type !== 'done');
|
|
61
129
|
// Fetch column scope for each pipeline column to get agent_config and tickets
|
|
62
130
|
await Promise.all(pipelineCols.map(async (col) => {
|
|
63
131
|
const colScope = await this.deps.fetchColumnScope(col.id);
|
|
@@ -65,13 +133,35 @@ export class PipelineOrchestrator {
|
|
|
65
133
|
this.pipelineColumns.set(col.id, {
|
|
66
134
|
columnId: col.id,
|
|
67
135
|
name: col.name,
|
|
136
|
+
columnType: col.type,
|
|
68
137
|
concurrency: cfg?.concurrency ?? 1,
|
|
69
138
|
maxIterations: cfg?.max_iterations ?? 10,
|
|
70
139
|
gutterThreshold: cfg?.gutter_threshold ?? 3,
|
|
71
140
|
modelPreference: cfg?.model_preference,
|
|
72
141
|
maxBudgetUsd: cfg?.max_budget_usd,
|
|
73
142
|
worktreeEnabled: cfg?.worktree?.enabled,
|
|
74
|
-
|
|
143
|
+
worktreeOnMove: cfg?.worktree?.on_move,
|
|
144
|
+
worktreeOnDone: cfg?.worktree?.on_done,
|
|
145
|
+
invocationTier: cfg?.invocation_tier,
|
|
146
|
+
lookaheadColumnId: cfg?.lookahead_column_id,
|
|
147
|
+
runMemory: cfg?.run_memory,
|
|
148
|
+
advisorEnabled: cfg?.advisor?.enabled,
|
|
149
|
+
advisorMaxInvocations: cfg?.advisor?.max_invocations ?? 2,
|
|
150
|
+
advisorModel: cfg?.advisor?.model ?? 'haiku',
|
|
151
|
+
checkpointEnabled: cfg?.checkpoint,
|
|
152
|
+
modelRouting: cfg?.model_routing ? {
|
|
153
|
+
initial: cfg.model_routing.initial,
|
|
154
|
+
escalation: cfg.model_routing.escalation,
|
|
155
|
+
escalateAfter: cfg.model_routing.escalate_after ?? 2,
|
|
156
|
+
} : undefined,
|
|
157
|
+
stuckDetection: cfg?.stuck_detection?.enabled ? {
|
|
158
|
+
enabled: true,
|
|
159
|
+
firstCheck: cfg.stuck_detection.first_check ?? 3,
|
|
160
|
+
interval: cfg.stuck_detection.interval ?? 2,
|
|
161
|
+
} : undefined,
|
|
162
|
+
builtinTools: cfg?.builtin_tools,
|
|
163
|
+
allowedTools: cfg?.allowed_tools,
|
|
164
|
+
disallowedTools: cfg?.disallowed_tools,
|
|
75
165
|
});
|
|
76
166
|
// Cache column scope for scanAndSpawn
|
|
77
167
|
this.columnScopes.set(col.id, colScope);
|
|
@@ -87,17 +177,258 @@ export class PipelineOrchestrator {
|
|
|
87
177
|
try {
|
|
88
178
|
const colScope = await this.deps.fetchColumnScope(columnId);
|
|
89
179
|
this.columnScopes.set(columnId, colScope);
|
|
180
|
+
this.columnScopeRefreshFailures.delete(columnId);
|
|
181
|
+
}
|
|
182
|
+
catch (err) {
|
|
183
|
+
const count = (this.columnScopeRefreshFailures.get(columnId) ?? 0) + 1;
|
|
184
|
+
this.columnScopeRefreshFailures.set(columnId, count);
|
|
185
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
186
|
+
console.error(` [warn] refreshColumnScope failed for ${columnId} (attempt ${count}, keeping stale): ${msg}`);
|
|
187
|
+
if (count >= 3) {
|
|
188
|
+
console.error(` [error] 3 consecutive scope refresh failures for column ${columnId} — pausing pipeline`);
|
|
189
|
+
this.pipelinePaused = true;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Refresh the cached board scope. Called at the start of each scan cycle
|
|
195
|
+
* to get fresh ticket counts for constraint evaluation.
|
|
196
|
+
*/
|
|
197
|
+
async refreshBoardScope() {
|
|
198
|
+
try {
|
|
199
|
+
this.cachedBoardScope = await this.deps.fetchBoardScope(this.boardId);
|
|
200
|
+
this.boardScopeRefreshFailures = 0;
|
|
201
|
+
// Auto-unpause if pipeline was paused due to scope refresh failures
|
|
202
|
+
if (this.pipelinePaused) {
|
|
203
|
+
const anyColumnFailing = Array.from(this.columnScopeRefreshFailures.values()).some(c => c >= 3);
|
|
204
|
+
if (!anyColumnFailing) {
|
|
205
|
+
console.error(' [info] Scope refresh succeeded — unpausing pipeline');
|
|
206
|
+
this.pipelinePaused = false;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
catch (err) {
|
|
211
|
+
this.boardScopeRefreshFailures++;
|
|
212
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
213
|
+
console.error(` [warn] refreshBoardScope failed (attempt ${this.boardScopeRefreshFailures}, keeping stale): ${msg}`);
|
|
214
|
+
if (this.boardScopeRefreshFailures >= 3) {
|
|
215
|
+
console.error(' [error] 3 consecutive scope refresh failures — pausing pipeline');
|
|
216
|
+
this.pipelinePaused = true;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Public method to invalidate constraint caches when WS events arrive
|
|
222
|
+
* (firing_constraint:created/updated/deleted). Forces re-fetch on next access.
|
|
223
|
+
*/
|
|
224
|
+
async refreshConstraints() {
|
|
225
|
+
// Re-fetch column scopes to get updated firing_constraints arrays
|
|
226
|
+
await Promise.all(Array.from(this.pipelineColumns.keys()).map((colId) => this.refreshColumnScope(colId)));
|
|
227
|
+
this.blockedColumns.clear();
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Build the BoardState required by the constraint evaluator from current
|
|
231
|
+
* orchestrator state + cached board scope.
|
|
232
|
+
*/
|
|
233
|
+
buildBoardState() {
|
|
234
|
+
const bs = this.cachedBoardScope;
|
|
235
|
+
if (!bs) {
|
|
236
|
+
return {
|
|
237
|
+
columns: [],
|
|
238
|
+
active_loops: new Map(),
|
|
239
|
+
last_fired_at: new Map(),
|
|
240
|
+
circuit_breaker_count: 0,
|
|
241
|
+
backlog_ticket_count: 0,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
// Compute active loops per column
|
|
245
|
+
const activeLoopsPerColumn = new Map();
|
|
246
|
+
for (const [, loop] of this.activeLoops) {
|
|
247
|
+
activeLoopsPerColumn.set(loop.columnId, (activeLoopsPerColumn.get(loop.columnId) ?? 0) + 1);
|
|
90
248
|
}
|
|
91
|
-
|
|
92
|
-
|
|
249
|
+
// Compute seconds since last fire per column
|
|
250
|
+
const now = Date.now();
|
|
251
|
+
const lastFiredSeconds = new Map();
|
|
252
|
+
for (const [colId, firedAt] of this.lastFiredAt) {
|
|
253
|
+
lastFiredSeconds.set(colId, Math.floor((now - firedAt.getTime()) / 1000));
|
|
254
|
+
}
|
|
255
|
+
// Circuit breaker count = ticket count in circuit breaker target column
|
|
256
|
+
let circuitBreakerCount = 0;
|
|
257
|
+
if (bs.circuit_breaker.target_column_id) {
|
|
258
|
+
const targetCol = bs.columns.find((c) => c.id === bs.circuit_breaker.target_column_id);
|
|
259
|
+
if (targetCol)
|
|
260
|
+
circuitBreakerCount = targetCol.ticket_count;
|
|
261
|
+
}
|
|
262
|
+
return {
|
|
263
|
+
columns: bs.columns.map((c) => ({
|
|
264
|
+
id: c.id,
|
|
265
|
+
name: c.name,
|
|
266
|
+
position: c.position,
|
|
267
|
+
column_type: c.type,
|
|
268
|
+
wip_limit: c.wip_limit,
|
|
269
|
+
ticket_count: c.ticket_count,
|
|
270
|
+
})),
|
|
271
|
+
active_loops: activeLoopsPerColumn,
|
|
272
|
+
last_fired_at: lastFiredSeconds,
|
|
273
|
+
circuit_breaker_count: circuitBreakerCount,
|
|
274
|
+
backlog_ticket_count: bs.backlog_ticket_count,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Convert FiringConstraintLite from column scope to full FiringConstraint
|
|
279
|
+
* expected by the evaluator.
|
|
280
|
+
*/
|
|
281
|
+
toFiringConstraints(lites) {
|
|
282
|
+
return lites.map((c) => ({
|
|
283
|
+
id: c.id,
|
|
284
|
+
project_id: this.projectId,
|
|
285
|
+
board_id: this.boardId,
|
|
286
|
+
column_id: c.column_id,
|
|
287
|
+
name: c.name,
|
|
288
|
+
description: null,
|
|
289
|
+
enabled: c.enabled,
|
|
290
|
+
subject_type: c.subject_type,
|
|
291
|
+
subject_ref: c.subject_ref,
|
|
292
|
+
subject_param: null,
|
|
293
|
+
operator: c.operator,
|
|
294
|
+
value: c.value,
|
|
295
|
+
scope: c.scope,
|
|
296
|
+
notify: c.notify,
|
|
297
|
+
position: 0,
|
|
298
|
+
fail_open: c.fail_open ?? false,
|
|
299
|
+
system: c.system ?? false,
|
|
300
|
+
created_at: '',
|
|
301
|
+
updated_at: '',
|
|
302
|
+
}));
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* Evaluate firing constraints for a column. Returns the EvalResult if any
|
|
306
|
+
* constraint failed, or null if all passed (column is clear to fire).
|
|
307
|
+
*
|
|
308
|
+
* Handles logging and optional signal creation for blocked columns.
|
|
309
|
+
*/
|
|
310
|
+
evaluateColumnConstraints(columnId) {
|
|
311
|
+
const colScope = this.columnScopes.get(columnId);
|
|
312
|
+
// Build constraint list: user-defined + synthetic safety net
|
|
313
|
+
let constraints = [];
|
|
314
|
+
if (colScope?.firing_constraints && colScope.firing_constraints.length > 0) {
|
|
315
|
+
constraints = this.toFiringConstraints(colScope.firing_constraints);
|
|
316
|
+
}
|
|
317
|
+
// Synthetic safety net: if this is a pipeline column with a prompt but no
|
|
318
|
+
// user-defined column.ticket_count constraint, inject one to prevent
|
|
319
|
+
// firing on empty lanes. This is the runtime guarantee — even if the
|
|
320
|
+
// persisted system constraint is deleted, we never waste tokens.
|
|
321
|
+
const hasTicketCountConstraint = constraints.some((c) => c.enabled && c.scope === 'column' && c.subject_type === 'column.ticket_count'
|
|
322
|
+
&& (c.subject_ref === null || c.subject_ref === 'self'));
|
|
323
|
+
if (!hasTicketCountConstraint && this.pipelineColumns.has(columnId)) {
|
|
324
|
+
constraints.push({
|
|
325
|
+
id: '00000000-0000-0000-0000-000000000000',
|
|
326
|
+
project_id: this.projectId,
|
|
327
|
+
board_id: this.boardId,
|
|
328
|
+
column_id: columnId,
|
|
329
|
+
name: 'Minimum tickets (synthetic)',
|
|
330
|
+
description: null,
|
|
331
|
+
enabled: true,
|
|
332
|
+
subject_type: 'column.ticket_count',
|
|
333
|
+
subject_ref: null,
|
|
334
|
+
subject_param: null,
|
|
335
|
+
operator: 'gt',
|
|
336
|
+
value: 0,
|
|
337
|
+
scope: 'column',
|
|
338
|
+
notify: false,
|
|
339
|
+
position: -1,
|
|
340
|
+
fail_open: false,
|
|
341
|
+
system: true,
|
|
342
|
+
created_at: '',
|
|
343
|
+
updated_at: '',
|
|
344
|
+
});
|
|
93
345
|
}
|
|
346
|
+
if (constraints.length === 0) {
|
|
347
|
+
return null;
|
|
348
|
+
}
|
|
349
|
+
const boardState = this.buildBoardState();
|
|
350
|
+
const result = evaluateConstraints(constraints, boardState, columnId);
|
|
351
|
+
if (result.summary.failed === 0) {
|
|
352
|
+
return null; // All passed
|
|
353
|
+
}
|
|
354
|
+
// Log each failed constraint clearly
|
|
355
|
+
const colName = colScope?.column.name ?? columnId;
|
|
356
|
+
for (const r of result.results) {
|
|
357
|
+
if (!r.passed && !r.error) {
|
|
358
|
+
console.error(` [blocked] Column "${colName}" (${columnId}): constraint "${r.name}" ` +
|
|
359
|
+
`FAILED — resolved=${String(r.resolved_value)} ${r.threshold.operator} ${String(r.threshold.value)}`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
// Emit constraint events
|
|
363
|
+
for (const r of result.results) {
|
|
364
|
+
if (!r.passed && !r.error) {
|
|
365
|
+
this.deps.eventEmitter?.emit({
|
|
366
|
+
layer: 'constraint',
|
|
367
|
+
eventType: 'constraint_blocked',
|
|
368
|
+
severity: 'failure',
|
|
369
|
+
summary: `Constraint "${r.name}" blocked [${colName}]`,
|
|
370
|
+
detail: {
|
|
371
|
+
constraint_name: r.name,
|
|
372
|
+
constraint_id: r.constraint_id,
|
|
373
|
+
resolved_value: r.resolved_value,
|
|
374
|
+
threshold: { operator: r.threshold.operator, value: r.threshold.value },
|
|
375
|
+
fail_open: false,
|
|
376
|
+
blocked_tickets: [],
|
|
377
|
+
column_name: colName,
|
|
378
|
+
},
|
|
379
|
+
columnId,
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
if (result.summary.errors > 0) {
|
|
384
|
+
for (const r of result.results) {
|
|
385
|
+
if (r.error) {
|
|
386
|
+
console.error(` [constraint-error] Column "${colName}" (${columnId}): "${r.name}" — ${r.error} (fail-open)`);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
// Handle notify flag: create signals for blocked constraints with notify=true
|
|
391
|
+
const constraintMap = new Map(constraints.map((c) => [c.id, c]));
|
|
392
|
+
for (const r of result.results) {
|
|
393
|
+
if (!r.passed && !r.error) {
|
|
394
|
+
const constraint = constraintMap.get(r.constraint_id);
|
|
395
|
+
if (constraint?.notify && this.deps.createColumnSignal) {
|
|
396
|
+
void this.deps.createColumnSignal(columnId, `Firing constraint "${r.name}" blocked column: ` +
|
|
397
|
+
`resolved=${String(r.resolved_value)} ${r.threshold.operator} ${String(r.threshold.value)}`).catch((err) => {
|
|
398
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
399
|
+
console.error(` [warn] Failed to create constraint signal: ${msg}`);
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
return result;
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Check if a column is blocked by firing constraints.
|
|
408
|
+
* Returns true if blocked (should not fire), false if clear.
|
|
409
|
+
*/
|
|
410
|
+
isColumnBlocked(columnId) {
|
|
411
|
+
const result = this.evaluateColumnConstraints(columnId);
|
|
412
|
+
return result !== null;
|
|
94
413
|
}
|
|
95
414
|
/**
|
|
96
415
|
* Scan all pipeline columns for existing tickets and spawn loops.
|
|
97
416
|
* Refreshes column scopes before scanning to avoid stale ticket lists.
|
|
417
|
+
* Evaluates firing constraints per column before processing tickets.
|
|
98
418
|
* Respects per-column concurrency limits — excess tickets are queued.
|
|
99
419
|
*/
|
|
100
420
|
async scanAndSpawn() {
|
|
421
|
+
if (this.pipelinePaused) {
|
|
422
|
+
console.error(' [scan] Pipeline paused by replanner — skipping scan');
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
if (this.deps.costTracker?.isExhausted()) {
|
|
426
|
+
if (!this.pipelinePaused) {
|
|
427
|
+
this.pipelinePaused = true;
|
|
428
|
+
console.error('[budget] Token budget exhausted — pausing pipeline');
|
|
429
|
+
}
|
|
430
|
+
return;
|
|
431
|
+
}
|
|
101
432
|
// Reset knownTickets each scan cycle so that completed loops can be
|
|
102
433
|
// re-evaluated on the next poll. The activeLoops and spawning checks
|
|
103
434
|
// already prevent true double-spawns; knownTickets only deduplicates
|
|
@@ -126,11 +457,16 @@ export class PipelineOrchestrator {
|
|
|
126
457
|
this.knownTickets.add(ticketId);
|
|
127
458
|
}
|
|
128
459
|
}
|
|
129
|
-
catch {
|
|
460
|
+
catch (err) {
|
|
130
461
|
// Leave deferred — will re-evaluate next cycle
|
|
462
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
463
|
+
console.error(` [warn] Deferred blocker re-evaluation failed for ${ticketId}: ${msg}`);
|
|
131
464
|
this.knownTickets.add(ticketId);
|
|
132
465
|
}
|
|
133
466
|
}
|
|
467
|
+
// Refresh board scope for fresh ticket counts used by constraint evaluation
|
|
468
|
+
await this.refreshBoardScope();
|
|
469
|
+
this.blockedColumns.clear();
|
|
134
470
|
for (const [columnId] of this.pipelineColumns) {
|
|
135
471
|
await this.refreshColumnScope(columnId);
|
|
136
472
|
const colScope = this.columnScopes.get(columnId);
|
|
@@ -138,6 +474,12 @@ export class PipelineOrchestrator {
|
|
|
138
474
|
console.error(` [scan] Column ${columnId}: no cached scope`);
|
|
139
475
|
continue;
|
|
140
476
|
}
|
|
477
|
+
// Evaluate column-scope firing constraints BEFORE processing tickets
|
|
478
|
+
if (this.isColumnBlocked(columnId)) {
|
|
479
|
+
this.blockedColumns.add(columnId);
|
|
480
|
+
console.error(` [scan] Column ${columnId} (${colScope.column.name}): BLOCKED by firing constraints — skipping ${String(colScope.tickets.length)} ticket(s)`);
|
|
481
|
+
continue;
|
|
482
|
+
}
|
|
141
483
|
console.error(` [scan] Column ${columnId} (${colScope.column.name}): ${String(colScope.tickets.length)} ticket(s)`);
|
|
142
484
|
for (const ticket of colScope.tickets) {
|
|
143
485
|
await this.spawnOrQueue(ticket.id, columnId);
|
|
@@ -152,15 +494,24 @@ export class PipelineOrchestrator {
|
|
|
152
494
|
case 'ticket:moved':
|
|
153
495
|
case 'ticket:created': {
|
|
154
496
|
if (event.columnId && this.pipelineColumns.has(event.columnId)) {
|
|
155
|
-
|
|
497
|
+
// Check firing constraints before spawning
|
|
498
|
+
if (this.isColumnBlocked(event.columnId)) {
|
|
499
|
+
console.error(` [event] ${event.type} ${event.ticketId} → column ${event.columnId}: BLOCKED by firing constraints — deferred`);
|
|
500
|
+
this.deferredTickets.set(event.ticketId, event.columnId);
|
|
501
|
+
// Don't break — still need to re-evaluate blockers below for ticket:moved
|
|
502
|
+
}
|
|
503
|
+
else {
|
|
504
|
+
await this.spawnOrQueue(event.ticketId, event.columnId, true);
|
|
505
|
+
}
|
|
156
506
|
}
|
|
157
|
-
// On any move, check if the moved ticket unblocks dependents
|
|
158
507
|
if (event.type === 'ticket:moved') {
|
|
508
|
+
let blockedIds = null;
|
|
159
509
|
try {
|
|
160
510
|
const blocked = await this.deps.fetchBlockedTickets(event.ticketId);
|
|
511
|
+
blockedIds = new Set(blocked.map(b => b.id));
|
|
161
512
|
for (const dep of blocked) {
|
|
162
513
|
if (dep.id === event.ticketId)
|
|
163
|
-
continue;
|
|
514
|
+
continue;
|
|
164
515
|
if (dep.column_id && this.pipelineColumns.has(dep.column_id)) {
|
|
165
516
|
this.knownTickets.delete(dep.id);
|
|
166
517
|
this.deferredTickets.delete(dep.id);
|
|
@@ -172,10 +523,10 @@ export class PipelineOrchestrator {
|
|
|
172
523
|
const msg = err instanceof Error ? err.message : String(err);
|
|
173
524
|
console.error(` [warn] Failed to fetch blocked tickets for ${event.ticketId}: ${msg}`);
|
|
174
525
|
}
|
|
175
|
-
// Re-evaluate
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
526
|
+
// Re-evaluate deferred tickets scoped to those blocked by the moved ticket.
|
|
527
|
+
// Falls back to re-evaluating all if fetchBlockedTickets failed (blockedIds is null).
|
|
528
|
+
const toReEvaluate = Array.from(this.deferredTickets).filter(([deferredId]) => deferredId !== event.ticketId && (!blockedIds || blockedIds.has(deferredId)));
|
|
529
|
+
for (const [deferredId, deferredCol] of toReEvaluate) {
|
|
179
530
|
this.knownTickets.delete(deferredId);
|
|
180
531
|
this.deferredTickets.delete(deferredId);
|
|
181
532
|
await this.spawnOrQueue(deferredId, deferredCol, true);
|
|
@@ -202,6 +553,7 @@ export class PipelineOrchestrator {
|
|
|
202
553
|
this.knownTickets.delete(event.ticketId);
|
|
203
554
|
this.deferredTickets.delete(event.ticketId);
|
|
204
555
|
this.spawning.delete(event.ticketId);
|
|
556
|
+
this.advisorBudget.delete(event.ticketId);
|
|
205
557
|
// Also remove from any queue
|
|
206
558
|
for (const [, queue] of this.loopQueues) {
|
|
207
559
|
const idx = queue.indexOf(event.ticketId);
|
|
@@ -217,19 +569,30 @@ export class PipelineOrchestrator {
|
|
|
217
569
|
* Spawn a loop for a ticket if under concurrency limit, otherwise queue it.
|
|
218
570
|
* @param skipKnownCheck - true for event-driven spawns (bypass scan dedup)
|
|
219
571
|
*/
|
|
220
|
-
async spawnOrQueue(ticketId, columnId, skipKnownCheck = false) {
|
|
572
|
+
async spawnOrQueue(ticketId, columnId, skipKnownCheck = false, skipCompletingCheck = false) {
|
|
573
|
+
if (this.pipelinePaused)
|
|
574
|
+
return;
|
|
221
575
|
// Don't double-spawn
|
|
222
576
|
if (this.activeLoops.has(ticketId))
|
|
223
577
|
return;
|
|
224
578
|
// Prevent double-spawn race: two rapid events passing the activeLoops check
|
|
225
579
|
if (this.spawning.has(ticketId))
|
|
226
580
|
return;
|
|
581
|
+
// Prevent double-spawn during async advisor recovery (H1)
|
|
582
|
+
// Skipped for retry paths that call spawnOrQueue from within onLoopComplete.
|
|
583
|
+
if (!skipCompletingCheck && this.completing.has(ticketId))
|
|
584
|
+
return;
|
|
227
585
|
// During scanAndSpawn, prevent re-processing tickets seen in other columns
|
|
228
586
|
if (!skipKnownCheck && this.knownTickets.has(ticketId))
|
|
229
587
|
return;
|
|
230
588
|
const colConfig = this.pipelineColumns.get(columnId);
|
|
231
589
|
if (!colConfig)
|
|
232
590
|
return;
|
|
591
|
+
// Belt-and-suspenders: check firing constraints even if already checked at column level.
|
|
592
|
+
// This catches paths like blocker re-evaluation and deferred ticket re-queue.
|
|
593
|
+
if (this.isColumnBlocked(columnId)) {
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
233
596
|
// Count active loops for this column
|
|
234
597
|
const activeInColumn = this.activeCountForColumn(columnId);
|
|
235
598
|
// Mark as known to prevent re-processing
|
|
@@ -253,7 +616,10 @@ export class PipelineOrchestrator {
|
|
|
253
616
|
this.releaseSlot(columnId);
|
|
254
617
|
this.deferredTickets.set(ticketId, columnId);
|
|
255
618
|
console.error(` [skip] ${ticketId} has unresolved blockers — deferred`);
|
|
256
|
-
void this.drainQueue(columnId).catch(() => {
|
|
619
|
+
void this.drainQueue(columnId).catch((err) => {
|
|
620
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
621
|
+
console.error(` [error] drainQueue failed for column ${columnId}: ${msg}`);
|
|
622
|
+
});
|
|
257
623
|
return;
|
|
258
624
|
}
|
|
259
625
|
}
|
|
@@ -265,7 +631,10 @@ export class PipelineOrchestrator {
|
|
|
265
631
|
this.spawning.delete(ticketId);
|
|
266
632
|
this.releaseSlot(columnId);
|
|
267
633
|
this.deferredTickets.set(ticketId, columnId);
|
|
268
|
-
void this.drainQueue(columnId)
|
|
634
|
+
void this.drainQueue(columnId).catch((err) => {
|
|
635
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
636
|
+
console.error(` [error] drainQueue failed for column ${columnId}: ${msg}`);
|
|
637
|
+
});
|
|
269
638
|
return;
|
|
270
639
|
}
|
|
271
640
|
// Spawn the loop
|
|
@@ -277,6 +646,7 @@ export class PipelineOrchestrator {
|
|
|
277
646
|
const msg = err instanceof Error ? err.message : String(err);
|
|
278
647
|
console.error(` [error] Failed to claim ticket ${ticketId}: ${msg}`);
|
|
279
648
|
this.knownTickets.delete(ticketId);
|
|
649
|
+
this.deferredTickets.set(ticketId, columnId); // Re-evaluate on next move/scan
|
|
280
650
|
}
|
|
281
651
|
finally {
|
|
282
652
|
this.spawning.delete(ticketId);
|
|
@@ -287,16 +657,161 @@ export class PipelineOrchestrator {
|
|
|
287
657
|
* Start a loop and track it. Attach completion handler for cleanup + queue drain.
|
|
288
658
|
*/
|
|
289
659
|
startTrackedLoop(ticketId, columnId, config) {
|
|
660
|
+
// Evaluator columns always use heavy tier (need full tool access for adversarial review)
|
|
661
|
+
const tier = config.columnType === 'evaluator' ? 'heavy' : classifyTier({
|
|
662
|
+
hasPromptDocument: this.columnScopes.get(columnId)?.prompt_document != null,
|
|
663
|
+
invocationTier: config.invocationTier,
|
|
664
|
+
});
|
|
665
|
+
if (tier === 'light' && this.deps.dispatchLightCall) {
|
|
666
|
+
const promise = this.deps.dispatchLightCall(ticketId, columnId).then(async (response) => {
|
|
667
|
+
try {
|
|
668
|
+
switch (response.action) {
|
|
669
|
+
case 'move_ticket': {
|
|
670
|
+
const targetColId = response.params.column_id;
|
|
671
|
+
if (targetColId && this.deps.moveTicketToColumn) {
|
|
672
|
+
await this.deps.moveTicketToColumn(ticketId, targetColId, {
|
|
673
|
+
reason: response.reason,
|
|
674
|
+
source: 'light_call',
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
return { reason: 'moved', iterations: 1, gutterCount: 0 };
|
|
678
|
+
}
|
|
679
|
+
case 'set_field_value': {
|
|
680
|
+
if (this.deps.setFieldValue) {
|
|
681
|
+
const fieldName = response.params.field_name;
|
|
682
|
+
const value = response.params.value;
|
|
683
|
+
if (fieldName)
|
|
684
|
+
await this.deps.setFieldValue(ticketId, fieldName, value);
|
|
685
|
+
}
|
|
686
|
+
return { reason: 'stalled', iterations: 1, gutterCount: 0 };
|
|
687
|
+
}
|
|
688
|
+
case 'archive_ticket': {
|
|
689
|
+
if (this.deps.archiveTicket) {
|
|
690
|
+
await this.deps.archiveTicket(ticketId);
|
|
691
|
+
}
|
|
692
|
+
return { reason: 'moved', iterations: 1, gutterCount: 0 };
|
|
693
|
+
}
|
|
694
|
+
case 'create_comment': {
|
|
695
|
+
const body = response.params.body;
|
|
696
|
+
if (body)
|
|
697
|
+
await this.deps.createComment(ticketId, body);
|
|
698
|
+
return { reason: 'stalled', iterations: 1, gutterCount: 0 };
|
|
699
|
+
}
|
|
700
|
+
case 'no_action':
|
|
701
|
+
default:
|
|
702
|
+
return { reason: 'stalled', iterations: 1, gutterCount: 0 };
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
catch (err) {
|
|
706
|
+
return {
|
|
707
|
+
reason: 'error',
|
|
708
|
+
iterations: 1,
|
|
709
|
+
gutterCount: 0,
|
|
710
|
+
lastError: err instanceof Error ? err.message : String(err),
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
}, (err) => ({
|
|
714
|
+
reason: 'error',
|
|
715
|
+
iterations: 1,
|
|
716
|
+
gutterCount: 0,
|
|
717
|
+
lastError: err instanceof Error ? err.message : String(err),
|
|
718
|
+
}));
|
|
719
|
+
this.activeLoops.set(ticketId, { columnId, promise });
|
|
720
|
+
// Track last fire time for column.last_fired_at constraint subject
|
|
721
|
+
this.lastFiredAt.set(columnId, new Date());
|
|
722
|
+
void promise.then((result) => this.onLoopComplete(ticketId, columnId, result).catch((err) => {
|
|
723
|
+
console.error(`onLoopComplete error for ${ticketId}:`, err);
|
|
724
|
+
}));
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
// Consume any model override (set by RETRY_DIFFERENT_MODEL advisor action)
|
|
728
|
+
const modelOverride = this.ticketModelOverrides.get(ticketId);
|
|
729
|
+
this.ticketModelOverrides.delete(ticketId);
|
|
730
|
+
// Read checkpoint if enabled — resume from saved iteration/gutter/model state
|
|
731
|
+
if (config.checkpointEnabled && this.deps.getFieldValues) {
|
|
732
|
+
// Placeholder to prevent double-spawn during async checkpoint read.
|
|
733
|
+
// Times out after 30s as a safety net — if checkpoint read hangs,
|
|
734
|
+
// the ticket won't be permanently locked.
|
|
735
|
+
let timeoutId;
|
|
736
|
+
const placeholder = new Promise((_, reject) => {
|
|
737
|
+
timeoutId = setTimeout(() => {
|
|
738
|
+
// Set completing guard FIRST to prevent scanAndSpawn from grabbing this ticket
|
|
739
|
+
this.completing.add(ticketId);
|
|
740
|
+
this.activeLoops.delete(ticketId);
|
|
741
|
+
this.knownTickets.delete(ticketId);
|
|
742
|
+
// Clear completing — ticket is fully released now, guard was only needed during cleanup
|
|
743
|
+
this.completing.delete(ticketId);
|
|
744
|
+
console.error(` [warn] Checkpoint read timed out for ${ticketId} — releasing lock`);
|
|
745
|
+
reject(new Error('Checkpoint read timeout'));
|
|
746
|
+
}, 30_000);
|
|
747
|
+
});
|
|
748
|
+
placeholder.catch(() => { }); // Prevent unhandled rejection
|
|
749
|
+
this.activeLoops.set(ticketId, { columnId, promise: placeholder });
|
|
750
|
+
this.lastFiredAt.set(columnId, new Date());
|
|
751
|
+
// Note: setFieldValue is required by CheckpointDeps interface but only used by
|
|
752
|
+
// writeCheckpoint/clearCheckpoint, not readCheckpoint.
|
|
753
|
+
void readCheckpoint({
|
|
754
|
+
setFieldValue: async () => { },
|
|
755
|
+
getFieldValues: this.deps.getFieldValues,
|
|
756
|
+
}, ticketId, columnId).then((checkpoint) => {
|
|
757
|
+
clearTimeout(timeoutId);
|
|
758
|
+
if (checkpoint) {
|
|
759
|
+
console.error(` [checkpoint] Resuming ${ticketId} at iteration ${String(checkpoint.iteration + 1)} (model: ${checkpoint.model_tier})`);
|
|
760
|
+
this.startLoopWithConfig(ticketId, columnId, config, checkpoint.iteration + 1, checkpoint.gutter_count, modelOverride ?? checkpoint.model_tier, checkpoint.last_fingerprint);
|
|
761
|
+
}
|
|
762
|
+
else {
|
|
763
|
+
this.startLoopWithConfig(ticketId, columnId, config, undefined, undefined, modelOverride);
|
|
764
|
+
}
|
|
765
|
+
}).catch((err) => {
|
|
766
|
+
clearTimeout(timeoutId);
|
|
767
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
768
|
+
console.error(` [warn] Checkpoint read failed for ${ticketId} (starting fresh): ${msg}`);
|
|
769
|
+
this.startLoopWithConfig(ticketId, columnId, config, undefined, undefined, modelOverride);
|
|
770
|
+
});
|
|
771
|
+
return;
|
|
772
|
+
}
|
|
773
|
+
// No checkpoint — start directly
|
|
774
|
+
this.startLoopWithConfig(ticketId, columnId, config, undefined, undefined, modelOverride);
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* Build a LoopConfig and start a loop, tracking it in activeLoops.
|
|
778
|
+
* Handles model routing resolution: resumeModelTier > modelRouting.initial > modelPreference.
|
|
779
|
+
*/
|
|
780
|
+
startLoopWithConfig(ticketId, columnId, config, startIteration, startGutterCount, resumeModelTier, startFingerprint) {
|
|
781
|
+
// Resolve effective model: resume tier > routing initial > column preference
|
|
782
|
+
const effectiveModel = resumeModelTier
|
|
783
|
+
?? config.modelRouting?.initial
|
|
784
|
+
?? config.modelPreference;
|
|
290
785
|
const loopConfig = {
|
|
291
786
|
maxIterations: config.maxIterations,
|
|
292
787
|
gutterThreshold: config.gutterThreshold,
|
|
293
|
-
...(
|
|
788
|
+
...(effectiveModel !== undefined && { model: effectiveModel }),
|
|
294
789
|
...(config.maxBudgetUsd !== undefined && { maxBudgetUsd: config.maxBudgetUsd }),
|
|
295
|
-
|
|
296
|
-
...(
|
|
790
|
+
// Resolve worktree name from ticket context
|
|
791
|
+
...((() => {
|
|
792
|
+
const colScope = this.columnScopes.get(columnId);
|
|
793
|
+
const ticket = colScope?.tickets.find(t => t.id === ticketId);
|
|
794
|
+
const wName = config.worktreeEnabled && ticket
|
|
795
|
+
? generateWorktreeName(ticket.ticket_number, config.name)
|
|
796
|
+
: undefined;
|
|
797
|
+
return wName !== undefined ? { worktreeName: wName } : {};
|
|
798
|
+
})()),
|
|
799
|
+
...(config.lookaheadColumnId !== undefined && { lookaheadColumnId: config.lookaheadColumnId }),
|
|
800
|
+
// Resume from checkpoint iteration/gutter if provided
|
|
801
|
+
...(startIteration !== undefined && { startIteration }),
|
|
802
|
+
...(startGutterCount !== undefined && { startGutterCount }),
|
|
803
|
+
...(startFingerprint !== undefined && { startFingerprint }),
|
|
804
|
+
...(config.stuckDetection && { stuckDetection: config.stuckDetection }),
|
|
297
805
|
};
|
|
806
|
+
// Resolve tool profile for this column
|
|
807
|
+
const toolRestrictions = resolveToolRestrictions(config.builtinTools, config.allowedTools, config.disallowedTools);
|
|
808
|
+
if (toolRestrictions.tools !== undefined || toolRestrictions.allowedTools || toolRestrictions.disallowedTools) {
|
|
809
|
+
loopConfig.toolRestrictions = toolRestrictions;
|
|
810
|
+
}
|
|
298
811
|
const promise = this.deps.startLoop(ticketId, columnId, loopConfig);
|
|
299
812
|
this.activeLoops.set(ticketId, { columnId, promise });
|
|
813
|
+
// Track last fire time for column.last_fired_at constraint subject
|
|
814
|
+
this.lastFiredAt.set(columnId, new Date());
|
|
300
815
|
// Attach completion handler — do NOT await
|
|
301
816
|
void promise.then((result) => this.onLoopComplete(ticketId, columnId, result).catch((err) => {
|
|
302
817
|
console.error(`onLoopComplete error for ${ticketId}:`, err);
|
|
@@ -309,54 +824,734 @@ export class PipelineOrchestrator {
|
|
|
309
824
|
console.error(`onLoopComplete error for ${ticketId}:`, completionErr);
|
|
310
825
|
}));
|
|
311
826
|
}
|
|
827
|
+
/**
|
|
828
|
+
* Phase 2: Invoke the advisor for failure recovery.
|
|
829
|
+
* Returns true if the ticket should be retried, false if handled.
|
|
830
|
+
*/
|
|
831
|
+
async invokeAdvisorRecovery(ticketId, columnId, result, colConfig) {
|
|
832
|
+
if (!this.deps.invokeAdvisor)
|
|
833
|
+
return false;
|
|
834
|
+
if (!colConfig.advisorEnabled)
|
|
835
|
+
return false;
|
|
836
|
+
const remaining = this.advisorBudget.get(ticketId) ?? colConfig.advisorMaxInvocations ?? 2;
|
|
837
|
+
if (remaining <= 0)
|
|
838
|
+
return false;
|
|
839
|
+
const colScope = this.columnScopes.get(columnId);
|
|
840
|
+
// ticketDescription: ColumnTicket does not carry a description field (only id, ticket_number,
|
|
841
|
+
// title). The full description is fetched lazily by the loop itself and is not propagated back
|
|
842
|
+
// in LoopResult. Leave empty — the advisor prompt shows the title and field values instead.
|
|
843
|
+
const input = {
|
|
844
|
+
ticketTitle: '',
|
|
845
|
+
ticketDescription: '',
|
|
846
|
+
ticketId,
|
|
847
|
+
ticketNumber: 0,
|
|
848
|
+
columnName: colScope?.column.name ?? columnId,
|
|
849
|
+
exitReason: result.reason,
|
|
850
|
+
iterations: result.iterations,
|
|
851
|
+
gutterCount: result.gutterCount,
|
|
852
|
+
...(result.lastError !== undefined && { lastError: result.lastError }),
|
|
853
|
+
// recentComments: LoopResult does not carry comments; leave empty.
|
|
854
|
+
recentComments: [],
|
|
855
|
+
fieldValues: [],
|
|
856
|
+
failurePatterns: '',
|
|
857
|
+
remainingBudget: remaining,
|
|
858
|
+
modelTier: result.model ?? colConfig.modelPreference ?? 'default',
|
|
859
|
+
escalationModels: colConfig.modelRouting?.escalation ?? [],
|
|
860
|
+
...(this.cachedBoardScope?.circuit_breaker.target_column_id != null && {
|
|
861
|
+
circuitBreakerTargetId: this.cachedBoardScope.circuit_breaker.target_column_id,
|
|
862
|
+
}),
|
|
863
|
+
};
|
|
864
|
+
const ticket = colScope?.tickets.find((t) => t.id === ticketId);
|
|
865
|
+
if (ticket) {
|
|
866
|
+
input.ticketTitle = ticket.title;
|
|
867
|
+
input.ticketNumber = ticket.ticket_number;
|
|
868
|
+
}
|
|
869
|
+
// Populate failurePatterns from the latest gate snapshot — gives the advisor concrete
|
|
870
|
+
// signal about which gates are failing and what their output looks like.
|
|
871
|
+
if (this.deps.gateSnapshotStore) {
|
|
872
|
+
const latest = this.deps.gateSnapshotStore.getLatest(ticketId);
|
|
873
|
+
if (latest) {
|
|
874
|
+
const failedGates = latest.results.filter((r) => !r.passed);
|
|
875
|
+
if (failedGates.length > 0) {
|
|
876
|
+
input.failurePatterns = failedGates
|
|
877
|
+
.map((r) => {
|
|
878
|
+
const label = r.required ? `${r.name} (required)` : r.name;
|
|
879
|
+
const snippet = r.output ? ` — ${r.output.slice(0, 300)}` : '';
|
|
880
|
+
return `${label}${snippet}`;
|
|
881
|
+
})
|
|
882
|
+
.join('\n');
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
// Enrich advisor context with field values
|
|
887
|
+
if (this.deps.getFieldValues) {
|
|
888
|
+
try {
|
|
889
|
+
input.fieldValues = await this.deps.getFieldValues(ticketId);
|
|
890
|
+
}
|
|
891
|
+
catch (err) {
|
|
892
|
+
// Non-blocking — advisor works with partial context
|
|
893
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
894
|
+
console.error(` [warn] Advisor getFieldValues failed for ${ticketId}: ${msg}`);
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
// Enrich with gate data from snapshot store
|
|
898
|
+
if (this.deps.gateSnapshotStore) {
|
|
899
|
+
const recentSnapshots = this.deps.gateSnapshotStore.getRecent(ticketId, 3);
|
|
900
|
+
if (recentSnapshots.length > 0) {
|
|
901
|
+
input.gateHistory = recentSnapshots;
|
|
902
|
+
const latest = this.deps.gateSnapshotStore.getLatest(ticketId);
|
|
903
|
+
if (latest) {
|
|
904
|
+
input.currentGateResults = latest.results;
|
|
905
|
+
}
|
|
906
|
+
input.trajectory = classifyTrajectory(recentSnapshots);
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
let response;
|
|
910
|
+
try {
|
|
911
|
+
response = await this.deps.invokeAdvisor(input);
|
|
912
|
+
}
|
|
913
|
+
catch (err) {
|
|
914
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
915
|
+
console.error(` [advisor] Failed for ${ticketId}: ${msg} — falling back to default behavior`);
|
|
916
|
+
return false;
|
|
917
|
+
}
|
|
918
|
+
// Decrement after successful call — transient errors don't waste budget
|
|
919
|
+
this.advisorBudget.set(ticketId, remaining - 1);
|
|
920
|
+
// Emit advisor event
|
|
921
|
+
const gateSnapshot = this.deps.gateSnapshotStore?.getLatest(ticketId);
|
|
922
|
+
const gatePassed = gateSnapshot ? gateSnapshot.results.filter(r => r.passed).length : 0;
|
|
923
|
+
const gateTotal = gateSnapshot ? gateSnapshot.results.length : 0;
|
|
924
|
+
this.deps.eventEmitter?.emit({
|
|
925
|
+
layer: 'advisor',
|
|
926
|
+
eventType: `advisor_${response.action.toLowerCase()}`,
|
|
927
|
+
severity: response.action === 'ESCALATE' ? 'failure' : response.action === 'RETRY_WITH_FEEDBACK' || response.action === 'RETRY_DIFFERENT_MODEL' ? 'info' : 'warning',
|
|
928
|
+
summary: `Advisor: ${response.action} — ${response.reason.slice(0, 200)}`,
|
|
929
|
+
detail: {
|
|
930
|
+
action: response.action,
|
|
931
|
+
reason: response.reason,
|
|
932
|
+
feedback: response.feedback ?? null,
|
|
933
|
+
budget_remaining: `${String(remaining - 1)}/${String(colConfig.advisorMaxInvocations ?? 2)}`,
|
|
934
|
+
gate_summary: gateTotal > 0 ? { passed: gatePassed, total: gateTotal } : null,
|
|
935
|
+
trajectory: input.trajectory ? { status: input.trajectory.status, confidence: input.trajectory.confidence } : null,
|
|
936
|
+
ticket_number: ticket?.ticket_number ?? null,
|
|
937
|
+
},
|
|
938
|
+
ticketId,
|
|
939
|
+
columnId,
|
|
940
|
+
});
|
|
941
|
+
switch (response.action) {
|
|
942
|
+
case 'RETRY_WITH_FEEDBACK': {
|
|
943
|
+
if (response.feedback) {
|
|
944
|
+
await this.safeAction(ticketId, 'advisor:createComment', () => this.deps.createComment(ticketId, `ADVISOR FEEDBACK:\n${response.feedback}`));
|
|
945
|
+
}
|
|
946
|
+
// activeLoops.delete already ran at top of onLoopComplete.
|
|
947
|
+
this.knownTickets.delete(ticketId);
|
|
948
|
+
await this.spawnOrQueue(ticketId, columnId, true, true);
|
|
949
|
+
// Clear completing guard AFTER spawnOrQueue to prevent concurrent scanAndSpawn racing in.
|
|
950
|
+
this.completing.delete(ticketId);
|
|
951
|
+
return true;
|
|
952
|
+
}
|
|
953
|
+
case 'RETRY_DIFFERENT_MODEL': {
|
|
954
|
+
// Resolve next model from escalation ladder
|
|
955
|
+
const currentModel = result.model ?? colConfig.modelPreference ?? 'default';
|
|
956
|
+
const escalation = colConfig.modelRouting?.escalation ?? [];
|
|
957
|
+
let nextModel;
|
|
958
|
+
// Find current model's position in the full ladder [initial, ...escalation]
|
|
959
|
+
const fullLadder = colConfig.modelRouting
|
|
960
|
+
? [colConfig.modelRouting.initial, ...colConfig.modelRouting.escalation]
|
|
961
|
+
: [];
|
|
962
|
+
const currentIdx = fullLadder.indexOf(currentModel);
|
|
963
|
+
if (currentIdx >= 0 && currentIdx < fullLadder.length - 1) {
|
|
964
|
+
nextModel = fullLadder[currentIdx + 1];
|
|
965
|
+
}
|
|
966
|
+
else if (escalation.length > 0) {
|
|
967
|
+
// Current model not in ladder — use first escalation model
|
|
968
|
+
nextModel = escalation[0];
|
|
969
|
+
}
|
|
970
|
+
const escalatedModel = nextModel ?? currentModel;
|
|
971
|
+
if (escalatedModel === currentModel) {
|
|
972
|
+
// Already at top of escalation ladder — cannot escalate further
|
|
973
|
+
return false;
|
|
974
|
+
}
|
|
975
|
+
await this.safeAction(ticketId, 'advisor:createComment', () => this.deps.createComment(ticketId, `ADVISOR: Escalating model ${currentModel} → ${escalatedModel} — ${response.reason}`));
|
|
976
|
+
this.knownTickets.delete(ticketId);
|
|
977
|
+
// Store model override and go through spawnOrQueue to respect concurrency
|
|
978
|
+
this.ticketModelOverrides.set(ticketId, escalatedModel);
|
|
979
|
+
await this.spawnOrQueue(ticketId, columnId, true, true);
|
|
980
|
+
// Clear completing guard AFTER spawnOrQueue to prevent concurrent scanAndSpawn racing in.
|
|
981
|
+
this.completing.delete(ticketId);
|
|
982
|
+
return true;
|
|
983
|
+
}
|
|
984
|
+
case 'RELAX_WITH_DEBT': {
|
|
985
|
+
// Extract waived gates once — used for both field storage and move metadata
|
|
986
|
+
const waivedGates = (response.debt_items ?? [])
|
|
987
|
+
.filter((d) => d.type === 'waived_gate')
|
|
988
|
+
.map((d) => d.description);
|
|
989
|
+
if (response.debt_items && response.debt_items.length > 0 && this.deps.setFieldValue) {
|
|
990
|
+
const items = response.debt_items.map((d) => ({ ...d, source_column: colScope?.column.name ?? '' }));
|
|
991
|
+
await this.safeAction(ticketId, 'advisor:setFieldValue:debt_items', () => this.deps.setFieldValue(ticketId, 'debt_items', items));
|
|
992
|
+
// Record waived gates on ticket — gate proxy will skip these
|
|
993
|
+
if (waivedGates.length > 0) {
|
|
994
|
+
await this.safeAction(ticketId, 'advisor:setFieldValue:gate_waiver', () => this.deps.setFieldValue(ticketId, 'gate_waiver', waivedGates));
|
|
995
|
+
}
|
|
996
|
+
await this.safeAction(ticketId, 'advisor:createSignal:debt', () => this.deps.createSignal(ticketId, `DEBT: This ticket advanced with ${String(items.length)} unresolved items (${String(items.filter((d) => d.severity === 'high').length)} high, ${String(items.filter((d) => d.severity === 'medium').length)} medium).`));
|
|
997
|
+
// G4: Propagate upstream debt signal to blocked tickets
|
|
998
|
+
if (this.deps.fetchBlockedTickets) {
|
|
999
|
+
try {
|
|
1000
|
+
const blocked = await this.deps.fetchBlockedTickets(ticketId);
|
|
1001
|
+
const ticketNum = colScope?.tickets.find((t) => t.id === ticketId)?.ticket_number;
|
|
1002
|
+
for (const dep of blocked) {
|
|
1003
|
+
if (dep.id === ticketId)
|
|
1004
|
+
continue;
|
|
1005
|
+
await this.safeAction(dep.id, 'advisor:createSignal:upstream_debt', () => this.deps.createSignal(dep.id, `UPSTREAM DEBT: Blocking ticket #${String(ticketNum ?? '?')} advanced with debt. Review debt_items before building on its interfaces.`));
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
catch (err) {
|
|
1009
|
+
// Non-blocking
|
|
1010
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1011
|
+
console.error(` [warn] Upstream debt signal propagation failed for ${ticketId}: ${msg}`);
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
await this.safeAction(ticketId, 'advisor:createComment', () => this.deps.createComment(ticketId, `ADVISOR: Relaxing with debt — ${response.reason}`));
|
|
1016
|
+
// G3: Move ticket to next column (by board position)
|
|
1017
|
+
if (this.deps.moveTicketToColumn) {
|
|
1018
|
+
const nextColumn = this.findNextColumn(columnId);
|
|
1019
|
+
if (nextColumn) {
|
|
1020
|
+
const moved = await this.safeAction(ticketId, 'advisor:moveTicket', () => this.deps.moveTicketToColumn(ticketId, nextColumn.columnId, {
|
|
1021
|
+
relaxed_with_debt: true,
|
|
1022
|
+
source_column: colScope?.column.name,
|
|
1023
|
+
debt_item_count: response.debt_items?.length ?? 0,
|
|
1024
|
+
gate_waivers: waivedGates,
|
|
1025
|
+
}));
|
|
1026
|
+
if (!moved) {
|
|
1027
|
+
await this.safeAction(ticketId, 'advisor:failComment', () => this.deps.createComment(ticketId, `ADVISOR: Attempted to move ticket but the operation failed. Manual intervention needed.`));
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
return false;
|
|
1032
|
+
}
|
|
1033
|
+
case 'SPLIT_TICKET': {
|
|
1034
|
+
if (response.split_specs && response.split_specs.length > 0 && this.deps.createTickets) {
|
|
1035
|
+
try {
|
|
1036
|
+
await this.deps.createTickets(ticketId, response.split_specs);
|
|
1037
|
+
// Only archive parent after children are successfully created
|
|
1038
|
+
if (this.deps.archiveTicket) {
|
|
1039
|
+
await this.safeAction(ticketId, 'advisor:archiveTicket', () => this.deps.archiveTicket(ticketId));
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
catch (err) {
|
|
1043
|
+
// createTickets failed — do not archive the parent
|
|
1044
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1045
|
+
console.error(` [warn] SPLIT_TICKET createTickets failed for ${ticketId}: ${msg}`);
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
await this.safeAction(ticketId, 'advisor:createComment', () => this.deps.createComment(ticketId, `ADVISOR: Split ticket — ${response.reason}`));
|
|
1049
|
+
return false;
|
|
1050
|
+
}
|
|
1051
|
+
case 'ESCALATE': {
|
|
1052
|
+
const targetColId = this.cachedBoardScope?.circuit_breaker.target_column_id;
|
|
1053
|
+
if (targetColId && this.deps.moveTicketToColumn) {
|
|
1054
|
+
const moved = await this.safeAction(ticketId, 'advisor:moveTicket', () => this.deps.moveTicketToColumn(ticketId, targetColId, {
|
|
1055
|
+
escalation_reason: response.reason,
|
|
1056
|
+
source_column: colScope?.column.name,
|
|
1057
|
+
}));
|
|
1058
|
+
if (!moved) {
|
|
1059
|
+
await this.safeAction(ticketId, 'advisor:failComment', () => this.deps.createComment(ticketId, `ADVISOR: Attempted to move ticket but the operation failed. Manual intervention needed.`));
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
await this.safeAction(ticketId, 'advisor:createComment', () => this.deps.createComment(ticketId, `ADVISOR: Escalated for human review — ${response.reason}`));
|
|
1063
|
+
return false;
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
return false;
|
|
1067
|
+
}
|
|
1068
|
+
/**
|
|
1069
|
+
* Handle evaluator column verdict: parse the output, determine action, move ticket accordingly.
|
|
1070
|
+
*/
|
|
1071
|
+
async handleEvaluatorVerdict(ticketId, columnId, result, colConfig) {
|
|
1072
|
+
const verdict = parseVerdict(result.output ?? '');
|
|
1073
|
+
// Circuit breaker: if verdict parse failed, hold ticket for manual review instead of bouncing
|
|
1074
|
+
if ('parseFailed' in verdict && verdict.parseFailed) {
|
|
1075
|
+
await this.deps.createComment(ticketId, `EVALUATOR: Verdict could not be parsed — ticket held for manual review.\n\nRaw output: ${(result.output ?? '').slice(0, 500)}`).catch((e) => {
|
|
1076
|
+
console.error(` [warn] Failed to write parse-failure comment for ${ticketId}: ${e instanceof Error ? e.message : String(e)}`);
|
|
1077
|
+
});
|
|
1078
|
+
return;
|
|
1079
|
+
}
|
|
1080
|
+
const action = resolveVerdictAction(verdict);
|
|
1081
|
+
// Emit evaluator verdict event
|
|
1082
|
+
const blockers = verdict.findings.filter((f) => f.severity === 'blocker');
|
|
1083
|
+
const warnings = verdict.findings.filter((f) => f.severity === 'warning');
|
|
1084
|
+
const nits = verdict.findings.filter((f) => f.severity === 'nit');
|
|
1085
|
+
this.deps.eventEmitter?.emit({
|
|
1086
|
+
layer: 'evaluator',
|
|
1087
|
+
eventType: action === 'forward' ? 'evaluator_approved'
|
|
1088
|
+
: action === 'reject' ? 'evaluator_rejected'
|
|
1089
|
+
: 'evaluator_forwarded',
|
|
1090
|
+
severity: action === 'forward' ? 'info'
|
|
1091
|
+
: action === 'reject' ? 'failure'
|
|
1092
|
+
: 'warning',
|
|
1093
|
+
summary: `Evaluator: ${action} (${String(blockers.length)} blockers, ${String(warnings.length)} warnings, ${String(nits.length)} nits)`,
|
|
1094
|
+
detail: {
|
|
1095
|
+
decision: verdict.decision,
|
|
1096
|
+
summary: verdict.summary,
|
|
1097
|
+
findings: verdict.findings,
|
|
1098
|
+
ticket_number: this.columnScopes.get(columnId)?.tickets.find(t => t.id === ticketId)?.ticket_number ?? null,
|
|
1099
|
+
},
|
|
1100
|
+
ticketId,
|
|
1101
|
+
columnId,
|
|
1102
|
+
});
|
|
1103
|
+
switch (action) {
|
|
1104
|
+
case 'forward': {
|
|
1105
|
+
// Move to next column with verdict in handoff
|
|
1106
|
+
if (this.deps.moveTicketToColumn) {
|
|
1107
|
+
const nextColumn = this.findNextColumn(columnId);
|
|
1108
|
+
if (nextColumn) {
|
|
1109
|
+
const moved = await this.safeAction(ticketId, 'evaluator:moveTicket:forward', () => this.deps.moveTicketToColumn(ticketId, nextColumn.columnId, {
|
|
1110
|
+
evaluator_verdict: 'approved',
|
|
1111
|
+
verdict_summary: verdict.summary,
|
|
1112
|
+
}));
|
|
1113
|
+
if (!moved) {
|
|
1114
|
+
await this.safeAction(ticketId, 'evaluator:failComment', () => this.deps.createComment(ticketId, `EVALUATOR: Attempted to move ticket forward but the operation failed. Manual intervention needed.`));
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
await this.safeAction(ticketId, 'evaluator:createComment', () => this.deps.createComment(ticketId, `EVALUATOR: Approved — ${verdict.summary}`));
|
|
1119
|
+
break;
|
|
1120
|
+
}
|
|
1121
|
+
case 'reject': {
|
|
1122
|
+
// Move BACK to previous column (source column) with rejection comment
|
|
1123
|
+
if (this.deps.moveTicketToColumn) {
|
|
1124
|
+
const prevColumn = this.findPreviousColumn(columnId);
|
|
1125
|
+
if (prevColumn) {
|
|
1126
|
+
const moved = await this.safeAction(ticketId, 'evaluator:moveTicket:reject', () => this.deps.moveTicketToColumn(ticketId, prevColumn.columnId, {
|
|
1127
|
+
evaluator_verdict: 'rejected',
|
|
1128
|
+
verdict_summary: verdict.summary,
|
|
1129
|
+
findings: verdict.findings,
|
|
1130
|
+
}));
|
|
1131
|
+
if (!moved) {
|
|
1132
|
+
await this.safeAction(ticketId, 'evaluator:failComment', () => this.deps.createComment(ticketId, `EVALUATOR: Attempted to reject ticket but the move operation failed. Manual intervention needed.`));
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
const findingsText = verdict.findings
|
|
1137
|
+
.map((f) => `- [${f.severity}] ${f.description}${f.file ? ` (${f.file}${f.line ? `:${String(f.line)}` : ''})` : ''}`)
|
|
1138
|
+
.join('\n');
|
|
1139
|
+
await this.safeAction(ticketId, 'evaluator:createComment', () => this.deps.createComment(ticketId, `EVALUATOR: Rejected — ${verdict.summary}\n\n${findingsText}`));
|
|
1140
|
+
break;
|
|
1141
|
+
}
|
|
1142
|
+
case 'forward_with_signals': {
|
|
1143
|
+
// Move forward, attach findings as signals
|
|
1144
|
+
if (this.deps.moveTicketToColumn) {
|
|
1145
|
+
const nextColumn = this.findNextColumn(columnId);
|
|
1146
|
+
if (nextColumn) {
|
|
1147
|
+
const moved = await this.safeAction(ticketId, 'evaluator:moveTicket:forward_with_signals', () => this.deps.moveTicketToColumn(ticketId, nextColumn.columnId, {
|
|
1148
|
+
evaluator_verdict: 'approved_with_warnings',
|
|
1149
|
+
verdict_summary: verdict.summary,
|
|
1150
|
+
}));
|
|
1151
|
+
if (!moved) {
|
|
1152
|
+
await this.safeAction(ticketId, 'evaluator:failComment', () => this.deps.createComment(ticketId, `EVALUATOR: Attempted to move ticket forward but the operation failed. Manual intervention needed.`));
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
// Create signals for each finding
|
|
1157
|
+
for (const f of verdict.findings) {
|
|
1158
|
+
await this.safeAction(ticketId, 'evaluator:createSignal', () => this.deps.createSignal(ticketId, `EVALUATOR: [${f.severity}] ${f.description}`));
|
|
1159
|
+
}
|
|
1160
|
+
await this.safeAction(ticketId, 'evaluator:createComment', () => this.deps.createComment(ticketId, `EVALUATOR: Forwarded with ${String(verdict.findings.length)} warning(s) — ${verdict.summary}`));
|
|
1161
|
+
break;
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
/**
|
|
1166
|
+
* Find the previous pipeline column by board position (for evaluator rejections — send back to worker).
|
|
1167
|
+
* Returns null if current column is the first pipeline column.
|
|
1168
|
+
*/
|
|
1169
|
+
findPreviousColumn(currentColumnId) {
|
|
1170
|
+
const bs = this.cachedBoardScope;
|
|
1171
|
+
if (!bs)
|
|
1172
|
+
return null;
|
|
1173
|
+
const currentBoardCol = bs.columns.find((c) => c.id === currentColumnId);
|
|
1174
|
+
if (!currentBoardCol)
|
|
1175
|
+
return null;
|
|
1176
|
+
// Find previous column by position that is a pipeline column
|
|
1177
|
+
const prevBoardCol = bs.columns
|
|
1178
|
+
.filter((c) => c.position < currentBoardCol.position && this.pipelineColumns.has(c.id))
|
|
1179
|
+
.sort((a, b) => b.position - a.position)[0]; // sort descending, take first (closest)
|
|
1180
|
+
return prevBoardCol ? this.pipelineColumns.get(prevBoardCol.id) ?? null : null;
|
|
1181
|
+
}
|
|
1182
|
+
/**
|
|
1183
|
+
* Find the next pipeline column by board position (for RELAX_WITH_DEBT forward moves).
|
|
1184
|
+
* Returns null if current column is the last pipeline column.
|
|
1185
|
+
*/
|
|
1186
|
+
findNextColumn(currentColumnId) {
|
|
1187
|
+
const bs = this.cachedBoardScope;
|
|
1188
|
+
if (!bs)
|
|
1189
|
+
return null;
|
|
1190
|
+
const currentBoardCol = bs.columns.find((c) => c.id === currentColumnId);
|
|
1191
|
+
if (!currentBoardCol)
|
|
1192
|
+
return null;
|
|
1193
|
+
// Find next column by position that is a pipeline column
|
|
1194
|
+
const nextBoardCol = bs.columns
|
|
1195
|
+
.filter((c) => c.position > currentBoardCol.position && this.pipelineColumns.has(c.id))
|
|
1196
|
+
.sort((a, b) => a.position - b.position)[0];
|
|
1197
|
+
return nextBoardCol ? this.pipelineColumns.get(nextBoardCol.id) ?? null : null;
|
|
1198
|
+
}
|
|
312
1199
|
/**
|
|
313
1200
|
* Called when a loop finishes. Cleans up tracking and drains the queue.
|
|
314
1201
|
*/
|
|
315
1202
|
async onLoopComplete(ticketId, columnId, result) {
|
|
1203
|
+
// H1: Mark as completing before removing from activeLoops.
|
|
1204
|
+
// Guards the async window (advisor LLM call, replanner, etc.) against double-spawn.
|
|
1205
|
+
this.completing.add(ticketId);
|
|
316
1206
|
this.activeLoops.delete(ticketId);
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
1207
|
+
try {
|
|
1208
|
+
// Resolve ticket_number and invocation_type for event detail
|
|
1209
|
+
const colScope = this.columnScopes.get(columnId);
|
|
1210
|
+
const ticket = colScope?.tickets.find(t => t.id === ticketId);
|
|
1211
|
+
const colCfg = this.pipelineColumns.get(columnId);
|
|
1212
|
+
const invocationType = colCfg?.columnType === 'evaluator' ? 'evaluator'
|
|
1213
|
+
: colCfg?.invocationTier === 'light' ? 'light' : 'heavy';
|
|
1214
|
+
// Emit session-end event
|
|
1215
|
+
this.deps.eventEmitter?.emit({
|
|
1216
|
+
layer: 'session',
|
|
1217
|
+
eventType: result.reason === 'error' ? 'session_error' : 'session_ended',
|
|
1218
|
+
severity: result.reason === 'error' ? 'failure' : 'info',
|
|
1219
|
+
summary: `Session ended: ${result.reason} (${String(result.iterations)} iter${result.model ? `, ${result.model}` : ''})`,
|
|
1220
|
+
detail: {
|
|
1221
|
+
model: result.model ?? null,
|
|
1222
|
+
invocation_type: invocationType,
|
|
1223
|
+
duration_ms: result.durationMs ?? null,
|
|
1224
|
+
tokens_in: result.tokensIn ?? null,
|
|
1225
|
+
tokens_out: result.tokensOut ?? null,
|
|
1226
|
+
tool_call_count: result.toolCallCount ?? null,
|
|
1227
|
+
exit_reason: result.reason,
|
|
1228
|
+
gutter_count: result.gutterCount,
|
|
1229
|
+
worktree_name: colCfg?.worktreeEnabled && ticket
|
|
1230
|
+
? generateWorktreeName(ticket.ticket_number, colCfg.name) : null,
|
|
1231
|
+
ticket_number: ticket?.ticket_number ?? null,
|
|
1232
|
+
},
|
|
1233
|
+
ticketId,
|
|
1234
|
+
columnId,
|
|
1235
|
+
iteration: result.iterations,
|
|
1236
|
+
});
|
|
1237
|
+
// Emit gate events from final snapshot
|
|
1238
|
+
if (result.finalGateSnapshot && this.deps.eventEmitter) {
|
|
1239
|
+
const gateSnapshots = this.deps.gateSnapshotStore?.getRecent(ticketId, 10) ?? [];
|
|
1240
|
+
const trajectory = gateSnapshots.length > 0 ? classifyTrajectory(gateSnapshots) : null;
|
|
1241
|
+
for (const gate of result.finalGateSnapshot.results) {
|
|
1242
|
+
const history = gateSnapshots.map(s => {
|
|
1243
|
+
const gr = s.results.find(r => r.name === gate.name);
|
|
1244
|
+
return gr ? { passed: gr.passed, iteration: s.iteration } : null;
|
|
1245
|
+
}).filter(Boolean);
|
|
1246
|
+
this.deps.eventEmitter.emit({
|
|
1247
|
+
layer: 'gate',
|
|
1248
|
+
eventType: gate.passed ? 'gate_passed' : 'gate_failed',
|
|
1249
|
+
severity: gate.passed ? 'info' : 'failure',
|
|
1250
|
+
summary: `Gate "${gate.name}" ${gate.passed ? 'passed' : gate.timed_out ? 'timed out' : 'failed'} for ${ticket ? `PET-${String(ticket.ticket_number)}` : ticketId.slice(0, 8)}`,
|
|
1251
|
+
detail: {
|
|
1252
|
+
gate_name: gate.name,
|
|
1253
|
+
exit_code: gate.exit_code,
|
|
1254
|
+
duration_ms: gate.duration_ms,
|
|
1255
|
+
required: gate.required,
|
|
1256
|
+
timed_out: gate.timed_out,
|
|
1257
|
+
output: gate.output.slice(0, 2000),
|
|
1258
|
+
gate_history: history,
|
|
1259
|
+
trajectory: trajectory?.status ?? null,
|
|
1260
|
+
ticket_number: ticket?.ticket_number ?? null,
|
|
1261
|
+
},
|
|
1262
|
+
ticketId,
|
|
1263
|
+
columnId,
|
|
1264
|
+
iteration: result.iterations,
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
// Emit cost event
|
|
1269
|
+
if (this.deps.eventEmitter && this.deps.costTracker) {
|
|
1270
|
+
const tc = this.deps.costTracker.getTicketCost(ticketId);
|
|
1271
|
+
if (tc) {
|
|
1272
|
+
const totalIn = this.deps.costTracker.totalTokensIn;
|
|
1273
|
+
const totalOut = this.deps.costTracker.totalTokensOut;
|
|
1274
|
+
const maxIn = this.deps.costTracker.maxInputTokens;
|
|
1275
|
+
const pctUsed = maxIn > 0 ? Math.round(((totalIn + totalOut) / (maxIn * 2)) * 100) : 0;
|
|
1276
|
+
const severity = this.deps.costTracker.isExhausted() ? 'failure'
|
|
1277
|
+
: this.deps.costTracker.isWarning() ? 'warning' : 'info';
|
|
1278
|
+
this.deps.eventEmitter.emit({
|
|
1279
|
+
layer: 'cost',
|
|
1280
|
+
eventType: this.deps.costTracker.isExhausted() ? 'budget_exhausted'
|
|
1281
|
+
: this.deps.costTracker.isWarning() ? 'budget_warning' : 'tokens_tracked',
|
|
1282
|
+
severity,
|
|
1283
|
+
summary: `Token usage: ${String(result.tokensIn ?? 0)} in / ${String(result.tokensOut ?? 0)} out`,
|
|
1284
|
+
detail: {
|
|
1285
|
+
tokens_in: result.tokensIn ?? 0,
|
|
1286
|
+
tokens_out: result.tokensOut ?? 0,
|
|
1287
|
+
pct_used: pctUsed,
|
|
1288
|
+
max_tokens: maxIn,
|
|
1289
|
+
ticket_number: ticket?.ticket_number ?? null,
|
|
1290
|
+
},
|
|
1291
|
+
ticketId,
|
|
1292
|
+
columnId,
|
|
1293
|
+
iteration: result.iterations,
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
// Evaluator verdict handling — overrides normal loop completion
|
|
1298
|
+
const colConfig = this.pipelineColumns.get(columnId);
|
|
1299
|
+
// Shared terminal cleanup — runs for ALL exit paths including evaluator
|
|
1300
|
+
const terminalCleanup = async () => {
|
|
1301
|
+
this.advisorBudget.delete(ticketId);
|
|
1302
|
+
if (colConfig?.checkpointEnabled && this.deps.setFieldValue) {
|
|
1303
|
+
await this.safeAction(ticketId, 'clearCheckpoint', () => this.deps.setFieldValue(ticketId, 'loop_checkpoint', null));
|
|
1304
|
+
}
|
|
1305
|
+
const isTerminal = result.reason === 'moved' || result.reason === 'max_iterations' || result.reason === 'error' || result.reason === 'stalled' || result.reason === 'stopped' || result.reason === 'deleted';
|
|
1306
|
+
if (isTerminal && colConfig?.worktreeOnDone === 'cleanup' && this.deps.cleanupWorktree) {
|
|
1307
|
+
const colScope = this.columnScopes.get(columnId);
|
|
1308
|
+
const ticket = colScope?.tickets.find(t => t.id === ticketId);
|
|
1309
|
+
if (ticket) {
|
|
1310
|
+
const worktreeName = generateWorktreeName(ticket.ticket_number, colConfig.name);
|
|
1311
|
+
void this.deps.cleanupWorktree(worktreeName).then((success) => {
|
|
1312
|
+
if (!success) {
|
|
1313
|
+
console.error(` [warn] Worktree cleanup failed for ${worktreeName} — may need manual removal`);
|
|
1314
|
+
}
|
|
1315
|
+
}).catch((err) => {
|
|
1316
|
+
console.error(` [warn] Worktree cleanup error for ${worktreeName}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1317
|
+
});
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
this.deps.gateSnapshotStore?.clear(ticketId);
|
|
1321
|
+
};
|
|
1322
|
+
if (colConfig?.columnType === 'evaluator') {
|
|
1323
|
+
await this.handleEvaluatorVerdict(ticketId, columnId, result, colConfig);
|
|
1324
|
+
await this.drainQueue(columnId);
|
|
1325
|
+
await terminalCleanup();
|
|
1326
|
+
return;
|
|
1327
|
+
}
|
|
1328
|
+
// Advisor recovery for failure exits — runs before drain
|
|
1329
|
+
// so the retry slot isn't filled by a queued ticket
|
|
1330
|
+
const isFailure = result.reason === 'stalled' || result.reason === 'error' || result.reason === 'max_iterations';
|
|
1331
|
+
if (isFailure && colConfig) {
|
|
1332
|
+
const retried = await this.invokeAdvisorRecovery(ticketId, columnId, result, colConfig);
|
|
1333
|
+
if (retried)
|
|
1334
|
+
return; // Advisor re-spawned the loop — skip drain + default behavior
|
|
1335
|
+
}
|
|
1336
|
+
// Drain queue to minimize slot idle time (H13) — after advisor to preserve retry slot
|
|
1337
|
+
await this.drainQueue(columnId);
|
|
1338
|
+
// Check replanner triggers after advisor recovery, before comments
|
|
1339
|
+
if (this.deps.invokeReplanner && !this.pipelinePaused) {
|
|
1340
|
+
const state = this.buildPipelineState();
|
|
1341
|
+
const triggers = {
|
|
1342
|
+
escalation_count: 3,
|
|
1343
|
+
cost_threshold_pct: 75,
|
|
1344
|
+
repeated_gate_failure_count: 3,
|
|
1345
|
+
duration_threshold_minutes: 480,
|
|
1346
|
+
};
|
|
1347
|
+
if (shouldFireReplanner(triggers, state) && this.replannerInvocations < this.replannerMaxInvocations) {
|
|
1348
|
+
this.replannerInvocations++;
|
|
1349
|
+
try {
|
|
1350
|
+
// Build ticket summaries from cached column scopes
|
|
1351
|
+
const ticketSummaries = [];
|
|
1352
|
+
for (const [colId, scope] of this.columnScopes) {
|
|
1353
|
+
const colConfig = this.pipelineColumns.get(colId);
|
|
1354
|
+
const columnName = colConfig?.name ?? colId;
|
|
1355
|
+
const colQueue = this.loopQueues.get(colId) ?? [];
|
|
1356
|
+
for (const ticket of scope.tickets) {
|
|
1357
|
+
const isActive = this.activeLoops.has(ticket.id);
|
|
1358
|
+
const isQueued = colQueue.includes(ticket.id);
|
|
1359
|
+
const status = isActive ? 'active' : isQueued ? 'queued' : 'idle';
|
|
1360
|
+
// Get gate pass rate from snapshot store if available
|
|
1361
|
+
let gatePassRate = 0;
|
|
1362
|
+
if (this.deps.gateSnapshotStore) {
|
|
1363
|
+
const latest = this.deps.gateSnapshotStore.getLatest(ticket.id);
|
|
1364
|
+
if (latest) {
|
|
1365
|
+
const total = latest.results.length;
|
|
1366
|
+
const passed = latest.results.filter(r => r.passed).length;
|
|
1367
|
+
gatePassRate = total > 0 ? passed / total : 0;
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
ticketSummaries.push({
|
|
1371
|
+
id: ticket.id,
|
|
1372
|
+
title: ticket.title,
|
|
1373
|
+
column: columnName,
|
|
1374
|
+
status,
|
|
1375
|
+
iterations: 0, // Not tracked at orchestrator level
|
|
1376
|
+
gatePassRate,
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
const triggerReason = this.identifyTriggerReason(triggers, state);
|
|
1381
|
+
const response = await this.deps.invokeReplanner({
|
|
1382
|
+
...state,
|
|
1383
|
+
ticketSummaries,
|
|
1384
|
+
triggerReason,
|
|
1385
|
+
});
|
|
1386
|
+
// Emit replanner event
|
|
1387
|
+
this.deps.eventEmitter?.emit({
|
|
1388
|
+
layer: 'replanner',
|
|
1389
|
+
eventType: `replanner_${response.action.toLowerCase()}`,
|
|
1390
|
+
severity: response.action === 'CONTINUE' ? 'info'
|
|
1391
|
+
: response.action === 'PAUSE_PIPELINE' ? 'warning' : 'info',
|
|
1392
|
+
summary: `Replanner: ${response.action} — ${response.reason.slice(0, 200)}`,
|
|
1393
|
+
detail: {
|
|
1394
|
+
action: response.action,
|
|
1395
|
+
reason: response.reason,
|
|
1396
|
+
},
|
|
1397
|
+
});
|
|
1398
|
+
await this.executeReplannerAction(response);
|
|
1399
|
+
// Only count non-CONTINUE responses toward the auto-pause limit
|
|
1400
|
+
if (response.action === 'CONTINUE') {
|
|
1401
|
+
this.replannerInvocations--; // Don't count CONTINUE toward limit
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
catch (err) {
|
|
1405
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1406
|
+
console.error(` [replanner] Failed: ${msg} — pausing pipeline for safety`);
|
|
1407
|
+
this.pipelinePaused = true;
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
if (this.replannerInvocations >= this.replannerMaxInvocations) {
|
|
1411
|
+
this.pipelinePaused = true;
|
|
1412
|
+
console.error('Replanner fired 3 non-CONTINUE actions — auto-pausing pipeline');
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
// Write reason-specific comment
|
|
1416
|
+
let comment;
|
|
1417
|
+
switch (result.reason) {
|
|
1418
|
+
case 'moved':
|
|
1419
|
+
comment = `Pipeline agent advanced ticket after ${result.iterations} iteration(s).`;
|
|
1420
|
+
// Append loop success to run memory
|
|
1421
|
+
if (this.deps.appendRunMemory) {
|
|
1422
|
+
await this.safeAction(ticketId, 'appendRunMemory', () => this.deps.appendRunMemory('Discovered Interfaces', `- Ticket ${ticketId} completed in ${colConfig?.name ?? columnId} after ${String(result.iterations)} iteration(s)`));
|
|
1423
|
+
}
|
|
1424
|
+
break;
|
|
1425
|
+
case 'max_iterations':
|
|
1426
|
+
comment = `Pipeline agent reached iteration limit (${result.iterations}) without advancing. Manual review needed.`;
|
|
1427
|
+
break;
|
|
1428
|
+
case 'stalled':
|
|
1429
|
+
comment = `Pipeline agent stalled — no progress for ${result.gutterCount} consecutive iterations (of ${result.iterations} total). Manual review needed.`;
|
|
1430
|
+
break;
|
|
1431
|
+
case 'error':
|
|
1432
|
+
comment = `Pipeline agent encountered an error after ${result.iterations} iteration(s): ${result.lastError ?? 'unknown error'}`;
|
|
1433
|
+
break;
|
|
1434
|
+
case 'stopped':
|
|
1435
|
+
comment = `Pipeline agent was stopped externally after ${result.iterations} iteration(s).`;
|
|
1436
|
+
break;
|
|
1437
|
+
case 'deleted':
|
|
1438
|
+
comment = `Pipeline agent stopped — ticket was deleted or archived during iteration ${result.iterations}.`;
|
|
1439
|
+
break;
|
|
1440
|
+
}
|
|
1441
|
+
if (result.reason !== 'deleted') {
|
|
1442
|
+
await this.deps.createComment(ticketId, comment).catch((err) => {
|
|
1443
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1444
|
+
console.error(` [warn] Failed to write completion comment for ${ticketId}: ${msg}`);
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
// Create signal for failure exits so future agents inherit the context
|
|
1448
|
+
if (result.reason === 'stalled') {
|
|
1449
|
+
await this.deps.createSignal(ticketId, `Previous pipeline run stalled after ${result.iterations} iterations with no progress. Review comments for details before retrying.`).catch((err) => {
|
|
1450
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1451
|
+
console.error(` [warn] Failed to write signal for ${ticketId}: ${msg}`);
|
|
1452
|
+
});
|
|
1453
|
+
}
|
|
1454
|
+
else if (result.reason === 'error') {
|
|
1455
|
+
await this.deps.createSignal(ticketId, `Previous pipeline run failed after ${result.iterations} iteration(s): ${result.lastError ?? 'unknown error'}. Review comments for details before retrying.`).catch((err) => {
|
|
1456
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1457
|
+
console.error(` [warn] Failed to write signal for ${ticketId}: ${msg}`);
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
// Shared terminal cleanup — advisor budget, checkpoint, worktree, snapshot memory
|
|
1461
|
+
await terminalCleanup();
|
|
1462
|
+
}
|
|
1463
|
+
finally {
|
|
1464
|
+
this.completing.delete(ticketId);
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
/**
|
|
1468
|
+
* Build PipelineState from orchestrator internals for replanner evaluation.
|
|
1469
|
+
*/
|
|
1470
|
+
buildPipelineState() {
|
|
1471
|
+
const costTracker = this.deps.costTracker;
|
|
1472
|
+
const totalTokensIn = costTracker ? costTracker.totalTokensIn : 0;
|
|
1473
|
+
const maxInputTokens = costTracker ? costTracker.maxInputTokens : 5_000_000;
|
|
1474
|
+
// Count escalated tickets — those moved to circuit breaker target column
|
|
1475
|
+
let escalatedTickets = 0;
|
|
1476
|
+
const targetColId = this.cachedBoardScope?.circuit_breaker.target_column_id;
|
|
1477
|
+
if (targetColId) {
|
|
1478
|
+
const targetCol = this.cachedBoardScope?.columns.find((c) => c.id === targetColId);
|
|
1479
|
+
if (targetCol)
|
|
1480
|
+
escalatedTickets = targetCol.ticket_count;
|
|
1481
|
+
}
|
|
1482
|
+
// Compute repeated gate failures from snapshot store
|
|
1483
|
+
const repeatedGateFailures = {};
|
|
1484
|
+
if (this.deps.gateSnapshotStore) {
|
|
1485
|
+
const ticketIds = this.deps.gateSnapshotStore.getAllTicketIds();
|
|
1486
|
+
for (const tid of ticketIds) {
|
|
1487
|
+
const latest = this.deps.gateSnapshotStore.getLatest(tid);
|
|
1488
|
+
if (latest) {
|
|
1489
|
+
for (const r of latest.results) {
|
|
1490
|
+
if (!r.passed) {
|
|
1491
|
+
repeatedGateFailures[r.name] = (repeatedGateFailures[r.name] ?? 0) + 1;
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
const durationMinutes = (Date.now() - this.pipelineStartTime) / 60_000;
|
|
1498
|
+
return { escalatedTickets, totalTokensIn, maxInputTokens, repeatedGateFailures, durationMinutes };
|
|
1499
|
+
}
|
|
1500
|
+
/**
|
|
1501
|
+
* Identify which trigger condition caused the replanner to fire.
|
|
1502
|
+
*/
|
|
1503
|
+
identifyTriggerReason(triggers, state) {
|
|
1504
|
+
const reasons = [];
|
|
1505
|
+
if (state.escalatedTickets >= triggers.escalation_count) {
|
|
1506
|
+
reasons.push(`${String(state.escalatedTickets)} tickets escalated (threshold: ${String(triggers.escalation_count)})`);
|
|
1507
|
+
}
|
|
1508
|
+
if (state.maxInputTokens > 0) {
|
|
1509
|
+
const pct = (state.totalTokensIn / state.maxInputTokens) * 100;
|
|
1510
|
+
if (pct >= triggers.cost_threshold_pct) {
|
|
1511
|
+
reasons.push(`Token usage at ${pct.toFixed(0)}% (threshold: ${String(triggers.cost_threshold_pct)}%)`);
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
for (const [gate, count] of Object.entries(state.repeatedGateFailures)) {
|
|
1515
|
+
if (count >= triggers.repeated_gate_failure_count) {
|
|
1516
|
+
reasons.push(`Gate "${gate}" failing on ${String(count)} ticket(s) (threshold: ${String(triggers.repeated_gate_failure_count)})`);
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
if (state.durationMinutes >= triggers.duration_threshold_minutes) {
|
|
1520
|
+
reasons.push(`Pipeline running for ${state.durationMinutes.toFixed(0)}min (threshold: ${String(triggers.duration_threshold_minutes)}min)`);
|
|
1521
|
+
}
|
|
1522
|
+
return reasons.length > 0 ? reasons.join('; ') : 'Unknown trigger';
|
|
1523
|
+
}
|
|
1524
|
+
/**
|
|
1525
|
+
* Execute the action returned by the replanner.
|
|
1526
|
+
*/
|
|
1527
|
+
async executeReplannerAction(response) {
|
|
1528
|
+
switch (response.action) {
|
|
1529
|
+
case 'CONTINUE':
|
|
324
1530
|
break;
|
|
325
|
-
case '
|
|
326
|
-
|
|
1531
|
+
case 'PAUSE_PIPELINE':
|
|
1532
|
+
this.pipelinePaused = true;
|
|
327
1533
|
break;
|
|
328
|
-
case '
|
|
329
|
-
|
|
1534
|
+
case 'ARCHIVE_TICKETS':
|
|
1535
|
+
if (response.ticket_ids && this.deps.archiveTicket) {
|
|
1536
|
+
for (const id of response.ticket_ids) {
|
|
1537
|
+
await this.safeAction(id, 'replanner:archiveTicket', () => this.deps.archiveTicket(id));
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
330
1540
|
break;
|
|
331
|
-
case '
|
|
332
|
-
|
|
1541
|
+
case 'CREATE_SIGNAL':
|
|
1542
|
+
if (response.signal_content && this.deps.createColumnSignal) {
|
|
1543
|
+
for (const colId of this.pipelineColumns.keys()) {
|
|
1544
|
+
await this.safeAction(colId, 'replanner:createColumnSignal', () => this.deps.createColumnSignal(colId, response.signal_content));
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
333
1547
|
break;
|
|
334
|
-
case '
|
|
335
|
-
|
|
1548
|
+
case 'ESCALATE_ALL':
|
|
1549
|
+
this.pipelinePaused = true;
|
|
336
1550
|
break;
|
|
337
|
-
case '
|
|
338
|
-
|
|
1551
|
+
case 'ADJUST_BUDGET':
|
|
1552
|
+
console.error(' [replanner] ADJUST_BUDGET requested but not yet implemented — treating as CONTINUE');
|
|
339
1553
|
break;
|
|
340
1554
|
}
|
|
341
|
-
if (result.reason !== 'deleted') {
|
|
342
|
-
await this.deps.createComment(ticketId, comment).catch((err) => {
|
|
343
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
344
|
-
console.error(` [warn] Failed to write completion comment for ${ticketId}: ${msg}`);
|
|
345
|
-
});
|
|
346
|
-
}
|
|
347
|
-
// Create signal for failure exits so future agents inherit the context
|
|
348
|
-
if (result.reason === 'stalled') {
|
|
349
|
-
await this.deps.createSignal(ticketId, `Previous pipeline run stalled after ${result.iterations} iterations with no progress. Review comments for details before retrying.`).catch((err) => {
|
|
350
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
351
|
-
console.error(` [warn] Failed to write signal for ${ticketId}: ${msg}`);
|
|
352
|
-
});
|
|
353
|
-
}
|
|
354
|
-
else if (result.reason === 'error') {
|
|
355
|
-
await this.deps.createSignal(ticketId, `Previous pipeline run failed after ${result.iterations} iteration(s): ${result.lastError ?? 'unknown error'}. Review comments for details before retrying.`).catch((err) => {
|
|
356
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
357
|
-
console.error(` [warn] Failed to write signal for ${ticketId}: ${msg}`);
|
|
358
|
-
});
|
|
359
|
-
}
|
|
360
1555
|
}
|
|
361
1556
|
/**
|
|
362
1557
|
* Start queued tickets for a column until concurrency limit is reached (H16: fills multiple slots).
|
|
@@ -368,6 +1563,11 @@ export class PipelineOrchestrator {
|
|
|
368
1563
|
const queue = this.loopQueues.get(columnId);
|
|
369
1564
|
if (!queue || queue.length === 0)
|
|
370
1565
|
return;
|
|
1566
|
+
// Check firing constraints before draining — if column is now blocked, stop
|
|
1567
|
+
if (this.isColumnBlocked(columnId)) {
|
|
1568
|
+
console.error(` [drain] Column ${columnId}: BLOCKED by firing constraints — ${String(queue.length)} ticket(s) remain queued`);
|
|
1569
|
+
return;
|
|
1570
|
+
}
|
|
371
1571
|
// Loop to fill all available slots (H16)
|
|
372
1572
|
while (queue.length > 0) {
|
|
373
1573
|
const activeInColumn = this.activeCountForColumn(columnId);
|
|
@@ -389,8 +1589,14 @@ export class PipelineOrchestrator {
|
|
|
389
1589
|
continue; // try next ticket in queue
|
|
390
1590
|
}
|
|
391
1591
|
}
|
|
392
|
-
catch {
|
|
393
|
-
// If check fails,
|
|
1592
|
+
catch (err) {
|
|
1593
|
+
// If check fails, defer rather than proceeding with potentially blocked ticket
|
|
1594
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1595
|
+
console.error(` [warn] drainQueue blocker check failed for ${nextTicketId}: ${msg} — deferring`);
|
|
1596
|
+
this.deferredTickets.set(nextTicketId, columnId);
|
|
1597
|
+
this.spawning.delete(nextTicketId);
|
|
1598
|
+
this.releaseSlot(columnId);
|
|
1599
|
+
continue;
|
|
394
1600
|
}
|
|
395
1601
|
try {
|
|
396
1602
|
await this.deps.claimTicket(nextTicketId);
|
|
@@ -398,7 +1604,8 @@ export class PipelineOrchestrator {
|
|
|
398
1604
|
}
|
|
399
1605
|
catch (err) {
|
|
400
1606
|
const msg = err instanceof Error ? err.message : String(err);
|
|
401
|
-
console.error(` [error] Failed to claim queued ticket ${nextTicketId}: ${msg}`);
|
|
1607
|
+
console.error(` [error] Failed to claim/start queued ticket ${nextTicketId}: ${msg}`);
|
|
1608
|
+
this.knownTickets.delete(nextTicketId);
|
|
402
1609
|
}
|
|
403
1610
|
finally {
|
|
404
1611
|
this.spawning.delete(nextTicketId);
|