@cat-factory/orchestration 0.176.0 → 0.177.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/execution/ExecutionService.d.ts +9 -62
- package/dist/modules/execution/ExecutionService.d.ts.map +1 -1
- package/dist/modules/execution/ExecutionService.js +46 -320
- package/dist/modules/execution/ExecutionService.js.map +1 -1
- package/dist/modules/execution/OneShotStepController.d.ts +92 -0
- package/dist/modules/execution/OneShotStepController.d.ts.map +1 -0
- package/dist/modules/execution/OneShotStepController.js +202 -0
- package/dist/modules/execution/OneShotStepController.js.map +1 -0
- package/dist/modules/execution/PollRunningController.d.ts +113 -0
- package/dist/modules/execution/PollRunningController.d.ts.map +1 -0
- package/dist/modules/execution/PollRunningController.js +298 -0
- package/dist/modules/execution/PollRunningController.js.map +1 -0
- package/dist/modules/execution/RunDispatcher.d.ts +4 -108
- package/dist/modules/execution/RunDispatcher.d.ts.map +1 -1
- package/dist/modules/execution/RunDispatcher.js +51 -468
- package/dist/modules/execution/RunDispatcher.js.map +1 -1
- package/dist/modules/execution/StepDecisionController.d.ts +120 -0
- package/dist/modules/execution/StepDecisionController.d.ts.map +1 -0
- package/dist/modules/execution/StepDecisionController.js +347 -0
- package/dist/modules/execution/StepDecisionController.js.map +1 -0
- package/dist/modules/observability/LlmObservabilityService.d.ts +5 -2
- package/dist/modules/observability/LlmObservabilityService.d.ts.map +1 -1
- package/dist/modules/observability/LlmObservabilityService.js +6 -3
- package/dist/modules/observability/LlmObservabilityService.js.map +1 -1
- package/package.json +8 -8
|
@@ -1,14 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getErrorMessage, isAsyncAgentExecutor, noopLogger, parseLocalModelId, runBestEffort, RunContendedError, } from '@cat-factory/kernel';
|
|
2
2
|
import { parseBlueprintService, parseSpecDoc } from '@cat-factory/contracts';
|
|
3
|
-
import { applyContainerRunning,
|
|
3
|
+
import { applyContainerRunning, applySubtaskProgress, recordDispatchAttribution, } from './step-fold.logic.js';
|
|
4
4
|
import { applyValidationReport } from './validation.logic.js';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { commitInitiativeTracker, FORK_PROPOSER_KIND, PR_REVIEWER_KIND, resolvePrNumber, } from '@cat-factory/agents';
|
|
5
|
+
import { recordReproductionOutcome } from './reproductionProof.logic.js';
|
|
6
|
+
import { FORK_PROPOSER_KIND, PR_REVIEWER_KIND, resolvePrNumber } from '@cat-factory/agents';
|
|
8
7
|
import { isDeployStep } from '@cat-factory/integrations';
|
|
9
8
|
import { reviewableArtifactOutput } from './artifact-review.logic.js';
|
|
10
|
-
import {
|
|
11
|
-
import { classifyDispatchFailure
|
|
9
|
+
import { HUMAN_TEST_AGENT_KIND } from './ci.logic.js';
|
|
10
|
+
import { classifyDispatchFailure } from './job.logic.js';
|
|
12
11
|
import { AgentContextBuilder } from './AgentContextBuilder.js';
|
|
13
12
|
import { DeployerStepController } from './DeployerStepController.js';
|
|
14
13
|
import { FollowUpGateController } from './FollowUpGateController.js';
|
|
@@ -26,6 +25,8 @@ import { PrReviewController } from './PrReviewController.js';
|
|
|
26
25
|
import { initialPrReviewState } from './prReview.logic.js';
|
|
27
26
|
import { PrReviewResolutionController } from './PrReviewResolutionController.js';
|
|
28
27
|
import { PollCompletionController } from './PollCompletionController.js';
|
|
28
|
+
import { PollRunningController } from './PollRunningController.js';
|
|
29
|
+
import { OneShotStepController } from './OneShotStepController.js';
|
|
29
30
|
import { DEFAULT_FORK_MAX_CHAT_TURNS, resolveForkTriState, shouldProposeForkAuto, } from './forkDecision.logic.js';
|
|
30
31
|
import { RunStateMachine } from './RunStateMachine.js';
|
|
31
32
|
import { StepGraph } from './StepGraph.js';
|
|
@@ -107,6 +108,10 @@ export class RunDispatcher {
|
|
|
107
108
|
prReviewResolution;
|
|
108
109
|
/** Settled-agent-poll completion (helper-phase branches + `failed` handling), extracted collaborator. */
|
|
109
110
|
pollCompletion;
|
|
111
|
+
/** The RUNNING half of the poll branch tree — the sibling of {@link pollCompletion}. */
|
|
112
|
+
pollRunning;
|
|
113
|
+
/** The one-shot engine steps (`tracker` / `bug-intake` / `initiative-committer`). */
|
|
114
|
+
oneShot;
|
|
110
115
|
/**
|
|
111
116
|
* The Follow-up companion gate (the future-looking Coder's streamed items, the
|
|
112
117
|
* park-until-decided gate, and the human-action API), extracted to
|
|
@@ -186,7 +191,7 @@ export class RunDispatcher {
|
|
|
186
191
|
recordStepResult: (ws, instance, step, isFinalStep, result) => this.recordStepResult(ws, instance, step, isFinalStep, result),
|
|
187
192
|
applyContainerRunning: (step, update) => applyContainerRunning(step, update),
|
|
188
193
|
applySubtaskProgress: (step, counts) => applySubtaskProgress(step, counts),
|
|
189
|
-
recoverContainerEviction: (ws, instance, step, failure, onBeforeRedispatch) => this.recoverContainerEviction(ws, instance, step, failure, onBeforeRedispatch),
|
|
194
|
+
recoverContainerEviction: (ws, instance, step, failure, onBeforeRedispatch) => this.pollRunning.recoverContainerEviction(ws, instance, step, failure, onBeforeRedispatch),
|
|
190
195
|
logger: deps.logger,
|
|
191
196
|
});
|
|
192
197
|
this.followUpGate = new FollowUpGateController({
|
|
@@ -216,6 +221,31 @@ export class RunDispatcher {
|
|
|
216
221
|
recordStepResult: (ws, instance, step, isFinalStep, result) => this.recordStepResult(ws, instance, step, isFinalStep, result),
|
|
217
222
|
handleAgentStep: (ctx, dispatchKind, augment) => this.handleAgentStep(ctx, dispatchKind, augment),
|
|
218
223
|
});
|
|
224
|
+
this.pollRunning = new PollRunningController({
|
|
225
|
+
blockRepository: deps.blockRepository,
|
|
226
|
+
clock: deps.clock,
|
|
227
|
+
runStateMachine: deps.runStateMachine,
|
|
228
|
+
deployer: this.deployer,
|
|
229
|
+
followUpGate: this.followUpGate,
|
|
230
|
+
runInitiatorScope: this.runInitiatorScope,
|
|
231
|
+
gateFor: (agentKind) => this.gateFor(agentKind),
|
|
232
|
+
recordStepResult: (ws, instance, step, isFinalStep, result) => this.recordStepResult(ws, instance, step, isFinalStep, result),
|
|
233
|
+
recordBackendDiagnostics: (instance, backend) => this.recordBackendDiagnostics(instance, backend),
|
|
234
|
+
});
|
|
235
|
+
this.oneShot = new OneShotStepController({
|
|
236
|
+
blockRepository: deps.blockRepository,
|
|
237
|
+
clock: deps.clock,
|
|
238
|
+
contextBuilder: deps.contextBuilder,
|
|
239
|
+
log: this.log,
|
|
240
|
+
repoOps: this.repoOps,
|
|
241
|
+
runStateMachine: deps.runStateMachine,
|
|
242
|
+
stepGraph: deps.stepGraph,
|
|
243
|
+
...(deps.bugIntakeService ? { bugIntakeService: deps.bugIntakeService } : {}),
|
|
244
|
+
...(deps.initiativeService ? { initiativeService: deps.initiativeService } : {}),
|
|
245
|
+
...(deps.issueWriteback ? { issueWriteback: deps.issueWriteback } : {}),
|
|
246
|
+
...(deps.ticketTrackerProvider ? { ticketTrackerProvider: deps.ticketTrackerProvider } : {}),
|
|
247
|
+
recordStepResult: (ws, instance, step, isFinalStep, result) => this.recordStepResult(ws, instance, step, isFinalStep, result),
|
|
248
|
+
});
|
|
219
249
|
this.pollCompletion = new PollCompletionController({
|
|
220
250
|
blockRepository: deps.blockRepository,
|
|
221
251
|
clock: deps.clock,
|
|
@@ -225,8 +255,8 @@ export class RunDispatcher {
|
|
|
225
255
|
visualConfirmationController: deps.visualConfirmationController,
|
|
226
256
|
prReviewController: deps.prReviewController,
|
|
227
257
|
recordBackendDiagnostics: (instance, backend) => this.recordBackendDiagnostics(instance, backend),
|
|
228
|
-
recoverContainerEviction: (ws, instance, step, failure) => this.recoverContainerEviction(ws, instance, step, failure),
|
|
229
|
-
markContainerErrored: (ws, instance, step) => this.markContainerErrored(ws, instance, step),
|
|
258
|
+
recoverContainerEviction: (ws, instance, step, failure) => this.pollRunning.recoverContainerEviction(ws, instance, step, failure),
|
|
259
|
+
markContainerErrored: (ws, instance, step) => this.pollRunning.markContainerErrored(ws, instance, step),
|
|
230
260
|
});
|
|
231
261
|
this.gateStepController = new GateStepController({
|
|
232
262
|
agentExecutor: deps.agentExecutor,
|
|
@@ -284,9 +314,9 @@ export class RunDispatcher {
|
|
|
284
314
|
architectureBrainstormKind: deps.architectureBrainstormKind,
|
|
285
315
|
interviewControllers: this.interviewControllers,
|
|
286
316
|
recordStepResult: (ws, instance, step, isFinalStep, result) => this.recordStepResult(ws, instance, step, isFinalStep, result),
|
|
287
|
-
runTracker: (ws, instance, block) => this.runTracker(ws, instance, block),
|
|
288
|
-
runBugIntake: (ws, instance, step, block, isFinalStep) => this.runBugIntake(ws, instance, step, block, isFinalStep),
|
|
289
|
-
runInitiativeCommitter: (ws, block) => this.runInitiativeCommitter(ws, block),
|
|
317
|
+
runTracker: (ws, instance, block) => this.oneShot.runTracker(ws, instance, block),
|
|
318
|
+
runBugIntake: (ws, instance, step, block, isFinalStep) => this.oneShot.runBugIntake(ws, instance, step, block, isFinalStep),
|
|
319
|
+
runInitiativeCommitter: (ws, block) => this.oneShot.runInitiativeCommitter(ws, block),
|
|
290
320
|
evaluateGate: (ws, instance, step, block, isFinalStep, gate) => this.evaluateGate(ws, instance, step, block, isFinalStep, gate),
|
|
291
321
|
gateFor: (kind) => this.gateFor(kind),
|
|
292
322
|
evaluateJudge: (ws, instance, step, block, isFinalStep, judge) => this.judgeController.evaluate(ws, instance, step, block, isFinalStep, judge),
|
|
@@ -640,7 +670,7 @@ export class RunDispatcher {
|
|
|
640
670
|
initiatedByUserId: step.initiatedByUserId,
|
|
641
671
|
});
|
|
642
672
|
if (update.state === 'running') {
|
|
643
|
-
return this.handleRunningPoll(workspaceId, executionId, instance, update, step.jobId);
|
|
673
|
+
return this.pollRunning.handleRunningPoll(workspaceId, executionId, instance, update, step.jobId);
|
|
644
674
|
}
|
|
645
675
|
// A gate whose helper INVESTIGATES instead of fixing (post-release-health → on-call)
|
|
646
676
|
// declares a `resolveHelperCompletion` hook on its definition. When such a helper's job
|
|
@@ -649,7 +679,7 @@ export class RunDispatcher {
|
|
|
649
679
|
// budget) and finish the gate step with the output it returns. The gate raises its own
|
|
650
680
|
// `release_regression` notification + enriches any open incident inside the hook (from the
|
|
651
681
|
// signals stashed at escalation); the run then completes for a human to act out-of-band.
|
|
652
|
-
const investigated = await this.resolveInvestigateHelperCompletion(workspaceId, instance, step, update);
|
|
682
|
+
const investigated = await this.pollRunning.resolveInvestigateHelperCompletion(workspaceId, instance, step, update);
|
|
653
683
|
if (investigated)
|
|
654
684
|
return investigated;
|
|
655
685
|
// A polling gate step's in-flight job is its helper agent (ci-fixer /
|
|
@@ -660,7 +690,12 @@ export class RunDispatcher {
|
|
|
660
690
|
// negative, so the next check re-dispatches (until the attempt budget is spent).
|
|
661
691
|
const reprobeGate = this.gateFor(step.agentKind);
|
|
662
692
|
if (reprobeGate) {
|
|
663
|
-
return this.reprobeGateAfterHelper(reprobeGate, {
|
|
693
|
+
return this.pollRunning.reprobeGateAfterHelper(reprobeGate, {
|
|
694
|
+
workspaceId,
|
|
695
|
+
instance,
|
|
696
|
+
step,
|
|
697
|
+
update,
|
|
698
|
+
});
|
|
664
699
|
}
|
|
665
700
|
// A helper job (Fixer / conflict-resolver) in flight for a tester / human-test /
|
|
666
701
|
// visual-confirmation gate is NOT the step's own work: settle that round and re-park/re-dispatch
|
|
@@ -683,277 +718,6 @@ export class RunDispatcher {
|
|
|
683
718
|
step.jobId = undefined;
|
|
684
719
|
return this.recordStepResult(workspaceId, instance, step, isFinalStep, update.result);
|
|
685
720
|
}
|
|
686
|
-
/**
|
|
687
|
-
* Handle a `running` poll: a successful poll proves the container is up, so surface live subtask
|
|
688
|
-
* progress (e.g. 3/8 todos) without advancing the step. Only persist + emit when something
|
|
689
|
-
* actually changed so an idle poll doesn't churn storage or the event stream. Folds the poll's
|
|
690
|
-
* delta via {@link applyRunningFold} — a cheap pre-check against the loaded snapshot, then the
|
|
691
|
-
* authoritative re-apply on fresh state under CAS (idempotent for the set-to-latest folds and
|
|
692
|
-
* correct for the drain-on-read follow-up append). Split from {@link pollAgentJobInner} to stay
|
|
693
|
-
* under the statement ceiling.
|
|
694
|
-
*/
|
|
695
|
-
async handleRunningPoll(workspaceId, executionId, instance, update, jobId) {
|
|
696
|
-
const foldCtx = { jobId, update, workspaceId };
|
|
697
|
-
// Cheap pre-check against the loaded snapshot: skip the write entirely on an idle poll
|
|
698
|
-
// (the common case). The mutation is discarded — the authoritative write re-applies the
|
|
699
|
-
// same fold on fresh state under CAS below.
|
|
700
|
-
if (await this.applyRunningFold(instance, foldCtx)) {
|
|
701
|
-
try {
|
|
702
|
-
const persisted = await this.runStateMachine.mutateInstance(workspaceId, executionId, async (fresh) => {
|
|
703
|
-
await this.applyRunningFold(fresh, foldCtx);
|
|
704
|
-
});
|
|
705
|
-
// Progress-only fold (subtask ticks / streamed follow-ups): skip the per-run
|
|
706
|
-
// LLM-metrics GROUP BY so a live container's poll cadence doesn't re-aggregate
|
|
707
|
-
// the run on every tick. The rollup refreshes on the step-boundary/terminal emit.
|
|
708
|
-
await this.runStateMachine.emitInstance(workspaceId, persisted, { rollUpMetrics: false });
|
|
709
|
-
}
|
|
710
|
-
catch (error) {
|
|
711
|
-
// The run was cancelled/removed mid-poll (`NotFoundError`) or stayed hot-contended
|
|
712
|
-
// past the retry budget (`ConflictError`) — re-drive on fresh state rather than
|
|
713
|
-
// failing the run; the next entry no-ops on a gone/terminal run.
|
|
714
|
-
if (error instanceof NotFoundError || error instanceof ConflictError) {
|
|
715
|
-
throw new RunContendedError(executionId);
|
|
716
|
-
}
|
|
717
|
-
throw error;
|
|
718
|
-
}
|
|
719
|
-
}
|
|
720
|
-
return { kind: 'awaiting_job', jobId, stepIndex: instance.currentStep };
|
|
721
|
-
}
|
|
722
|
-
/**
|
|
723
|
-
* A gate whose helper INVESTIGATES instead of fixing (post-release-health → on-call) declares a
|
|
724
|
-
* `resolveHelperCompletion` hook. When such a helper's job settles — done OR failed — call the
|
|
725
|
-
* hook INSTEAD of re-probing the precheck (re-probing an investigate-don't-fix helper would just
|
|
726
|
-
* regress again and burn the budget) and finish the gate step with the output it returns. Returns
|
|
727
|
-
* the resulting {@link AdvanceResult}, or `null` when this branch doesn't apply (the caller falls
|
|
728
|
-
* through to the re-probe / other completion paths).
|
|
729
|
-
*/
|
|
730
|
-
async resolveInvestigateHelperCompletion(workspaceId, instance, step, update) {
|
|
731
|
-
const completionGate = this.gateFor(step.agentKind);
|
|
732
|
-
if (completionGate?.resolveHelperCompletion &&
|
|
733
|
-
step.gate?.phase === 'working' &&
|
|
734
|
-
(update.state === 'done' || update.state === 'failed')) {
|
|
735
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
736
|
-
step.jobId = undefined;
|
|
737
|
-
step.subtasks = undefined;
|
|
738
|
-
if (!block)
|
|
739
|
-
return { kind: 'noop' };
|
|
740
|
-
const isFinalStep = instance.currentStep === instance.steps.length - 1;
|
|
741
|
-
const jobResult = update.state === 'done'
|
|
742
|
-
? { state: 'done', result: update.result }
|
|
743
|
-
: { state: 'failed', error: update.error ?? null };
|
|
744
|
-
const resolution = await completionGate.resolveHelperCompletion({
|
|
745
|
-
workspaceId,
|
|
746
|
-
instance,
|
|
747
|
-
block,
|
|
748
|
-
step,
|
|
749
|
-
result: jobResult,
|
|
750
|
-
});
|
|
751
|
-
// Preserve the done-result's fields (usage metering etc.) while recording the gate's
|
|
752
|
-
// resolved output; a failed investigation has no result to carry.
|
|
753
|
-
const base = update.state === 'done' ? update.result : { output: '' };
|
|
754
|
-
return this.recordStepResult(workspaceId, instance, step, isFinalStep, {
|
|
755
|
-
...base,
|
|
756
|
-
output: resolution.output,
|
|
757
|
-
});
|
|
758
|
-
}
|
|
759
|
-
return null;
|
|
760
|
-
}
|
|
761
|
-
/**
|
|
762
|
-
* Fold a running poll's container signals into `step.container`: a successful poll
|
|
763
|
-
* proves the container is `up`, and the harness's live phase (clone / agent / push)
|
|
764
|
-
* plus the transport's container id/url enrich it. Returns whether anything changed,
|
|
765
|
-
* so the caller only persists + emits on a real transition (an idle poll is a no-op).
|
|
766
|
-
* Prior id/url/phase are preserved when a poll omits them (drain-on-read semantics).
|
|
767
|
-
*/
|
|
768
|
-
/**
|
|
769
|
-
* Fold a running poll's live delta (container status/phase, subtask counts, backend, streamed
|
|
770
|
-
* follow-ups, env projection) onto `target`, returning whether anything changed. Idempotent for
|
|
771
|
-
* the set-to-latest folds and correct under CAS retry for the drain-on-read follow-up append —
|
|
772
|
-
* see the call site in {@link pollAgentJobInner}. A concurrent write that advanced the step (or
|
|
773
|
-
* superseded the job) makes it a no-op.
|
|
774
|
-
*/
|
|
775
|
-
async applyRunningFold(target, ctx) {
|
|
776
|
-
const { jobId, update, workspaceId } = ctx;
|
|
777
|
-
const s = target.steps[target.currentStep];
|
|
778
|
-
// The step advanced (or the job was superseded) under a concurrent write — nothing to fold.
|
|
779
|
-
if (!s || s.jobId !== jobId)
|
|
780
|
-
return false;
|
|
781
|
-
let changed = false;
|
|
782
|
-
if (applyContainerRunning(s, update))
|
|
783
|
-
changed = true;
|
|
784
|
-
if (applySubtaskProgress(s, update.subtasks))
|
|
785
|
-
changed = true;
|
|
786
|
-
// Persist the harness liveness heartbeat (throttled) so a quiet-but-alive container keeps the
|
|
787
|
-
// run's `updated_at` fresh — the signal a long, output-less phase (a reviewer reading files)
|
|
788
|
-
// would otherwise never emit, leaving it indistinguishable from a wedged run to the sweeper + UI.
|
|
789
|
-
if (applyLastActivity(s, update.lastActivityAt))
|
|
790
|
-
changed = true;
|
|
791
|
-
// Republish the latest pre-PR validation attempt so the repair loop is visible WHILE it
|
|
792
|
-
// runs ("lint failed, repairing — attempt 2 of 3") instead of only at the end.
|
|
793
|
-
if (applyValidationReport(s, update.validationReport))
|
|
794
|
-
changed = true;
|
|
795
|
-
// Republish the reproduction proof so a failed verification is visible WHILE the repair loop
|
|
796
|
-
// still runs, for the same reason as the validation republish above.
|
|
797
|
-
if (applyReproductionReport(s, update.reproductionReport))
|
|
798
|
-
changed = true;
|
|
799
|
-
// Persist each PR-review slice's captured report as its subagent returns. Unlike the two
|
|
800
|
-
// republishes above this is not for visibility: the reviewer emits findings only in its
|
|
801
|
-
// terminal output, so this is the one thing that makes finished slices survive a review that
|
|
802
|
-
// never gets there, and the only state a manual resume can preserve work from.
|
|
803
|
-
if (applySliceReviews(s, update.sliceReviews))
|
|
804
|
-
changed = true;
|
|
805
|
-
// The transport reports WHICH backend served the job on the first poll (native host
|
|
806
|
-
// process vs. sandboxed container) — record it in the run diagnostics.
|
|
807
|
-
if (this.recordBackendDiagnostics(target, update.backend))
|
|
808
|
-
changed = true;
|
|
809
|
-
// Append any forward-looking items the Coder streamed since the last poll so the
|
|
810
|
-
// Follow-up companion lights up + accrues items LIVE while the container still runs.
|
|
811
|
-
if (this.followUpGate.appendStreamedFollowUps(s, update.followUps))
|
|
812
|
-
changed = true;
|
|
813
|
-
// Refresh the env projection so its status transitions (provisioning→ready→
|
|
814
|
-
// expired/torn_down) and any error stay live in the run details during the run.
|
|
815
|
-
if (await this.deployer.attachEnvironmentProjection(workspaceId, target.blockId, s)) {
|
|
816
|
-
changed = true;
|
|
817
|
-
}
|
|
818
|
-
return changed;
|
|
819
|
-
}
|
|
820
|
-
/**
|
|
821
|
-
* A polling gate step's in-flight job is its helper agent (ci-fixer / conflict-resolver / the
|
|
822
|
-
* human-review fixer), NOT the step's own work: when it finishes (or fails) we don't record a
|
|
823
|
-
* result or advance — we run any deterministic post-helper bookkeeping hook, record the attempt,
|
|
824
|
-
* drop the handle, return the gate to `checking`, and re-run the precheck (the helper's push
|
|
825
|
-
* triggers a fresh CI run / updates mergeability). A helper that failed without pushing leaves the
|
|
826
|
-
* precheck negative, so the next check re-dispatches (until the attempt budget is spent). Split
|
|
827
|
-
* from {@link pollAgentJobInner} to keep it under the complexity ceiling.
|
|
828
|
-
*/
|
|
829
|
-
async reprobeGateAfterHelper(gate, ctx) {
|
|
830
|
-
const { workspaceId, instance, step, update } = ctx;
|
|
831
|
-
// A gate may need deterministic GitHub-side bookkeeping to land BEFORE the re-probe
|
|
832
|
-
// reads it (the human-review gate replies to + RESOLVES the threads it handed the
|
|
833
|
-
// fixer, so the next probe counts them addressed). Run that side-effect hook first;
|
|
834
|
-
// it does NOT replace the re-probe (unlike resolveHelperCompletion).
|
|
835
|
-
if (gate.onHelperComplete && step.gate) {
|
|
836
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
837
|
-
if (block) {
|
|
838
|
-
const jobResult = update.state === 'done'
|
|
839
|
-
? { state: 'done', result: update.result }
|
|
840
|
-
: { state: 'failed', error: update.error ?? null };
|
|
841
|
-
await this.runInitiatorScope(instance.initiatedBy, () => gate.onHelperComplete({
|
|
842
|
-
workspaceId,
|
|
843
|
-
instance,
|
|
844
|
-
block,
|
|
845
|
-
step,
|
|
846
|
-
result: jobResult,
|
|
847
|
-
}));
|
|
848
|
-
}
|
|
849
|
-
}
|
|
850
|
-
// Record the just-finished helper attempt before re-probing. The gate's next
|
|
851
|
-
// precheck stays the source of truth for pass/fail, but the helper's own account
|
|
852
|
-
// (what it did, and for the conflict-resolver which files it left conflicting) is
|
|
853
|
-
// otherwise discarded here — leaving the gate window with only a bare attempt
|
|
854
|
-
// count. Capture it so the UI can show what each attempt tried.
|
|
855
|
-
if (step.gate) {
|
|
856
|
-
const attempt = recordGateAttempt(step.gate, update.state === 'done'
|
|
857
|
-
? { state: 'done', output: update.result.output ?? null }
|
|
858
|
-
: { state: 'failed', error: update.error ?? null }, this.clock.now());
|
|
859
|
-
step.gate.attemptLog = [...(step.gate.attemptLog ?? []), attempt];
|
|
860
|
-
// Same reasoning for the helper's effort self-assessment: a gate step runs no agent of
|
|
861
|
-
// its own, so its report is its LAST helper's (what made fixing CI / resolving the
|
|
862
|
-
// conflicts hard). This path deliberately never records a result, so without this the
|
|
863
|
-
// gate window could only ever show a bare attempt count.
|
|
864
|
-
if (update.state === 'done' && update.result.effortReport) {
|
|
865
|
-
step.effortReport = update.result.effortReport;
|
|
866
|
-
}
|
|
867
|
-
// The conflicts gate's precheck carries no failure detail of its own (GitHub
|
|
868
|
-
// reports mergeability as a single bit), so surface the resolver's account as
|
|
869
|
-
// the gate's last failure summary. CI's probe already sets a richer summary
|
|
870
|
-
// (the red checks) — don't clobber it with the fixer's push note.
|
|
871
|
-
if (step.agentKind === CONFLICTS_AGENT_KIND && attempt.summary) {
|
|
872
|
-
step.gate.lastFailureSummary = attempt.summary;
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
step.jobId = undefined;
|
|
876
|
-
step.subtasks = undefined;
|
|
877
|
-
if (step.gate)
|
|
878
|
-
step.gate.phase = 'checking';
|
|
879
|
-
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
880
|
-
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
881
|
-
return { kind: 'awaiting_gate', stepIndex: instance.currentStep };
|
|
882
|
-
}
|
|
883
|
-
/**
|
|
884
|
-
* Shared container-eviction recovery for an async step (agent or deployer). When `error` is a
|
|
885
|
-
* container-eviction error and the per-flavour budget (transient vs genuine) isn't spent, resets
|
|
886
|
-
* the step so the driver re-dispatches a fresh container (returns `continue`); once the budget is
|
|
887
|
-
* spent, marks the container errored and returns the terminal `job_evicted`. Returns null when
|
|
888
|
-
* `error` is NOT an eviction, so the caller proceeds with its own genuine-failure handling.
|
|
889
|
-
* `onBeforeRedispatch` runs the kind-specific reclaim (the deployer releases its separately
|
|
890
|
-
* dispatched deploy-job runner) before the step state is reset. Keeps the eviction budgets +
|
|
891
|
-
* the user-facing "still evicting…" wording uniform across the agent and deployer paths.
|
|
892
|
-
*/
|
|
893
|
-
async recoverContainerEviction(workspaceId, instance, step, failure, onBeforeRedispatch) {
|
|
894
|
-
const { error, evicted, detail } = failure;
|
|
895
|
-
// The eviction verdict rides the transport's STRUCTURED `evicted` field (every transport
|
|
896
|
-
// mints it). Absent ⇒ not an eviction, so the caller proceeds with genuine-failure handling.
|
|
897
|
-
const kind = evicted;
|
|
898
|
-
if (!kind)
|
|
899
|
-
return null;
|
|
900
|
-
const transient = kind === 'transient';
|
|
901
|
-
const limit = transient ? MAX_TRANSIENT_EVICTION_RECOVERIES : MAX_EVICTION_RECOVERIES;
|
|
902
|
-
const recoveries = transient
|
|
903
|
-
? (step.transientEvictionRecoveries ?? 0)
|
|
904
|
-
: (step.evictionRecoveries ?? 0);
|
|
905
|
-
if (recoveries < limit) {
|
|
906
|
-
if (transient)
|
|
907
|
-
step.transientEvictionRecoveries = recoveries + 1;
|
|
908
|
-
else
|
|
909
|
-
step.evictionRecoveries = recoveries + 1;
|
|
910
|
-
// Retain the FIRST death's post-mortem before re-dispatching: the dead container is
|
|
911
|
-
// removed right now, so this recovery is the last moment its evidence exists — and it is
|
|
912
|
-
// usually the informative one (the retry is a fresh container hitting the same wall).
|
|
913
|
-
// `evictionFailureDetail` folds it into the failure if the budget later runs out.
|
|
914
|
-
if (detail && !step.firstEvictionDetail)
|
|
915
|
-
step.firstEvictionDetail = detail;
|
|
916
|
-
if (onBeforeRedispatch)
|
|
917
|
-
await onBeforeRedispatch();
|
|
918
|
-
step.jobId = undefined;
|
|
919
|
-
step.subtasks = undefined;
|
|
920
|
-
step.progress = 0;
|
|
921
|
-
// The container vanished and a fresh one is about to boot for the re-dispatch, so the
|
|
922
|
-
// details show it spinning up again rather than a stale "up".
|
|
923
|
-
step.container = { status: 'starting' };
|
|
924
|
-
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
925
|
-
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
926
|
-
return { kind: 'continue' };
|
|
927
|
-
}
|
|
928
|
-
// Eviction budget spent — the container is gone for good. Mark it errored and persist so the
|
|
929
|
-
// failed details show the errored container (failRun re-reads the run from storage, so an
|
|
930
|
-
// in-memory-only mutation would be lost; it emits the terminal frame, so markContainerErrored
|
|
931
|
-
// deliberately doesn't).
|
|
932
|
-
await this.markContainerErrored(workspaceId, instance, step);
|
|
933
|
-
// The transports' post-mortems of the containers that died (exit state + log tail). Each
|
|
934
|
-
// container is reclaimed as the run settles or re-dispatches, so this is the only place the
|
|
935
|
-
// cause survives — carry it onto the failure rather than reporting a bare "still evicting".
|
|
936
|
-
const evictionDetail = evictionFailureDetail(step.firstEvictionDetail, detail);
|
|
937
|
-
return {
|
|
938
|
-
kind: 'job_evicted',
|
|
939
|
-
error: transient
|
|
940
|
-
? `${error} (still evicting after ${recoveries} automatic restarts through the infrastructure churn — treating as deterministic)`
|
|
941
|
-
: `${error ?? 'Container evicted'} (still evicting after ${recoveries} automatic container restart${recoveries === 1 ? '' : 's'} — treating as deterministic)`,
|
|
942
|
-
...(evictionDetail ? { detail: evictionDetail } : {}),
|
|
943
|
-
};
|
|
944
|
-
}
|
|
945
|
-
/**
|
|
946
|
-
* Mark a container step's container `errored` (preserving the id/url/phase it reached) and
|
|
947
|
-
* PERSIST it, so a failed run's details show the errored container. Called on the genuine
|
|
948
|
-
* job-failure / exhausted-eviction paths before the result funnels to `failRun`, which
|
|
949
|
-
* re-reads the run from storage (so an in-memory-only mutation here would be lost) and emits
|
|
950
|
-
* the terminal frame itself — so we deliberately persist WITHOUT emitting here, to avoid a
|
|
951
|
-
* redundant transient "errored but still running" broadcast right before the "failed" one.
|
|
952
|
-
*/
|
|
953
|
-
async markContainerErrored(workspaceId, instance, step) {
|
|
954
|
-
step.container = { ...step.container, status: 'errored' };
|
|
955
|
-
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
956
|
-
}
|
|
957
721
|
/**
|
|
958
722
|
* Re-run a polling gate step's precheck from the durable driver's `awaiting_gate`
|
|
959
723
|
* loop: which gate (ci / conflicts) is resolved from the current step's `agentKind`,
|
|
@@ -1391,187 +1155,6 @@ export class RunDispatcher {
|
|
|
1391
1155
|
step.output = resolution.output;
|
|
1392
1156
|
return resolution?.ownsTerminalStatus ?? false;
|
|
1393
1157
|
}
|
|
1394
|
-
/**
|
|
1395
|
-
* File a tracking issue/ticket for a `tracker` step from the preceding `analysis`
|
|
1396
|
-
* output. Non-LLM and best-effort: when no provider is wired or none is configured
|
|
1397
|
-
* for the workspace it simply notes the skip; a filing error is folded into the
|
|
1398
|
-
* step output rather than failing the run (the implementation still proceeds).
|
|
1399
|
-
*/
|
|
1400
|
-
async runTracker(workspaceId, instance, block) {
|
|
1401
|
-
if (!this.ticketTrackerProvider) {
|
|
1402
|
-
return { output: 'No issue tracker configured; skipped ticket creation.' };
|
|
1403
|
-
}
|
|
1404
|
-
// The report to file is the closest preceding `analysis` output, falling back
|
|
1405
|
-
// to the block description when the pipeline has no analysis step.
|
|
1406
|
-
const analysis = instance.steps
|
|
1407
|
-
.slice(0, instance.currentStep)
|
|
1408
|
-
.filter((s) => s.agentKind === ANALYSIS_AGENT_KIND && s.output)
|
|
1409
|
-
.map((s) => s.output)
|
|
1410
|
-
.pop();
|
|
1411
|
-
const body = (analysis ?? block.description ?? '').trim() || 'Automated tech-debt remediation.';
|
|
1412
|
-
const frameId = (await this.contextBuilder.resolveServiceFrameId(workspaceId, block.id)) ?? block.id;
|
|
1413
|
-
try {
|
|
1414
|
-
const ticket = await this.ticketTrackerProvider.createTicket({
|
|
1415
|
-
workspaceId,
|
|
1416
|
-
frameId,
|
|
1417
|
-
title: `Tech debt: ${block.title}`,
|
|
1418
|
-
body,
|
|
1419
|
-
});
|
|
1420
|
-
if (!ticket) {
|
|
1421
|
-
return { output: 'No issue tracker configured; skipped ticket creation.' };
|
|
1422
|
-
}
|
|
1423
|
-
return { output: `Filed tracking ticket ${ticket.externalId}: ${ticket.url}` };
|
|
1424
|
-
}
|
|
1425
|
-
catch (error) {
|
|
1426
|
-
return { output: `Could not file a tracking ticket: ${getErrorMessage(error)}` };
|
|
1427
|
-
}
|
|
1428
|
-
}
|
|
1429
|
-
/**
|
|
1430
|
-
* Run a `bug-intake` step — the recurring bug-triage pipeline's inbound dual of `tracker`
|
|
1431
|
-
* (design §3). Pull ONE matching open issue from the schedule's configured tracker board,
|
|
1432
|
-
* claim it (import + replace-link onto the reused block, mark it in-progress + comment), and
|
|
1433
|
-
* seed the block's title/description from it so every downstream step works THAT bug. When
|
|
1434
|
-
* nothing matches — or no task source is wired — the run completes SUCCESSFULLY with every
|
|
1435
|
-
* remaining step skipped (there is nothing to investigate / reproduce / fix), no notification.
|
|
1436
|
-
* Best-effort throughout: the intake helper never throws (a tracker outage resolves to a
|
|
1437
|
-
* no-op), and the pickup writeback is fire-and-forget.
|
|
1438
|
-
*/
|
|
1439
|
-
async runBugIntake(workspaceId, instance, step, block, isFinalStep) {
|
|
1440
|
-
const outcome = this.bugIntakeService
|
|
1441
|
-
? await this.bugIntakeService.pickForBlock(workspaceId, block.id)
|
|
1442
|
-
: { picked: null, summary: 'Issue intake is not configured on this deployment.' };
|
|
1443
|
-
if (!outcome.picked) {
|
|
1444
|
-
return this.completeRunSkippingRemaining(workspaceId, instance, step, outcome.summary);
|
|
1445
|
-
}
|
|
1446
|
-
const pickup = outcome.picked;
|
|
1447
|
-
// Seed the reused recurring block from the picked issue so each fire works a different bug
|
|
1448
|
-
// through the same block (the same block-seeding `createTaskFromIssue` does, applied in place).
|
|
1449
|
-
// Clear the previous fire's peer PRs too — this fire works a DIFFERENT bug, so a prior bug's
|
|
1450
|
-
// connected-repo PRs must not linger on the block. (The own-service `pullRequest` is overwritten
|
|
1451
|
-
// by this run's coder step before any step reads it; it is a non-nullable `BlockPatch` field, so
|
|
1452
|
-
// it cannot be cleared here anyway.)
|
|
1453
|
-
await this.blockRepository.update(workspaceId, block.id, {
|
|
1454
|
-
title: pickup.seedTitle,
|
|
1455
|
-
description: pickup.seedDescription,
|
|
1456
|
-
peerPullRequests: [],
|
|
1457
|
-
});
|
|
1458
|
-
// Best-effort: claim the issue where it was filed (in-progress mark + "taken by cat-factory"
|
|
1459
|
-
// comment). Fire-and-forget — a tracker hiccup must never fail the run, mirroring the PR
|
|
1460
|
-
// open/merge writeback hooks; and unlike them this is NOT gated on the writeback settings.
|
|
1461
|
-
const writeback = this.issueWriteback;
|
|
1462
|
-
if (writeback) {
|
|
1463
|
-
await runBestEffort(this.log, 'writeback.onIssuePickedUp', () => writeback.onIssuePickedUp(workspaceId, block.id, pickup.inProgressLabel ? { inProgressLabel: pickup.inProgressLabel } : {}), { workspaceId, executionId: instance.id, blockId: block.id });
|
|
1464
|
-
}
|
|
1465
|
-
return this.recordStepResult(workspaceId, instance, step, isFinalStep, {
|
|
1466
|
-
output: pickup.summary,
|
|
1467
|
-
});
|
|
1468
|
-
}
|
|
1469
|
-
/**
|
|
1470
|
-
* Complete the run successfully after a `bug-intake` step found no issue to work: record the
|
|
1471
|
-
* intake step's own no-match output (it SUCCEEDED — it made the decision), then mark every
|
|
1472
|
-
* REMAINING step `skipped` and finalize the reused block `done`, with NO notification (the
|
|
1473
|
-
* outcome is visible in the schedule's run history).
|
|
1474
|
-
*
|
|
1475
|
-
* The block is finalized `done` DIRECTLY here rather than through `RunStateMachine.finalizeBlock`:
|
|
1476
|
-
* for a mergerless task block (every bug-triage pipeline) finalizeBlock's terminal branch treats
|
|
1477
|
-
* the run as "work complete but unmerged" — it flips the block `pr_ready` and raises a
|
|
1478
|
-
* `pipeline_complete` "confirm + merge the PR" notification. This fire did NO work and opened NO
|
|
1479
|
-
* PR, so that card would be spurious (and its payload would reference a STALE PR carried over from
|
|
1480
|
-
* a prior fire). Setting the terminal status inline keeps the no-op silent, as documented.
|
|
1481
|
-
*/
|
|
1482
|
-
async completeRunSkippingRemaining(workspaceId, instance, step, summary) {
|
|
1483
|
-
step.output = summary;
|
|
1484
|
-
step.progress = 1;
|
|
1485
|
-
step.subtasks = undefined;
|
|
1486
|
-
this.stepGraph.finishStep(step);
|
|
1487
|
-
for (let i = instance.currentStep + 1; i < instance.steps.length; i++) {
|
|
1488
|
-
const remaining = instance.steps[i];
|
|
1489
|
-
if (!remaining)
|
|
1490
|
-
continue;
|
|
1491
|
-
remaining.skipped = true;
|
|
1492
|
-
remaining.output = '';
|
|
1493
|
-
remaining.progress = 1;
|
|
1494
|
-
remaining.subtasks = undefined;
|
|
1495
|
-
this.stepGraph.finishStep(remaining);
|
|
1496
|
-
}
|
|
1497
|
-
instance.currentStep = instance.steps.length - 1;
|
|
1498
|
-
instance.status = 'done';
|
|
1499
|
-
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
1500
|
-
if (block && block.status !== 'done') {
|
|
1501
|
-
await this.blockRepository.update(workspaceId, instance.blockId, {
|
|
1502
|
-
status: 'done',
|
|
1503
|
-
progress: 1,
|
|
1504
|
-
});
|
|
1505
|
-
}
|
|
1506
|
-
await this.runStateMachine.casPersist(workspaceId, instance);
|
|
1507
|
-
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1508
|
-
await this.runStateMachine.stopRunContainer(workspaceId, instance);
|
|
1509
|
-
return { kind: 'done' };
|
|
1510
|
-
}
|
|
1511
|
-
/**
|
|
1512
|
-
* Persist an APPROVED initiative plan for an `initiative-committer` step: flip the
|
|
1513
|
-
* entity to `executing` and mirror the tracker into the repo's default branch
|
|
1514
|
-
* (`docs/initiatives/<slug>/`) via the checkout-free {@link RepoFiles}. Deterministic,
|
|
1515
|
-
* no LLM. REPLAY-SAFE: the tracker commit hash-short-circuits (an unchanged entity
|
|
1516
|
-
* commits nothing) and `markExecuting` is content-idempotent, so a durable-driver
|
|
1517
|
-
* replay re-enters harmlessly. The repo mirror is skipped gracefully when GitHub
|
|
1518
|
-
* isn't wired (the DB entity stays the source of truth); a missing entity or an
|
|
1519
|
-
* empty plan is a REAL failure — completing the run would strand the initiative in
|
|
1520
|
-
* `planning` behind a green run.
|
|
1521
|
-
*/
|
|
1522
|
-
async runInitiativeCommitter(workspaceId, block) {
|
|
1523
|
-
if (!this.initiativeService) {
|
|
1524
|
-
return { kind: 'failed', error: 'Initiative module is not wired on this deployment.' };
|
|
1525
|
-
}
|
|
1526
|
-
const initiative = await this.initiativeService.getByBlock(workspaceId, block.id);
|
|
1527
|
-
if (!initiative) {
|
|
1528
|
-
return { kind: 'failed', error: 'No initiative entity found for this block.' };
|
|
1529
|
-
}
|
|
1530
|
-
if ((initiative.items ?? []).length === 0) {
|
|
1531
|
-
return {
|
|
1532
|
-
kind: 'failed',
|
|
1533
|
-
error: 'No approved plan to commit — the planner produced no usable items.',
|
|
1534
|
-
};
|
|
1535
|
-
}
|
|
1536
|
-
// Resolve the run repo BEFORE flipping status. `resolveRunRepo` returns null only when
|
|
1537
|
-
// GitHub is entirely unwired (skip the mirror gracefully — the DB entity stays the source
|
|
1538
|
-
// of truth), but it THROWS for a GitHub-connected workspace whose frame isn't linked to a
|
|
1539
|
-
// repo (`resolveRepoTarget` fails loudly rather than guessing one). Doing it first means
|
|
1540
|
-
// such a misconfiguration aborts the committer with the entity still truthfully
|
|
1541
|
-
// `awaiting_approval` — instead of flipping to `executing` and THEN throwing, which would
|
|
1542
|
-
// fail the run while leaving a committed status whose plan never got mirrored (a lie).
|
|
1543
|
-
const runRepo = await this.repoOps.resolveRunRepo(workspaceId, block.id);
|
|
1544
|
-
// Now flip to `executing` and render the tracker from the flipped entity — the committed
|
|
1545
|
-
// mirror (and its content hash) must record the REAL `executing` status. Committing the
|
|
1546
|
-
// pre-flip entity would bake a stale `awaiting_approval` status into
|
|
1547
|
-
// `initiative.json`/`tracker.md` that nothing re-commits in this slice, AND would break
|
|
1548
|
-
// replay-safety: a durable-driver replay re-reads the now-`executing` entity, whose hash
|
|
1549
|
-
// no longer matches the committed `version.json`, so the no-change short-circuit would miss
|
|
1550
|
-
// and re-commit. `markExecuting` is a committed CAS write that still runs before the git
|
|
1551
|
-
// side effect, so a CAS conflict aborts before any commit lands (no orphaned tracker commit).
|
|
1552
|
-
const executing = (await this.initiativeService.markExecuting(workspaceId, block.id, null)) ?? initiative;
|
|
1553
|
-
let doc = null;
|
|
1554
|
-
let mirror = 'Repo tracker mirror skipped (GitHub not connected).';
|
|
1555
|
-
if (runRepo) {
|
|
1556
|
-
doc = await commitInitiativeTracker(runRepo.repo, runRepo.baseBranch, executing, new Date(this.clock.now()));
|
|
1557
|
-
mirror = doc
|
|
1558
|
-
? `Committed docs/initiatives/${executing.slug}/ (v${doc.version}) to ${runRepo.baseBranch}.`
|
|
1559
|
-
: `Tracker already up to date in docs/initiatives/${executing.slug}/.`;
|
|
1560
|
-
// Stamp the committed version/hash back onto the entity (content-unchanged tick ⇒
|
|
1561
|
-
// no commit ⇒ nothing to stamp, so a replay skips this second write too).
|
|
1562
|
-
if (doc)
|
|
1563
|
-
await this.initiativeService.markExecuting(workspaceId, block.id, doc);
|
|
1564
|
-
}
|
|
1565
|
-
const phases = (executing.phases ?? []).length;
|
|
1566
|
-
const items = (executing.items ?? []).length;
|
|
1567
|
-
return {
|
|
1568
|
-
kind: 'ok',
|
|
1569
|
-
result: {
|
|
1570
|
-
output: `Initiative plan approved: ${phases} phase${phases === 1 ? '' : 's'}, ` +
|
|
1571
|
-
`${items} item${items === 1 ? '' : 's'}. ${mirror}`,
|
|
1572
|
-
},
|
|
1573
|
-
};
|
|
1574
|
-
}
|
|
1575
1158
|
/**
|
|
1576
1159
|
* The polling-gate registry, keyed by `agentKind`. A gate runs a programmatic
|
|
1577
1160
|
* precheck against a provider and only escalates to a helper container agent on a
|