@cat-factory/orchestration 0.63.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/container.d.ts +4 -1
- package/dist/container.d.ts.map +1 -1
- package/dist/container.js +45 -12
- package/dist/container.js.map +1 -1
- package/dist/modules/execution/AgentContextBuilder.d.ts +11 -0
- package/dist/modules/execution/AgentContextBuilder.d.ts.map +1 -1
- package/dist/modules/execution/AgentContextBuilder.js +49 -3
- package/dist/modules/execution/AgentContextBuilder.js.map +1 -1
- package/dist/modules/execution/ExecutionService.d.ts +9 -1
- package/dist/modules/execution/ExecutionService.d.ts.map +1 -1
- package/dist/modules/execution/ExecutionService.js +9 -1
- package/dist/modules/execution/ExecutionService.js.map +1 -1
- package/dist/modules/execution/RunDispatcher.d.ts +62 -1
- package/dist/modules/execution/RunDispatcher.d.ts.map +1 -1
- package/dist/modules/execution/RunDispatcher.js +327 -90
- package/dist/modules/execution/RunDispatcher.js.map +1 -1
- package/dist/modules/execution/RunStateMachine.d.ts +7 -0
- package/dist/modules/execution/RunStateMachine.d.ts.map +1 -1
- package/dist/modules/execution/RunStateMachine.js +8 -0
- package/dist/modules/execution/RunStateMachine.js.map +1 -1
- package/dist/modules/execution/deployer.logic.d.ts +15 -1
- package/dist/modules/execution/deployer.logic.d.ts.map +1 -1
- package/dist/modules/execution/deployer.logic.js +41 -2
- package/dist/modules/execution/deployer.logic.js.map +1 -1
- package/dist/modules/execution/frame.logic.d.ts +20 -0
- package/dist/modules/execution/frame.logic.d.ts.map +1 -0
- package/dist/modules/execution/frame.logic.js +40 -0
- package/dist/modules/execution/frame.logic.js.map +1 -0
- package/dist/modules/initiative/InitiativeLoopService.d.ts +88 -0
- package/dist/modules/initiative/InitiativeLoopService.d.ts.map +1 -0
- package/dist/modules/initiative/InitiativeLoopService.js +356 -0
- package/dist/modules/initiative/InitiativeLoopService.js.map +1 -0
- package/dist/modules/initiative/InitiativeService.d.ts +31 -1
- package/dist/modules/initiative/InitiativeService.d.ts.map +1 -1
- package/dist/modules/initiative/InitiativeService.js +53 -0
- package/dist/modules/initiative/InitiativeService.js.map +1 -1
- package/dist/modules/initiative/initiative.logic.d.ts +58 -1
- package/dist/modules/initiative/initiative.logic.d.ts.map +1 -1
- package/dist/modules/initiative/initiative.logic.js +157 -0
- package/dist/modules/initiative/initiative.logic.js.map +1 -1
- package/package.json +10 -10
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { ConflictError, DEFAULT_MERGE_PRESET, getErrorMessage, getProvider, INITIATIVE_ANALYST_AGENT_KIND, INITIATIVE_COMMITTER_AGENT_KIND, INITIATIVE_INTERVIEWER_AGENT_KIND, INITIATIVE_PLANNER_AGENT_KIND, isAsyncAgentExecutor, NotFoundError, parseLocalModelId, recordGateAttempt, registeredGateFactories, registeredStepResolverFactories, requireProvider, sameSubtasks, } from '@cat-factory/kernel';
|
|
2
2
|
import { frontendOriginsForService, parseBlueprintService, parseSpecDoc, } from '@cat-factory/contracts';
|
|
3
|
-
import { blueprintPostOp, commitInitiativeTracker, isCompanionKind, isContainerBackedCompanion, registeredAgentStep, registeredPostOps, registeredPreOps, runRepoOps, specPostOp, TASK_ESTIMATOR_AGENT_KIND, } from '@cat-factory/agents';
|
|
3
|
+
import { blueprintPostOp, commitInitiativeTracker, isCompanionKind, isContainerBackedCompanion, moduleSlug, registeredAgentStep, registeredPostOps, registeredPreOps, runRepoOps, specPostOp, TASK_ESTIMATOR_AGENT_KIND, } from '@cat-factory/agents';
|
|
4
4
|
import { DEPLOYER_AGENT_KIND, isDeployStep } from '@cat-factory/integrations';
|
|
5
5
|
import { coerceTaskEstimate, summarizeEstimate } from '../estimation/estimate.logic.js';
|
|
6
6
|
import { reviewableArtifactOutput } from './artifact-review.logic.js';
|
|
7
|
-
import { deployEvictionEpoch, deployJobId } from './deployer.logic.js';
|
|
7
|
+
import { deployEvictionEpoch, deployJobId, orderProvisionTargets } from './deployer.logic.js';
|
|
8
|
+
import { frameOf, validInvolvedServiceFrames } from './frame.logic.js';
|
|
8
9
|
import { ANALYSIS_AGENT_KIND, ARCHITECTURE_BRAINSTORM_AGENT_KIND, BLUEPRINTS_AGENT_KIND, CLARITY_REVIEW_AGENT_KIND, CONFLICTS_AGENT_KIND, HUMAN_TEST_AGENT_KIND, isTesterKind, MERGER_AGENT_KIND, REQUIREMENTS_BRAINSTORM_AGENT_KIND, REQUIREMENTS_REVIEW_AGENT_KIND, SPEC_WRITER_AGENT_KIND, TESTER_AGENT_KIND, TRACKER_AGENT_KIND, UI_TESTER_AGENT_KIND, VISUAL_CONFIRM_AGENT_KIND, } from './ci.logic.js';
|
|
9
10
|
import { followUpsToSendBack, hasPendingFollowUps, renderFollowUpRework, shouldLoopCoder, } from './followUp.logic.js';
|
|
10
11
|
import { agentFailureKindFromCause, classifyAgentFailure, isContainerEvictionError, isTransientEviction, MAX_EVICTION_RECOVERIES, MAX_TRANSIENT_EVICTION_RECOVERIES, } from './job.logic.js';
|
|
@@ -51,6 +52,31 @@ function parseRepoFromPullUrl(url) {
|
|
|
51
52
|
return undefined;
|
|
52
53
|
return { owner: match[1], repo: match[2] };
|
|
53
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* The `peerEnvUrls` provision input for the frame about to be provisioned: a comma-joined set of
|
|
57
|
+
* `slug=url` pairs for every target frame whose env is ALREADY ready this run — so a later
|
|
58
|
+
* provider (own frame, provisioned last in provider-before-consumer order) can template a
|
|
59
|
+
* connected service's URL into its manifest via `{{input.peerEnvUrls}}`. Empty when no peer is
|
|
60
|
+
* ready yet. Documented limitation: a provider needing its consumer's URL (a cyclic env
|
|
61
|
+
* dependency) is out of scope — there is no reconfigure pass.
|
|
62
|
+
*/
|
|
63
|
+
function buildPeerEnvUrls(targets, done) {
|
|
64
|
+
const parts = [];
|
|
65
|
+
const seen = new Map();
|
|
66
|
+
for (const target of targets) {
|
|
67
|
+
const env = done[target.frameId];
|
|
68
|
+
if (env?.status !== 'ready' || !env.url)
|
|
69
|
+
continue;
|
|
70
|
+
// Two ready providers can slugify to the same name; suffix the collision with an ordinal so a
|
|
71
|
+
// second provider's URL isn't silently dropped (or its entry made ambiguous) in the joined set.
|
|
72
|
+
const base = moduleSlug(target.frame.title);
|
|
73
|
+
const count = seen.get(base) ?? 0;
|
|
74
|
+
seen.set(base, count + 1);
|
|
75
|
+
const slug = count === 0 ? base : `${base}-${count + 1}`;
|
|
76
|
+
parts.push(`${slug}=${env.url}`);
|
|
77
|
+
}
|
|
78
|
+
return parts.join(',');
|
|
79
|
+
}
|
|
54
80
|
/**
|
|
55
81
|
* The per-step dispatch + completion spine of the execution engine. It owns the four
|
|
56
82
|
* registries (step handlers, completion interceptors, post-completion / terminal resolvers,
|
|
@@ -736,7 +762,7 @@ export class RunDispatcher {
|
|
|
736
762
|
* its own `humanTest.environment`, so this is for the other env-consuming steps
|
|
737
763
|
* (tester/coder/deployer).
|
|
738
764
|
*/
|
|
739
|
-
async attachEnvironmentProjection(workspaceId, blockId, step) {
|
|
765
|
+
async attachEnvironmentProjection(workspaceId, blockId, step, frameId) {
|
|
740
766
|
if (!this.environmentProvisioning)
|
|
741
767
|
return false;
|
|
742
768
|
// Only the env-aware kinds run against an ephemeral environment (the `deployer`
|
|
@@ -746,7 +772,13 @@ export class RunDispatcher {
|
|
|
746
772
|
if (!ENV_PROJECTION_KINDS.has(step.agentKind))
|
|
747
773
|
return false;
|
|
748
774
|
try {
|
|
749
|
-
|
|
775
|
+
// Project the SPECIFIED service frame's env when given (the in-flight / failed frame of a
|
|
776
|
+
// multi-env deploy); otherwise the task's OWN frame (a task provisions several envs under
|
|
777
|
+
// one block, so an un-keyed newest-wins read could surface a peer's). Absent frame ⇒ own.
|
|
778
|
+
const resolvedFrameId = frameId ??
|
|
779
|
+
(await this.contextBuilder.resolveServiceFrameId(workspaceId, blockId)) ??
|
|
780
|
+
undefined;
|
|
781
|
+
const handle = await this.environmentProvisioning.getHandleForBlock(workspaceId, blockId, resolvedFrameId);
|
|
750
782
|
const next = handle
|
|
751
783
|
? {
|
|
752
784
|
id: handle.id,
|
|
@@ -1045,63 +1077,194 @@ export class RunDispatcher {
|
|
|
1045
1077
|
* the root cause. So do NOT reintroduce a `rethrowAgentErrors` branch here.
|
|
1046
1078
|
*/
|
|
1047
1079
|
async runDeployerStep(workspaceId, instance, step, block, isFinalStep) {
|
|
1048
|
-
// The service frame (the run may target a task/module inside it) owns the provisioning
|
|
1049
|
-
// config — the "what + where". Resolve it so the deployer routes to the workspace handler
|
|
1050
|
-
// for its declared type. A service explicitly declaring `infraless` stands nothing up —
|
|
1051
|
-
// record a no-op step output. An UNDECLARED service falls through to the legacy
|
|
1052
|
-
// single-connection path (the compat bridge), so existing workspaces keep provisioning.
|
|
1053
|
-
const provisioning = await this.resolveServiceProvisioning(workspaceId, block);
|
|
1054
|
-
if (provisioning?.type === 'infraless') {
|
|
1055
|
-
// A service flipped to `infraless` provisions nothing — but it may have a prior live
|
|
1056
|
-
// environment from when it declared a real type. Tombstone it so the block stops
|
|
1057
|
-
// showing a stale live environment (no-op when there's none / provisioning unwired).
|
|
1058
|
-
await this.environmentProvisioning?.supersedeForBlock(workspaceId, block.id);
|
|
1059
|
-
return this.recordStepResult(workspaceId, instance, step, isFinalStep, {
|
|
1060
|
-
output: 'Service is infraless; no environment provisioned.',
|
|
1061
|
-
model: 'environment:none',
|
|
1062
|
-
});
|
|
1063
|
-
}
|
|
1064
1080
|
// A set `jobId` means a prior (possibly replayed) dispatch already started an async deploy
|
|
1065
|
-
// job — re-attach by polling instead of re-provisioning (mirrors
|
|
1081
|
+
// job for the IN-FLIGHT frame — re-attach by polling instead of re-provisioning (mirrors
|
|
1082
|
+
// {@link handleAgentStep}). Short-circuit BEFORE resolving targets so a parked re-attach
|
|
1083
|
+
// skips the workspace block-list read.
|
|
1066
1084
|
if (step.jobId) {
|
|
1067
1085
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
1068
1086
|
}
|
|
1069
|
-
//
|
|
1070
|
-
//
|
|
1071
|
-
//
|
|
1072
|
-
//
|
|
1073
|
-
|
|
1074
|
-
//
|
|
1075
|
-
//
|
|
1087
|
+
// Fan out over every service frame this run provisions an env for — the task's OWN frame plus
|
|
1088
|
+
// each still-valid involved-service frame (the connections initiative), ordered provider-
|
|
1089
|
+
// before-consumer. Resolve the target set ONCE here (one workspace block-list read); the
|
|
1090
|
+
// synchronous/infraless recursion threads it rather than re-reading per frame.
|
|
1091
|
+
const targets = await this.resolveDeployTargets(workspaceId, block, step.deployPrimaryFrameId);
|
|
1092
|
+
// Pin the primary (own) frame once, so every later re-entry/replay classifies it identically
|
|
1093
|
+
// regardless of a mid-flight reparent (see {@link resolveDeployTargets}). Persisted with the
|
|
1094
|
+
// first synchronous-settle / async-park upsert below.
|
|
1095
|
+
step.deployPrimaryFrameId ??= targets.find((t) => t.isPrimary)?.frameId;
|
|
1096
|
+
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1097
|
+
}
|
|
1098
|
+
/**
|
|
1099
|
+
* Advance a `deployer` fan-out over its already-resolved `targets`: dispatch the first un-settled
|
|
1100
|
+
* frame (parking on an async deploy job) or, once every frame has settled, complete the step. One
|
|
1101
|
+
* deploy job per frame, dispatched SEQUENTIALLY (parking between) so a later provider can receive
|
|
1102
|
+
* the already-ready peers' URLs. `step.deployEnvs` records each frame's TERMINAL outcome, so a
|
|
1103
|
+
* replay resumes at the first un-settled frame. Re-entered (with the SAME targets) after each
|
|
1104
|
+
* synchronous/infraless/failed-peer frame settles — never re-reading the block list per frame.
|
|
1105
|
+
*/
|
|
1106
|
+
async advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets) {
|
|
1107
|
+
const done = step.deployEnvs ?? {};
|
|
1108
|
+
const next = targets.find((t) => !done[t.frameId]);
|
|
1109
|
+
if (!next) {
|
|
1110
|
+
// Every frame settled: finish the step (all ready → done; a primary failure short-circuited).
|
|
1111
|
+
return this.completeDeployerStep(workspaceId, instance, step, isFinalStep, targets);
|
|
1112
|
+
}
|
|
1113
|
+
// A frame explicitly declaring `infraless` stands nothing up — tombstone any prior env for it
|
|
1114
|
+
// (a service flipped to `infraless` from a real type), mark the frame skipped, and re-enter to
|
|
1115
|
+
// process the next frame. An UNDECLARED own frame falls through to the legacy single-connection
|
|
1116
|
+
// path (the compat bridge), so existing workspaces keep provisioning.
|
|
1117
|
+
if (next.provisioning?.type === 'infraless') {
|
|
1118
|
+
await this.environmentProvisioning?.supersedeForBlock(workspaceId, block.id, next.frameId);
|
|
1119
|
+
step.deployEnvs = { ...done, [next.frameId]: { status: 'skipped' } };
|
|
1120
|
+
// Persist this frame's TERMINAL outcome BEFORE processing the next frame, so a crash/replay
|
|
1121
|
+
// mid-fan-out resumes at the first un-settled frame rather than re-doing an already-settled
|
|
1122
|
+
// one (which, on the synchronous REST path, would re-hit the provider — no idempotency guard
|
|
1123
|
+
// there, unlike the deterministic async job ref).
|
|
1124
|
+
await this.executionRepository.upsert(workspaceId, instance);
|
|
1125
|
+
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1126
|
+
}
|
|
1127
|
+
// Start provisioning the next frame: a raw-manifest config provisions SYNCHRONOUSLY over REST
|
|
1128
|
+
// (a final handle); a config that needs rendering dispatches a CONTAINER-backed deploy job we
|
|
1129
|
+
// park on and poll. The job ref is DETERMINISTIC (run id + deployer kind + FRAME + eviction
|
|
1130
|
+
// epoch), so a Workflows replay reproduces the same id and the transport re-attaches instead
|
|
1131
|
+
// of double-dispatching. The frame discriminator keeps each fanned-out job distinct.
|
|
1076
1132
|
const ref = {
|
|
1077
1133
|
runId: instance.id,
|
|
1078
|
-
jobId: deployJobId(instance.id, deployEvictionEpoch(step)),
|
|
1134
|
+
jobId: deployJobId(instance.id, deployEvictionEpoch(step), next.frameId),
|
|
1079
1135
|
};
|
|
1136
|
+
const peerEnvUrls = buildPeerEnvUrls(targets, done);
|
|
1080
1137
|
let dispatch;
|
|
1081
1138
|
try {
|
|
1082
|
-
dispatch = await this.environmentProvisioning.startProvision(await this.deployerProvisionArgs(workspaceId, instance, block,
|
|
1139
|
+
dispatch = await this.environmentProvisioning.startProvision(await this.deployerProvisionArgs(workspaceId, instance, block, next, peerEnvUrls), ref);
|
|
1083
1140
|
}
|
|
1084
1141
|
catch (error) {
|
|
1085
|
-
return this.
|
|
1142
|
+
return this.settleDeployerFailure(workspaceId, instance, step, block, isFinalStep, targets, next, null, getErrorMessage(error));
|
|
1086
1143
|
}
|
|
1087
1144
|
if (dispatch.kind === 'completed') {
|
|
1088
|
-
|
|
1145
|
+
// Synchronous provision: record this frame's outcome, then continue to the next frame.
|
|
1146
|
+
return this.settleDeployerFrame(workspaceId, instance, step, block, isFinalStep, targets, next, dispatch.handle);
|
|
1089
1147
|
}
|
|
1090
|
-
// An async deploy job was dispatched: park on
|
|
1148
|
+
// An async deploy job was dispatched: park on this frame. `dispatch` blocked until the job was
|
|
1091
1149
|
// accepted, so the container is up; the live phase + the provisioned outcome arrive on the
|
|
1092
|
-
// deployer poll branch. Surface the env spinning up alongside the parked step.
|
|
1150
|
+
// deployer poll branch. Surface the frame's env spinning up alongside the parked step.
|
|
1093
1151
|
step.jobId = dispatch.ref.jobId;
|
|
1152
|
+
step.deployFrameId = next.frameId;
|
|
1094
1153
|
step.container = { status: 'up' };
|
|
1095
1154
|
// Pin the provisioning config the container was built from, so the later poll/finalize maps
|
|
1096
1155
|
// the job against THIS config rather than a fresh read of the frame (which a person may edit
|
|
1097
1156
|
// mid-flight). Absent for the undeclared legacy path, which re-resolution handles harmlessly.
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step);
|
|
1157
|
+
step.deployProvisioning = next.provisioning;
|
|
1158
|
+
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step, next.frameId);
|
|
1101
1159
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1102
1160
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1103
1161
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
1104
1162
|
}
|
|
1163
|
+
/**
|
|
1164
|
+
* Resolve the ordered set of service frames a `deployer` step provisions environments for: the
|
|
1165
|
+
* task's OWN service frame (always, `isPrimary`) plus each involved-service frame (read-time
|
|
1166
|
+
* stale-filtered to ids that are still a connection neighbour AND resolve to a `service` frame
|
|
1167
|
+
* WITH declared provisioning — an involved frame with none stands nothing up here). Ordered
|
|
1168
|
+
* PROVIDER-before-CONSUMER over the connection edges among the targets (see
|
|
1169
|
+
* {@link orderProvisionTargets}) so a later provision can receive its ready peers' URLs. One
|
|
1170
|
+
* workspace block-list read; no per-frame point read.
|
|
1171
|
+
*
|
|
1172
|
+
* `pinnedPrimaryFrameId` (from {@link PipelineStep.deployPrimaryFrameId}, set on the first
|
|
1173
|
+
* resolution) keeps the OWN/primary frame STABLE across re-entries: once the fan-out has started,
|
|
1174
|
+
* a mid-flight reparent must not re-classify which frame is primary — that would flip an
|
|
1175
|
+
* own-frame failure from terminal to a non-terminal peer failure. Prefer the pinned frame when it
|
|
1176
|
+
* still resolves; fall back to a fresh `frameOf` walk otherwise.
|
|
1177
|
+
*/
|
|
1178
|
+
async resolveDeployTargets(workspaceId, block, pinnedPrimaryFrameId) {
|
|
1179
|
+
const blocks = await this.blockRepository.listByWorkspace(workspaceId);
|
|
1180
|
+
const byId = new Map(blocks.map((b) => [b.id, b]));
|
|
1181
|
+
const ownFrame = (pinnedPrimaryFrameId ? byId.get(pinnedPrimaryFrameId) : undefined) ??
|
|
1182
|
+
frameOf(byId, block.id) ??
|
|
1183
|
+
block;
|
|
1184
|
+
const targets = [
|
|
1185
|
+
{
|
|
1186
|
+
frameId: ownFrame.id,
|
|
1187
|
+
isPrimary: true,
|
|
1188
|
+
provisioning: ownFrame.provisioning,
|
|
1189
|
+
frame: ownFrame,
|
|
1190
|
+
},
|
|
1191
|
+
];
|
|
1192
|
+
// The connected involved-service frames, read-time stale-filtered by the shared helper (kept in
|
|
1193
|
+
// sync with `AgentContextBuilder.resolveInvolvedServices`). Include each regardless of declared
|
|
1194
|
+
// provisioning — like the OWN frame, an undeclared service falls through to the legacy
|
|
1195
|
+
// single-connection compat bridge, and an `infraless` one is skipped by the dispatch loop. Only
|
|
1196
|
+
// the dispatch decides what actually stands up.
|
|
1197
|
+
for (const frame of validInvolvedServiceFrames(blocks, block, ownFrame.id)) {
|
|
1198
|
+
if (targets.some((t) => t.frameId === frame.id))
|
|
1199
|
+
continue;
|
|
1200
|
+
targets.push({
|
|
1201
|
+
frameId: frame.id,
|
|
1202
|
+
isPrimary: false,
|
|
1203
|
+
provisioning: frame.provisioning,
|
|
1204
|
+
frame,
|
|
1205
|
+
});
|
|
1206
|
+
}
|
|
1207
|
+
const targetIds = new Set(targets.map((t) => t.frameId));
|
|
1208
|
+
const providersOf = new Map();
|
|
1209
|
+
for (const target of targets) {
|
|
1210
|
+
const providers = new Set();
|
|
1211
|
+
for (const connection of target.frame.serviceConnections ?? []) {
|
|
1212
|
+
if (connection.serviceBlockId !== target.frameId &&
|
|
1213
|
+
targetIds.has(connection.serviceBlockId)) {
|
|
1214
|
+
providers.add(connection.serviceBlockId);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
providersOf.set(target.frameId, providers);
|
|
1218
|
+
}
|
|
1219
|
+
const order = orderProvisionTargets(targets.map((t) => ({ frameId: t.frameId, isPrimary: t.isPrimary })), providersOf);
|
|
1220
|
+
const byFrame = new Map(targets.map((t) => [t.frameId, t]));
|
|
1221
|
+
return order.map((id) => byFrame.get(id));
|
|
1222
|
+
}
|
|
1223
|
+
/**
|
|
1224
|
+
* Record one frame's TERMINAL deploy outcome onto `step.deployEnvs`, then continue the fan-out.
|
|
1225
|
+
* A `ready` handle records the env and re-enters {@link advanceDeployerFrames} for the next
|
|
1226
|
+
* frame; a `failed` handle routes to {@link settleDeployerFailure} (terminal only for the own
|
|
1227
|
+
* frame). Shared by the synchronous-provision and async-finalized paths.
|
|
1228
|
+
*/
|
|
1229
|
+
async settleDeployerFrame(workspaceId, instance, step, block, isFinalStep, targets, target, handle) {
|
|
1230
|
+
if (handle.status === 'failed') {
|
|
1231
|
+
return this.settleDeployerFailure(workspaceId, instance, step, block, isFinalStep, targets, target, handle.url, handle.lastError ?? 'Provisioning failed.');
|
|
1232
|
+
}
|
|
1233
|
+
if (handle.status !== 'ready' && !target.isPrimary) {
|
|
1234
|
+
// A PEER env that isn't `ready` (`provisioning`, `expired`, `tearing_down`, …) is not usable
|
|
1235
|
+
// context: `deployEnvs` can only record `ready`/`failed`/`skipped`, and recording it `ready`
|
|
1236
|
+
// would BOTH advertise it "Provisioned involved-service environment …" AND inject its not-live
|
|
1237
|
+
// URL into a consumer's `peerEnvUrls`. Drop it as a non-terminal peer failure instead. (The
|
|
1238
|
+
// OWN frame keeps the historical behaviour — its env is the deploy's product; its live status
|
|
1239
|
+
// is surfaced via the Environment projection, and the run proceeds as before.)
|
|
1240
|
+
return this.settleDeployerFailure(workspaceId, instance, step, block, isFinalStep, targets, target, handle.url, `Environment not ready (status: ${handle.status}).`);
|
|
1241
|
+
}
|
|
1242
|
+
const done = step.deployEnvs ?? {};
|
|
1243
|
+
step.deployEnvs = { ...done, [target.frameId]: { status: 'ready', url: handle.url } };
|
|
1244
|
+
// Persist this frame's TERMINAL outcome BEFORE provisioning the next frame (see the infraless
|
|
1245
|
+
// branch) so a crash/replay resumes at the first un-settled frame, not re-provisioning this one.
|
|
1246
|
+
await this.executionRepository.upsert(workspaceId, instance);
|
|
1247
|
+
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1248
|
+
}
|
|
1249
|
+
/**
|
|
1250
|
+
* Record a frame's FAILED deploy outcome and decide whether it is terminal. The task's OWN
|
|
1251
|
+
* (primary) service frame failing fails the whole deploy step (unchanged from the single-env
|
|
1252
|
+
* path). An involved PEER frame failing is NON-terminal — the peer's env is best-effort context
|
|
1253
|
+
* enrichment, so the run proceeds to the remaining frames without that peer's URL rather than
|
|
1254
|
+
* failing a task because a service it merely "involves" has a misconfigured provider. The failed
|
|
1255
|
+
* outcome is still recorded (surfaced in {@link completeDeployerStep}).
|
|
1256
|
+
*/
|
|
1257
|
+
async settleDeployerFailure(workspaceId, instance, step, block, isFinalStep, targets, target, url, error) {
|
|
1258
|
+
const done = step.deployEnvs ?? {};
|
|
1259
|
+
step.deployEnvs = { ...done, [target.frameId]: { status: 'failed', url: url ?? null, error } };
|
|
1260
|
+
if (target.isPrimary) {
|
|
1261
|
+
return this.failDeployerStep(workspaceId, instance, step, target.frameId, error);
|
|
1262
|
+
}
|
|
1263
|
+
// A PEER failure is non-terminal — persist it BEFORE moving to the next frame so a replay
|
|
1264
|
+
// doesn't re-attempt this failed peer (same rationale as the ready/infraless settle paths).
|
|
1265
|
+
await this.executionRepository.upsert(workspaceId, instance);
|
|
1266
|
+
return this.advanceDeployerFrames(workspaceId, instance, step, block, isFinalStep, targets);
|
|
1267
|
+
}
|
|
1105
1268
|
/**
|
|
1106
1269
|
* Poll a `deployer` step's dispatched CONTAINER-backed deploy job (the async kustomize/helm
|
|
1107
1270
|
* path) through the environment provisioning service — NOT the agent executor. Mirrors
|
|
@@ -1111,6 +1274,10 @@ export class RunDispatcher {
|
|
|
1111
1274
|
*/
|
|
1112
1275
|
async pollDeployerJob(workspaceId, instance, step) {
|
|
1113
1276
|
const ref = { runId: instance.id, jobId: step.jobId };
|
|
1277
|
+
// The service frame this in-flight deploy job is provisioning (a multi-env fan-out dispatches
|
|
1278
|
+
// one job per frame). Falls back to the own frame for a single-frame deploy that predates the
|
|
1279
|
+
// discriminator / never fanned out.
|
|
1280
|
+
const inFlightFrameId = step.deployFrameId ?? undefined;
|
|
1114
1281
|
// Let a status-read failure THROW to the driver, exactly as `pollAgentJob` lets
|
|
1115
1282
|
// `executor.pollJob` throw: the driver counts consecutive read failures and fast-fails the
|
|
1116
1283
|
// run as `timeout` once `jobPollFailureTolerance` is hit. Swallowing it here would hide every
|
|
@@ -1123,7 +1290,7 @@ export class RunDispatcher {
|
|
|
1123
1290
|
changed = true;
|
|
1124
1291
|
if (this.applySubtaskProgress(step, view.progress))
|
|
1125
1292
|
changed = true;
|
|
1126
|
-
if (await this.attachEnvironmentProjection(workspaceId, instance.blockId, step)) {
|
|
1293
|
+
if (await this.attachEnvironmentProjection(workspaceId, instance.blockId, step, inFlightFrameId)) {
|
|
1127
1294
|
changed = true;
|
|
1128
1295
|
}
|
|
1129
1296
|
if (changed) {
|
|
@@ -1133,26 +1300,43 @@ export class RunDispatcher {
|
|
|
1133
1300
|
return { kind: 'awaiting_job', jobId: step.jobId, stepIndex: instance.currentStep };
|
|
1134
1301
|
}
|
|
1135
1302
|
// The deploy container vanished (evicted/crashed). The shared recovery re-dispatches a fresh
|
|
1136
|
-
// deploy job (the driver loops back into `runDeployerStep`, which re-provisions
|
|
1137
|
-
// `step.jobId` is cleared) within the same per-flavour budgets as the
|
|
1138
|
-
// the dead job's runner first.
|
|
1303
|
+
// deploy job (the driver loops back into `runDeployerStep`, which re-provisions the same
|
|
1304
|
+
// un-settled frame since `step.jobId` is cleared) within the same per-flavour budgets as the
|
|
1305
|
+
// agent path, reclaiming the dead job's runner first. Null for a non-eviction failure.
|
|
1139
1306
|
if (view.state === 'failed') {
|
|
1140
1307
|
const recovered = await this.recoverContainerEviction(workspaceId, instance, step, view.error, () => this.environmentProvisioning.releaseProvisionJob(workspaceId, ref).catch(() => { }));
|
|
1141
1308
|
if (recovered)
|
|
1142
1309
|
return recovered;
|
|
1143
1310
|
}
|
|
1144
1311
|
// Genuine terminal (done, or a non-eviction failure): finalize the deploy job into an
|
|
1145
|
-
// environment record and
|
|
1146
|
-
// which `
|
|
1312
|
+
// environment record and record this frame's outcome. A `failed` view maps to a failed env,
|
|
1313
|
+
// which `settleDeployerFrame` surfaces as a displayed step failure.
|
|
1147
1314
|
const block = await this.blockRepository.get(workspaceId, instance.blockId);
|
|
1148
1315
|
if (!block)
|
|
1149
1316
|
return { kind: 'noop' };
|
|
1150
1317
|
const isFinalStep = instance.currentStep === instance.steps.length - 1;
|
|
1318
|
+
// Resolve the full target set once (also drives the remaining-frames fan-out after this one
|
|
1319
|
+
// settles), honouring the pinned primary frame. Derive the own/primary frame id from that set
|
|
1320
|
+
// rather than a SECOND `resolveServiceFrameId` point-read walk — the primary target's frame id
|
|
1321
|
+
// is the own frame (pinned, so a mid-flight reparent can't flip an own failure to a peer one).
|
|
1322
|
+
const targets = await this.resolveDeployTargets(workspaceId, block, step.deployPrimaryFrameId);
|
|
1323
|
+
const ownFrameId = step.deployPrimaryFrameId ?? targets.find((t) => t.isPrimary)?.frameId ?? block.id;
|
|
1324
|
+
const frameId = inFlightFrameId ?? ownFrameId;
|
|
1325
|
+
// Recover the in-flight frame's real service-frame block from the target set so finalize
|
|
1326
|
+
// provisions with the FRAME's identity/inputs (a peer's env must not reuse the task block's —
|
|
1327
|
+
// see {@link deployerProvisionArgs}); fall back to a point-read (then the task block) if a
|
|
1328
|
+
// connection was removed mid-flight so the frame is no longer a target.
|
|
1329
|
+
const known = targets.find((t) => t.frameId === frameId);
|
|
1330
|
+
const frame = known?.frame ?? (await this.blockRepository.get(workspaceId, frameId)) ?? block;
|
|
1151
1331
|
// Map the job against the provisioning config the container was BUILT from (pinned at
|
|
1152
1332
|
// dispatch), not a fresh read of the frame a person may have edited mid-flight — else a
|
|
1153
|
-
// config flip (e.g. → `infraless`) would fail a deploy whose container already succeeded.
|
|
1333
|
+
// config flip (e.g. → `infraless`) would fail a deploy whose container already succeeded. The
|
|
1334
|
+
// pinned config is the in-flight frame's; the fallback resolution is only ever hit for the
|
|
1335
|
+
// undeclared-own compat path (which resolves the own frame correctly).
|
|
1154
1336
|
const provisioning = step.deployProvisioning ?? (await this.resolveServiceProvisioning(workspaceId, block));
|
|
1337
|
+
const target = { frameId, isPrimary: frameId === ownFrameId, provisioning, frame };
|
|
1155
1338
|
step.jobId = undefined;
|
|
1339
|
+
step.deployFrameId = undefined;
|
|
1156
1340
|
step.subtasks = undefined;
|
|
1157
1341
|
// The one-shot deploy container reached a terminal state: reclaim its runner now rather than
|
|
1158
1342
|
// letting it idle out its sleepAfter window (billed-but-useless compute) / leak a self-hosted
|
|
@@ -1162,50 +1346,76 @@ export class RunDispatcher {
|
|
|
1162
1346
|
await this.environmentProvisioning.releaseProvisionJob(workspaceId, ref).catch(() => { });
|
|
1163
1347
|
let handle;
|
|
1164
1348
|
try {
|
|
1165
|
-
handle = await this.environmentProvisioning.finalizeProvision(await this.deployerProvisionArgs(workspaceId, instance, block,
|
|
1349
|
+
handle = await this.environmentProvisioning.finalizeProvision(await this.deployerProvisionArgs(workspaceId, instance, block, target, ''), view);
|
|
1166
1350
|
}
|
|
1167
1351
|
catch (error) {
|
|
1168
1352
|
// The deploy container is gone (released above) but finalize failed: stamp the container
|
|
1169
|
-
// errored so the failed details don't keep showing it "up".
|
|
1353
|
+
// errored so the failed details don't keep showing it "up". A primary failure is terminal; a
|
|
1354
|
+
// peer's is not (the fan-out proceeds), so route through `settleDeployerFailure`.
|
|
1170
1355
|
if (step.container)
|
|
1171
1356
|
step.container = { ...step.container, status: 'errored' };
|
|
1172
|
-
|
|
1357
|
+
step.deployProvisioning = undefined;
|
|
1358
|
+
return this.settleDeployerFailure(workspaceId, instance, step, block, isFinalStep, targets, target, null, getErrorMessage(error));
|
|
1173
1359
|
}
|
|
1360
|
+
step.deployProvisioning = undefined;
|
|
1174
1361
|
// Reflect the container's terminal state from the RESOLVED outcome, not the raw view: a `done`
|
|
1175
1362
|
// view the provider maps to a FAILED env (e.g. the harness exited 0 but the namespace is
|
|
1176
1363
|
// missing) must still show the container errored — keying off `view.state` alone missed that.
|
|
1177
1364
|
if (handle.status === 'failed' && step.container) {
|
|
1178
1365
|
step.container = { ...step.container, status: 'errored' };
|
|
1179
1366
|
}
|
|
1180
|
-
return this.
|
|
1367
|
+
return this.settleDeployerFrame(workspaceId, instance, step, block, isFinalStep, targets, target, handle);
|
|
1181
1368
|
}
|
|
1182
|
-
/**
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1369
|
+
/**
|
|
1370
|
+
* The {@link ProvisionArgs} for provisioning ONE target frame's environment (synchronous or
|
|
1371
|
+
* async). The env is keyed by the task `block.id` + the target `frameId` — so a task's own env
|
|
1372
|
+
* and each involved-service env coexist under the same block, discriminated by frame (see the
|
|
1373
|
+
* per-`(blockId, frameId)` supersede). The repo/clone the provider resolves is the TARGET
|
|
1374
|
+
* FRAME's (via `frameId`), so an involved-service env clones that peer's repo at its default
|
|
1375
|
+
* branch, while the OWN frame targets the task's PR branch (its git/PR context); a peer carries
|
|
1376
|
+
* no PR context. The `{{input.*}}` identity (blockId/title/…) is the TARGET FRAME's for a peer
|
|
1377
|
+
* (see {@link deployTargetInputs}) so each peer's provider namespace is distinct — the task-
|
|
1378
|
+
* scoped inputs would collapse every peer onto one namespace. Injects `frontendOrigins` (the
|
|
1379
|
+
* browser origins binding this service) and `peerEnvUrls` (the already-ready peers) too.
|
|
1380
|
+
*/
|
|
1381
|
+
async deployerProvisionArgs(workspaceId, instance, block, target, peerEnvUrls) {
|
|
1382
|
+
const frontendOrigins = await this.frontendOriginsInput(workspaceId, target.frameId);
|
|
1383
|
+
// The OWN frame deploys the task's PR branch (its git/PR context); an involved peer carries no
|
|
1384
|
+
// PR context, so its clone target falls back to that repo's default branch.
|
|
1385
|
+
const context = target.isPrimary ? this.deployContext(block) : { blockId: block.id };
|
|
1195
1386
|
return {
|
|
1196
1387
|
workspaceId,
|
|
1197
1388
|
blockId: block.id,
|
|
1198
|
-
frameId,
|
|
1389
|
+
frameId: target.frameId,
|
|
1199
1390
|
executionId: instance.id,
|
|
1200
1391
|
inputs: {
|
|
1201
|
-
...this.
|
|
1392
|
+
...this.deployTargetInputs(block, target),
|
|
1202
1393
|
...(frontendOrigins ? { frontendOrigins } : {}),
|
|
1394
|
+
...(peerEnvUrls ? { peerEnvUrls } : {}),
|
|
1203
1395
|
},
|
|
1204
|
-
context
|
|
1205
|
-
...(provisioning ? { serviceProvisioning: provisioning } : {}),
|
|
1396
|
+
context,
|
|
1397
|
+
...(target.provisioning ? { serviceProvisioning: target.provisioning } : {}),
|
|
1206
1398
|
initiatedBy: instance.initiatedBy,
|
|
1207
1399
|
};
|
|
1208
1400
|
}
|
|
1401
|
+
/**
|
|
1402
|
+
* The `{{input.*}}` identity a target frame provisions with. The OWN frame keeps the historical
|
|
1403
|
+
* task-scoped inputs (its namespace is uniquified by the task's PR repo/number). An involved PEER
|
|
1404
|
+
* frame is scoped to the PEER FRAME's identity, with a `(task, peer)` composite `blockId` — so
|
|
1405
|
+
* the provider namespace derived from `{{input.blockId}}` is distinct per peer AND per task,
|
|
1406
|
+
* where the task-scoped inputs would collapse every peer of a task onto ONE namespace (each
|
|
1407
|
+
* clobbering the previous, teardown deleting the wrong one).
|
|
1408
|
+
*/
|
|
1409
|
+
deployTargetInputs(block, target) {
|
|
1410
|
+
if (target.isPrimary)
|
|
1411
|
+
return this.deployInputs(block);
|
|
1412
|
+
return {
|
|
1413
|
+
blockId: `${block.id}-${target.frameId}`,
|
|
1414
|
+
title: target.frame.title,
|
|
1415
|
+
type: target.frame.type,
|
|
1416
|
+
description: target.frame.description,
|
|
1417
|
+
};
|
|
1418
|
+
}
|
|
1209
1419
|
/**
|
|
1210
1420
|
* The `frontendOrigins` provision input for a service frame: the comma-joined browser origins
|
|
1211
1421
|
* of every `frontend` frame that binds this service (see `frontendOriginsForService`), for a
|
|
@@ -1223,31 +1433,55 @@ export class RunDispatcher {
|
|
|
1223
1433
|
* panel); otherwise the env summary (status / URL / provision type / engine) is recorded as the
|
|
1224
1434
|
* step output. Shared by the synchronous and async-finalized provision paths.
|
|
1225
1435
|
*/
|
|
1226
|
-
async completeDeployerStep(workspaceId, instance, step, isFinalStep,
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
//
|
|
1231
|
-
//
|
|
1232
|
-
//
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
if (
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1436
|
+
async completeDeployerStep(workspaceId, instance, step, isFinalStep, targets) {
|
|
1437
|
+
const byFrame = new Map(targets.map((t) => [t.frameId, t]));
|
|
1438
|
+
const primaryFrameId = step.deployPrimaryFrameId ?? targets.find((t) => t.isPrimary)?.frameId;
|
|
1439
|
+
// Re-project the now-final OWN environment (ready/expired + URL) so the deployer step's
|
|
1440
|
+
// Environment panel + the downstream tester see the task's own service env, not a peer's or the
|
|
1441
|
+
// dispatch-time `provisioning` snapshot the async poll last wrote. Pass the pinned primary frame
|
|
1442
|
+
// so it needn't re-walk the tree to find the own frame.
|
|
1443
|
+
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step, primaryFrameId);
|
|
1444
|
+
// Summarise from the recorded per-frame OUTCOMES (`deployEnvs`), NOT the current `targets`: a
|
|
1445
|
+
// mid-flight involved-services / connection edit can drop a frame from `targets` while its env
|
|
1446
|
+
// is still recorded and live, so iterating outcomes keeps that env visible (never silently
|
|
1447
|
+
// orphaned). Titles come from the target set when the frame is still resolvable, else the id.
|
|
1448
|
+
const done = step.deployEnvs ?? {};
|
|
1449
|
+
const titleOf = (frameId) => byFrame.get(frameId)?.frame.title ?? frameId;
|
|
1450
|
+
const isPrimaryFrame = (frameId) => byFrame.get(frameId)?.isPrimary ?? frameId === primaryFrameId;
|
|
1451
|
+
const readyEntries = Object.entries(done).filter(([, env]) => env.status === 'ready');
|
|
1452
|
+
if (readyEntries.length === 0) {
|
|
1453
|
+
// Every target was `infraless`/skipped — nothing stood up (the single-service infraless case
|
|
1454
|
+
// plus the all-infraless fan-out).
|
|
1455
|
+
return this.recordStepResult(workspaceId, instance, step, isFinalStep, {
|
|
1456
|
+
output: 'Service is infraless; no environment provisioned.',
|
|
1457
|
+
model: 'environment:none',
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
const own = step.environment;
|
|
1461
|
+
const lines = [];
|
|
1462
|
+
for (const [frameId, env] of readyEntries) {
|
|
1463
|
+
const url = env.url ?? '(pending)';
|
|
1464
|
+
lines.push(isPrimaryFrame(frameId)
|
|
1465
|
+
? `Provisioned ephemeral environment for '${titleOf(frameId)}': ${url}`
|
|
1466
|
+
: `Provisioned involved-service environment for '${titleOf(frameId)}': ${url}`);
|
|
1467
|
+
}
|
|
1468
|
+
// A PEER frame that failed is non-terminal (the own deploy proceeded); note it so the failure
|
|
1469
|
+
// is visible rather than silently absent from the fan-out summary. (A primary failure never
|
|
1470
|
+
// reaches here — it fails the step in `settleDeployerFailure`.)
|
|
1471
|
+
for (const [frameId, env] of Object.entries(done)) {
|
|
1472
|
+
if (env.status !== 'failed' || isPrimaryFrame(frameId))
|
|
1473
|
+
continue;
|
|
1474
|
+
lines.push(`Involved-service environment for '${titleOf(frameId)}' failed: ${env.error ?? 'unknown error'}`);
|
|
1475
|
+
}
|
|
1476
|
+
if (own?.expiresAt)
|
|
1477
|
+
lines.push(`Expires: ${new Date(own.expiresAt).toISOString()}`);
|
|
1478
|
+
if (own?.provisionType)
|
|
1479
|
+
lines.push(`Provision type: ${own.provisionType}`);
|
|
1480
|
+
if (own?.engine)
|
|
1481
|
+
lines.push(`Engine: ${own.engine}`);
|
|
1246
1482
|
return this.recordStepResult(workspaceId, instance, step, isFinalStep, {
|
|
1247
1483
|
output: lines.join('\n'),
|
|
1248
|
-
model:
|
|
1249
|
-
? `environment:${handle.engine}:${handle.providerId}`
|
|
1250
|
-
: `environment:${handle.providerId}`,
|
|
1484
|
+
model: `environment:${readyEntries.length > 1 ? 'multi' : (own?.engine ?? 'single')}`,
|
|
1251
1485
|
});
|
|
1252
1486
|
}
|
|
1253
1487
|
/**
|
|
@@ -1265,8 +1499,11 @@ export class RunDispatcher {
|
|
|
1265
1499
|
* `lastError`), persist + emit, then return a structured `environment` failure carrying the
|
|
1266
1500
|
* provider's message as the detail. Mirrors {@link handleAgentStep}'s dispatch-failure path.
|
|
1267
1501
|
*/
|
|
1268
|
-
async failDeployerStep(workspaceId, instance, step, message) {
|
|
1269
|
-
|
|
1502
|
+
async failDeployerStep(workspaceId, instance, step, frameId, message) {
|
|
1503
|
+
// Project the FAILED frame's env (so its `lastError` renders in the Environment panel) — for a
|
|
1504
|
+
// single-frame deploy that is the own env; for a failed involved-service env it surfaces the
|
|
1505
|
+
// peer's error rather than a sibling's healthy env.
|
|
1506
|
+
await this.attachEnvironmentProjection(workspaceId, instance.blockId, step, frameId);
|
|
1270
1507
|
await this.executionRepository.upsert(workspaceId, instance);
|
|
1271
1508
|
await this.runStateMachine.emitInstance(workspaceId, instance);
|
|
1272
1509
|
return {
|