@mcoda/core 0.1.9 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/api/AgentsApi.d.ts +1 -0
- package/dist/api/AgentsApi.d.ts.map +1 -1
- package/dist/api/AgentsApi.js +136 -11
- package/dist/api/QaTasksApi.d.ts.map +1 -1
- package/dist/api/QaTasksApi.js +4 -0
- package/dist/prompts/PdrPrompts.d.ts.map +1 -1
- package/dist/prompts/PdrPrompts.js +6 -0
- package/dist/prompts/SdsPrompts.d.ts.map +1 -1
- package/dist/prompts/SdsPrompts.js +7 -0
- package/dist/services/agents/AgentRatingService.d.ts +19 -0
- package/dist/services/agents/AgentRatingService.d.ts.map +1 -1
- package/dist/services/agents/AgentRatingService.js +66 -2
- package/dist/services/agents/GatewayAgentService.d.ts +8 -0
- package/dist/services/agents/GatewayAgentService.d.ts.map +1 -1
- package/dist/services/agents/GatewayAgentService.js +462 -65
- package/dist/services/agents/GatewayHandoff.d.ts +5 -1
- package/dist/services/agents/GatewayHandoff.d.ts.map +1 -1
- package/dist/services/agents/GatewayHandoff.js +65 -32
- package/dist/services/agents/RoutingService.d.ts +1 -0
- package/dist/services/agents/RoutingService.d.ts.map +1 -1
- package/dist/services/agents/RoutingService.js +4 -4
- package/dist/services/backlog/BacklogService.d.ts +23 -0
- package/dist/services/backlog/BacklogService.d.ts.map +1 -1
- package/dist/services/backlog/BacklogService.js +62 -7
- package/dist/services/backlog/TaskOrderingHeuristics.d.ts +12 -0
- package/dist/services/backlog/TaskOrderingHeuristics.d.ts.map +1 -0
- package/dist/services/backlog/TaskOrderingHeuristics.js +56 -0
- package/dist/services/backlog/TaskOrderingService.d.ts +16 -4
- package/dist/services/backlog/TaskOrderingService.d.ts.map +1 -1
- package/dist/services/backlog/TaskOrderingService.js +529 -73
- package/dist/services/docs/DocInventory.d.ts +11 -0
- package/dist/services/docs/DocInventory.d.ts.map +1 -0
- package/dist/services/docs/DocInventory.js +230 -0
- package/dist/services/docs/DocgenRunContext.d.ts +59 -0
- package/dist/services/docs/DocgenRunContext.d.ts.map +1 -0
- package/dist/services/docs/DocgenRunContext.js +4 -0
- package/dist/services/docs/DocsService.d.ts +59 -2
- package/dist/services/docs/DocsService.d.ts.map +1 -1
- package/dist/services/docs/DocsService.js +1701 -48
- package/dist/services/docs/alignment/DocAlignmentGraph.d.ts +23 -0
- package/dist/services/docs/alignment/DocAlignmentGraph.d.ts.map +1 -0
- package/dist/services/docs/alignment/DocAlignmentGraph.js +78 -0
- package/dist/services/docs/alignment/DocAlignmentPatcher.d.ts +19 -0
- package/dist/services/docs/alignment/DocAlignmentPatcher.d.ts.map +1 -0
- package/dist/services/docs/alignment/DocAlignmentPatcher.js +222 -0
- package/dist/services/docs/patch/DocPatchEngine.d.ts +57 -0
- package/dist/services/docs/patch/DocPatchEngine.d.ts.map +1 -0
- package/dist/services/docs/patch/DocPatchEngine.js +331 -0
- package/dist/services/docs/review/Glossary.d.ts +16 -0
- package/dist/services/docs/review/Glossary.d.ts.map +1 -0
- package/dist/services/docs/review/Glossary.js +47 -0
- package/dist/services/docs/review/ReviewReportRenderer.d.ts +3 -0
- package/dist/services/docs/review/ReviewReportRenderer.d.ts.map +1 -0
- package/dist/services/docs/review/ReviewReportRenderer.js +133 -0
- package/dist/services/docs/review/ReviewReportSchema.d.ts +39 -0
- package/dist/services/docs/review/ReviewReportSchema.d.ts.map +1 -0
- package/dist/services/docs/review/ReviewReportSchema.js +47 -0
- package/dist/services/docs/review/ReviewTypes.d.ts +76 -0
- package/dist/services/docs/review/ReviewTypes.d.ts.map +1 -0
- package/dist/services/docs/review/ReviewTypes.js +94 -0
- package/dist/services/docs/review/gates/AdminOpenApiSpecGate.d.ts +7 -0
- package/dist/services/docs/review/gates/AdminOpenApiSpecGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/AdminOpenApiSpecGate.js +93 -0
- package/dist/services/docs/review/gates/ApiPathConsistencyGate.d.ts +7 -0
- package/dist/services/docs/review/gates/ApiPathConsistencyGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/ApiPathConsistencyGate.js +308 -0
- package/dist/services/docs/review/gates/BuildReadyCompletenessGate.d.ts +8 -0
- package/dist/services/docs/review/gates/BuildReadyCompletenessGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/BuildReadyCompletenessGate.js +278 -0
- package/dist/services/docs/review/gates/DeploymentBlueprintGate.d.ts +8 -0
- package/dist/services/docs/review/gates/DeploymentBlueprintGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/DeploymentBlueprintGate.js +487 -0
- package/dist/services/docs/review/gates/NoMaybesGate.d.ts +8 -0
- package/dist/services/docs/review/gates/NoMaybesGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/NoMaybesGate.js +145 -0
- package/dist/services/docs/review/gates/OpenApiCoverageGate.d.ts +7 -0
- package/dist/services/docs/review/gates/OpenApiCoverageGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/OpenApiCoverageGate.js +266 -0
- package/dist/services/docs/review/gates/OpenApiSchemaSanityGate.d.ts +7 -0
- package/dist/services/docs/review/gates/OpenApiSchemaSanityGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/OpenApiSchemaSanityGate.js +59 -0
- package/dist/services/docs/review/gates/OpenQuestionsGate.d.ts +7 -0
- package/dist/services/docs/review/gates/OpenQuestionsGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/OpenQuestionsGate.js +200 -0
- package/dist/services/docs/review/gates/PdrInterfacesGate.d.ts +7 -0
- package/dist/services/docs/review/gates/PdrInterfacesGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/PdrInterfacesGate.js +159 -0
- package/dist/services/docs/review/gates/PdrOpenQuestionsGate.d.ts +8 -0
- package/dist/services/docs/review/gates/PdrOpenQuestionsGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/PdrOpenQuestionsGate.js +129 -0
- package/dist/services/docs/review/gates/PdrOwnershipGate.d.ts +7 -0
- package/dist/services/docs/review/gates/PdrOwnershipGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/PdrOwnershipGate.js +169 -0
- package/dist/services/docs/review/gates/PlaceholderArtifactGate.d.ts +10 -0
- package/dist/services/docs/review/gates/PlaceholderArtifactGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/PlaceholderArtifactGate.js +261 -0
- package/dist/services/docs/review/gates/RfpConsentGate.d.ts +6 -0
- package/dist/services/docs/review/gates/RfpConsentGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/RfpConsentGate.js +127 -0
- package/dist/services/docs/review/gates/RfpDefinitionGate.d.ts +7 -0
- package/dist/services/docs/review/gates/RfpDefinitionGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/RfpDefinitionGate.js +173 -0
- package/dist/services/docs/review/gates/SdsAdaptersGate.d.ts +7 -0
- package/dist/services/docs/review/gates/SdsAdaptersGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/SdsAdaptersGate.js +196 -0
- package/dist/services/docs/review/gates/SdsDecisionsGate.d.ts +7 -0
- package/dist/services/docs/review/gates/SdsDecisionsGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/SdsDecisionsGate.js +89 -0
- package/dist/services/docs/review/gates/SdsOpsGate.d.ts +7 -0
- package/dist/services/docs/review/gates/SdsOpsGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/SdsOpsGate.js +162 -0
- package/dist/services/docs/review/gates/SdsPolicyTelemetryGate.d.ts +7 -0
- package/dist/services/docs/review/gates/SdsPolicyTelemetryGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/SdsPolicyTelemetryGate.js +166 -0
- package/dist/services/docs/review/gates/SqlRequiredTablesGate.d.ts +7 -0
- package/dist/services/docs/review/gates/SqlRequiredTablesGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/SqlRequiredTablesGate.js +273 -0
- package/dist/services/docs/review/gates/SqlSyntaxGate.d.ts +7 -0
- package/dist/services/docs/review/gates/SqlSyntaxGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/SqlSyntaxGate.js +203 -0
- package/dist/services/docs/review/gates/TerminologyNormalizationGate.d.ts +9 -0
- package/dist/services/docs/review/gates/TerminologyNormalizationGate.d.ts.map +1 -0
- package/dist/services/docs/review/gates/TerminologyNormalizationGate.js +217 -0
- package/dist/services/docs/review/glossary.json +47 -0
- package/dist/services/estimate/EstimateService.d.ts +2 -0
- package/dist/services/estimate/EstimateService.d.ts.map +1 -1
- package/dist/services/estimate/EstimateService.js +66 -18
- package/dist/services/estimate/VelocityService.d.ts +4 -0
- package/dist/services/estimate/VelocityService.d.ts.map +1 -1
- package/dist/services/estimate/VelocityService.js +179 -36
- package/dist/services/estimate/types.d.ts +1 -0
- package/dist/services/estimate/types.d.ts.map +1 -1
- package/dist/services/execution/GatewayTrioService.d.ts +71 -4
- package/dist/services/execution/GatewayTrioService.d.ts.map +1 -1
- package/dist/services/execution/GatewayTrioService.js +1695 -328
- package/dist/services/execution/QaApiRunner.d.ts +30 -0
- package/dist/services/execution/QaApiRunner.d.ts.map +1 -0
- package/dist/services/execution/QaApiRunner.js +881 -0
- package/dist/services/execution/QaFollowupService.d.ts +1 -0
- package/dist/services/execution/QaFollowupService.d.ts.map +1 -1
- package/dist/services/execution/QaFollowupService.js +8 -2
- package/dist/services/execution/QaPlanValidator.d.ts +10 -0
- package/dist/services/execution/QaPlanValidator.d.ts.map +1 -0
- package/dist/services/execution/QaPlanValidator.js +128 -0
- package/dist/services/execution/QaProfileService.d.ts +21 -1
- package/dist/services/execution/QaProfileService.d.ts.map +1 -1
- package/dist/services/execution/QaProfileService.js +214 -29
- package/dist/services/execution/QaTasksService.d.ts +41 -1
- package/dist/services/execution/QaTasksService.d.ts.map +1 -1
- package/dist/services/execution/QaTasksService.js +2851 -500
- package/dist/services/execution/QaTestCommandBuilder.d.ts +51 -0
- package/dist/services/execution/QaTestCommandBuilder.d.ts.map +1 -0
- package/dist/services/execution/QaTestCommandBuilder.js +495 -0
- package/dist/services/execution/TaskSelectionService.d.ts +4 -2
- package/dist/services/execution/TaskSelectionService.d.ts.map +1 -1
- package/dist/services/execution/TaskSelectionService.js +144 -28
- package/dist/services/execution/TaskStateService.d.ts +19 -6
- package/dist/services/execution/TaskStateService.d.ts.map +1 -1
- package/dist/services/execution/TaskStateService.js +128 -13
- package/dist/services/execution/WorkOnTasksService.d.ts +19 -2
- package/dist/services/execution/WorkOnTasksService.d.ts.map +1 -1
- package/dist/services/execution/WorkOnTasksService.js +3913 -1225
- package/dist/services/jobs/JobInsightsService.d.ts +4 -0
- package/dist/services/jobs/JobInsightsService.d.ts.map +1 -1
- package/dist/services/jobs/JobInsightsService.js +51 -5
- package/dist/services/jobs/JobResumeService.d.ts.map +1 -1
- package/dist/services/jobs/JobResumeService.js +23 -10
- package/dist/services/jobs/JobService.d.ts +56 -4
- package/dist/services/jobs/JobService.d.ts.map +1 -1
- package/dist/services/jobs/JobService.js +232 -1
- package/dist/services/openapi/OpenApiService.d.ts +41 -0
- package/dist/services/openapi/OpenApiService.d.ts.map +1 -1
- package/dist/services/openapi/OpenApiService.js +889 -98
- package/dist/services/planning/CreateTasksService.d.ts +15 -0
- package/dist/services/planning/CreateTasksService.d.ts.map +1 -1
- package/dist/services/planning/CreateTasksService.js +311 -6
- package/dist/services/planning/RefineTasksService.d.ts +4 -0
- package/dist/services/planning/RefineTasksService.d.ts.map +1 -1
- package/dist/services/planning/RefineTasksService.js +225 -24
- package/dist/services/review/CodeReviewService.d.ts +4 -0
- package/dist/services/review/CodeReviewService.d.ts.map +1 -1
- package/dist/services/review/CodeReviewService.js +778 -232
- package/dist/services/review/ReviewNormalizer.d.ts +9 -0
- package/dist/services/review/ReviewNormalizer.d.ts.map +1 -0
- package/dist/services/review/ReviewNormalizer.js +147 -0
- package/dist/services/shared/AuthErrors.d.ts +3 -0
- package/dist/services/shared/AuthErrors.d.ts.map +1 -0
- package/dist/services/shared/AuthErrors.js +17 -0
- package/dist/services/shared/DocdexGuidance.d.ts +7 -0
- package/dist/services/shared/DocdexGuidance.d.ts.map +1 -0
- package/dist/services/shared/DocdexGuidance.js +12 -0
- package/dist/services/shared/ProjectGuidance.d.ts +12 -1
- package/dist/services/shared/ProjectGuidance.d.ts.map +1 -1
- package/dist/services/shared/ProjectGuidance.js +64 -7
- package/dist/services/system/ToolDenylist.d.ts +13 -0
- package/dist/services/system/ToolDenylist.d.ts.map +1 -0
- package/dist/services/system/ToolDenylist.js +85 -0
- package/dist/services/telemetry/TelemetryService.d.ts.map +1 -1
- package/dist/services/telemetry/TelemetryService.js +39 -7
- package/dist/workspace/WorkspaceManager.d.ts +22 -0
- package/dist/workspace/WorkspaceManager.d.ts.map +1 -1
- package/dist/workspace/WorkspaceManager.js +203 -32
- package/package.json +6 -5
|
@@ -1,30 +1,96 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { WorkspaceRepository } from "@mcoda/db";
|
|
4
|
-
import { PathHelper } from "@mcoda/shared";
|
|
4
|
+
import { PathHelper, READY_TO_CODE_REVIEW, isReadyToReviewStatus, normalizeReviewStatuses, } from "@mcoda/shared";
|
|
5
|
+
import { readDocdexCheck, summarizeDocdexCheck } from "@mcoda/integrations";
|
|
5
6
|
import { GatewayAgentService } from "../agents/GatewayAgentService.js";
|
|
7
|
+
import { RoutingService } from "../agents/RoutingService.js";
|
|
6
8
|
import { buildGatewayHandoffContent, buildGatewayHandoffDocdexUsage, withGatewayHandoff, writeGatewayHandoffFile, } from "../agents/GatewayHandoff.js";
|
|
7
9
|
import { JobService } from "../jobs/JobService.js";
|
|
8
10
|
import { TaskSelectionService } from "./TaskSelectionService.js";
|
|
9
11
|
import { WorkOnTasksService } from "./WorkOnTasksService.js";
|
|
10
12
|
import { CodeReviewService } from "../review/CodeReviewService.js";
|
|
11
13
|
import { QaTasksService } from "./QaTasksService.js";
|
|
12
|
-
const DEFAULT_STATUS_FILTER = ["not_started", "in_progress", "
|
|
14
|
+
const DEFAULT_STATUS_FILTER = ["not_started", "in_progress", "changes_requested", READY_TO_CODE_REVIEW, "ready_to_qa"];
|
|
13
15
|
const TERMINAL_STATUSES = new Set(["completed", "cancelled", "failed"]);
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
+
const GATEWAY_FAILED_REASON = "gateway_failed";
|
|
17
|
+
const ESCALATION_REASONS = new Set([
|
|
16
18
|
"missing_patch",
|
|
17
19
|
"patch_failed",
|
|
18
|
-
"tests_not_configured",
|
|
19
20
|
"tests_failed",
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
21
|
+
"agent_timeout",
|
|
22
|
+
"review_invalid_output",
|
|
23
|
+
"work_status_not_ready",
|
|
23
24
|
]);
|
|
24
|
-
const ESCALATION_REASONS = new Set(["missing_patch", "patch_failed", "tests_failed", "agent_timeout"]);
|
|
25
25
|
const NO_CHANGE_REASON = "no_changes";
|
|
26
|
+
const STRONG_TIER_MIN_COMPLEXITY = 5;
|
|
27
|
+
const SPECIALIST_TIER_MIN_COMPLEXITY = 8;
|
|
26
28
|
const DONE_DEPENDENCY_STATUSES = new Set(["completed", "cancelled"]);
|
|
27
29
|
const HEARTBEAT_INTERVAL_MS = 30000;
|
|
30
|
+
const ZERO_TOKEN_BACKOFF_MS = 750;
|
|
31
|
+
const PSEUDO_TASK_PREFIX = "[RUN]";
|
|
32
|
+
const ZERO_TOKEN_ERROR = "zero_tokens";
|
|
33
|
+
const FAILED_REOPEN_COOLDOWN_MS = 2 * 60 * 1000;
|
|
34
|
+
const MAX_FAILURE_REOPENS_PER_REASON = 2;
|
|
35
|
+
const AUTH_ERROR_REASON = "auth_error";
|
|
36
|
+
const AUTH_ERROR_PATTERNS = [
|
|
37
|
+
/auth_error/i,
|
|
38
|
+
/usage_limit_reached/i,
|
|
39
|
+
/too many requests/i,
|
|
40
|
+
/http\s*429/i,
|
|
41
|
+
/rate limit/i,
|
|
42
|
+
/usage limit/i,
|
|
43
|
+
];
|
|
44
|
+
const NON_RETRYABLE_FAILURE_REASONS = new Set([
|
|
45
|
+
"patch_failed",
|
|
46
|
+
"scope_violation",
|
|
47
|
+
"doc_edit_guard",
|
|
48
|
+
"merge_conflict",
|
|
49
|
+
"vcs_failed",
|
|
50
|
+
"task_lock_lost",
|
|
51
|
+
"missing_context",
|
|
52
|
+
"missing_docdex",
|
|
53
|
+
"review_invalid_output",
|
|
54
|
+
"gateway_invalid_output",
|
|
55
|
+
GATEWAY_FAILED_REASON,
|
|
56
|
+
AUTH_ERROR_REASON,
|
|
57
|
+
]);
|
|
58
|
+
const DOCDEX_SKIP_PATTERN = /\b(?:not executed|would run|not run|skipped)\b/i;
|
|
59
|
+
const DOCDEX_MISSING_PATTERN = /\b(?:docdex unavailable|docdex missing|no matching docs?|no matching documents?|no results|not provided)\b/i;
|
|
60
|
+
const MISSING_TASK_PATTERN = /\b(?:no concrete task|no task provided|task details are missing|task details missing|need the specific change request|no specific change request|no task context|task details are missing so no file paths can be named)\b/i;
|
|
61
|
+
const GUARDRAIL_REASON_PATTERN = /^guardrail:(retryable|non_retryable):([a-z0-9._-]+)$/i;
|
|
62
|
+
const GUARDRAIL_REASON_ALT_PATTERN = /^guardrail_(retryable|non_retryable):([a-z0-9._-]+)$/i;
|
|
63
|
+
const GOLDEN_EXAMPLES_REL_PATH = ".mcoda/codali/golden-examples.jsonl";
|
|
64
|
+
const GOLDEN_EXAMPLES_MAX = 50;
|
|
65
|
+
const DECISION_HISTORY_MAX = 30;
|
|
66
|
+
const normalizeFailureReason = (value) => {
|
|
67
|
+
if (!value)
|
|
68
|
+
return undefined;
|
|
69
|
+
const lower = value.toLowerCase();
|
|
70
|
+
if (lower === GATEWAY_FAILED_REASON || lower.startsWith(`${GATEWAY_FAILED_REASON}:`)) {
|
|
71
|
+
return GATEWAY_FAILED_REASON;
|
|
72
|
+
}
|
|
73
|
+
if (AUTH_ERROR_PATTERNS.some((pattern) => pattern.test(lower))) {
|
|
74
|
+
return AUTH_ERROR_REASON;
|
|
75
|
+
}
|
|
76
|
+
return lower.trim();
|
|
77
|
+
};
|
|
78
|
+
const parseGuardrailReason = (value) => {
|
|
79
|
+
const raw = value?.trim();
|
|
80
|
+
if (!raw)
|
|
81
|
+
return undefined;
|
|
82
|
+
const match = GUARDRAIL_REASON_PATTERN.exec(raw) ?? GUARDRAIL_REASON_ALT_PATTERN.exec(raw);
|
|
83
|
+
if (!match)
|
|
84
|
+
return undefined;
|
|
85
|
+
const disposition = match[1].toLowerCase();
|
|
86
|
+
const reason = normalizeFailureReason(match[2]) ?? match[2].toLowerCase();
|
|
87
|
+
if (!reason)
|
|
88
|
+
return undefined;
|
|
89
|
+
return {
|
|
90
|
+
retryable: disposition === "retryable",
|
|
91
|
+
reason,
|
|
92
|
+
};
|
|
93
|
+
};
|
|
28
94
|
export class GatewayTrioService {
|
|
29
95
|
constructor(workspace, deps) {
|
|
30
96
|
this.workspace = workspace;
|
|
@@ -37,6 +103,7 @@ export class GatewayTrioService {
|
|
|
37
103
|
const workspaceRepo = await WorkspaceRepository.create(workspace.workspaceRoot);
|
|
38
104
|
const jobService = new JobService(workspace, workspaceRepo);
|
|
39
105
|
const gatewayService = await GatewayAgentService.create(workspace);
|
|
106
|
+
const routingService = await RoutingService.create();
|
|
40
107
|
const workService = await WorkOnTasksService.create(workspace);
|
|
41
108
|
const reviewService = await CodeReviewService.create(workspace);
|
|
42
109
|
const qaService = await QaTasksService.create(workspace, { noTelemetry: options.noTelemetry ?? false });
|
|
@@ -45,6 +112,7 @@ export class GatewayTrioService {
|
|
|
45
112
|
workspaceRepo,
|
|
46
113
|
jobService,
|
|
47
114
|
gatewayService,
|
|
115
|
+
routingService,
|
|
48
116
|
workService,
|
|
49
117
|
reviewService,
|
|
50
118
|
qaService,
|
|
@@ -63,14 +131,76 @@ export class GatewayTrioService {
|
|
|
63
131
|
};
|
|
64
132
|
await maybeClose(this.selectionService);
|
|
65
133
|
await maybeClose(this.deps.gatewayService);
|
|
134
|
+
await maybeClose(this.deps.routingService);
|
|
66
135
|
await maybeClose(this.deps.workService);
|
|
67
136
|
await maybeClose(this.deps.reviewService);
|
|
68
137
|
await maybeClose(this.deps.qaService);
|
|
69
138
|
await maybeClose(this.deps.jobService);
|
|
70
139
|
await maybeClose(this.deps.workspaceRepo);
|
|
71
140
|
}
|
|
141
|
+
disableDocdex(reason) {
|
|
142
|
+
this.deps.gatewayService.setDocdexAvailability(false, reason);
|
|
143
|
+
this.deps.workService.setDocdexAvailability(false, reason);
|
|
144
|
+
this.deps.reviewService.setDocdexAvailability(false, reason);
|
|
145
|
+
this.deps.qaService.setDocdexAvailability(false, reason);
|
|
146
|
+
}
|
|
147
|
+
async writeDocdexCheckArtifact(jobId, payload) {
|
|
148
|
+
const dir = path.join(this.trioDir(jobId), "docdex");
|
|
149
|
+
await PathHelper.ensureDir(dir);
|
|
150
|
+
const target = path.join(dir, "docdex-check.json");
|
|
151
|
+
await fs.writeFile(target, JSON.stringify(payload, null, 2), "utf8");
|
|
152
|
+
return path.relative(this.workspace.mcodaDir, target);
|
|
153
|
+
}
|
|
154
|
+
async runDocdexPreflight(jobId, warnings) {
|
|
155
|
+
const usingCustomCheck = Boolean(this.deps.docdexCheck);
|
|
156
|
+
if (!usingCustomCheck &&
|
|
157
|
+
(process.env.MCODA_SKIP_DOCDEX_CHECKS === "1" || process.env.MCODA_SKIP_DOCDEX_RUNTIME_CHECKS === "1")) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
const configuredUrl = this.workspace.config?.docdexUrl ?? process.env.MCODA_DOCDEX_URL ?? process.env.DOCDEX_URL;
|
|
161
|
+
if (configuredUrl && !usingCustomCheck) {
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const checkFn = this.deps.docdexCheck ?? readDocdexCheck;
|
|
165
|
+
let summary;
|
|
166
|
+
let artifactPath;
|
|
167
|
+
try {
|
|
168
|
+
const check = await checkFn({ cwd: this.workspace.workspaceRoot });
|
|
169
|
+
summary = summarizeDocdexCheck(check);
|
|
170
|
+
artifactPath = await this.writeDocdexCheckArtifact(jobId, {
|
|
171
|
+
ok: summary.ok,
|
|
172
|
+
summary,
|
|
173
|
+
check,
|
|
174
|
+
timestamp: new Date().toISOString(),
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
179
|
+
summary = { ok: false, message };
|
|
180
|
+
artifactPath = await this.writeDocdexCheckArtifact(jobId, {
|
|
181
|
+
ok: false,
|
|
182
|
+
error: message,
|
|
183
|
+
timestamp: new Date().toISOString(),
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
187
|
+
stage: "docdex:check",
|
|
188
|
+
timestamp: new Date().toISOString(),
|
|
189
|
+
details: {
|
|
190
|
+
ok: summary?.ok ?? false,
|
|
191
|
+
message: summary?.message,
|
|
192
|
+
artifactPath,
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
if (!summary?.ok) {
|
|
196
|
+
const hint = "Run `docdex check` to diagnose; ensure docdexd and ollama are running.";
|
|
197
|
+
const detail = summary?.message ? `Docdex unavailable: ${summary.message}.` : "Docdex unavailable.";
|
|
198
|
+
warnings.push(artifactPath ? `${detail} ${hint} (artifact: ${artifactPath})` : `${detail} ${hint}`);
|
|
199
|
+
this.disableDocdex(summary?.message ?? "docdex unavailable");
|
|
200
|
+
}
|
|
201
|
+
}
|
|
72
202
|
trioDir(jobId) {
|
|
73
|
-
return path.join(this.workspace.
|
|
203
|
+
return path.join(this.workspace.mcodaDir, "jobs", jobId, "gateway-trio");
|
|
74
204
|
}
|
|
75
205
|
statePath(jobId) {
|
|
76
206
|
return path.join(this.trioDir(jobId), "state.json");
|
|
@@ -89,7 +219,7 @@ export class GatewayTrioService {
|
|
|
89
219
|
}
|
|
90
220
|
}
|
|
91
221
|
async readManifest(jobId) {
|
|
92
|
-
const manifestPath = path.join(this.workspace.
|
|
222
|
+
const manifestPath = path.join(this.workspace.mcodaDir, "jobs", jobId, "manifest.json");
|
|
93
223
|
try {
|
|
94
224
|
const raw = await fs.readFile(manifestPath, "utf8");
|
|
95
225
|
return JSON.parse(raw);
|
|
@@ -98,6 +228,78 @@ export class GatewayTrioService {
|
|
|
98
228
|
return undefined;
|
|
99
229
|
}
|
|
100
230
|
}
|
|
231
|
+
isPseudoTaskKey(key) {
|
|
232
|
+
return key.trim().toUpperCase().startsWith(PSEUDO_TASK_PREFIX);
|
|
233
|
+
}
|
|
234
|
+
skipPseudoTask(state, taskKey, warnings) {
|
|
235
|
+
const progress = this.ensureProgress(state, taskKey);
|
|
236
|
+
progress.status = "skipped";
|
|
237
|
+
progress.lastError = "pseudo_task";
|
|
238
|
+
state.tasks[taskKey] = progress;
|
|
239
|
+
warnings.push(`Skipping pseudo task ${taskKey}.`);
|
|
240
|
+
}
|
|
241
|
+
async isTokenUsageCheckEnabled() {
|
|
242
|
+
if (this.tokenUsageCheckEnabled !== undefined)
|
|
243
|
+
return this.tokenUsageCheckEnabled;
|
|
244
|
+
const configPath = path.join(this.workspace.mcodaDir, "config.json");
|
|
245
|
+
try {
|
|
246
|
+
const raw = await fs.readFile(configPath, "utf8");
|
|
247
|
+
const parsed = JSON.parse(raw);
|
|
248
|
+
this.tokenUsageCheckEnabled = !parsed?.telemetry?.strict;
|
|
249
|
+
}
|
|
250
|
+
catch {
|
|
251
|
+
this.tokenUsageCheckEnabled = true;
|
|
252
|
+
}
|
|
253
|
+
return this.tokenUsageCheckEnabled;
|
|
254
|
+
}
|
|
255
|
+
async isZeroTokenRun(jobId, commandRunId) {
|
|
256
|
+
if (!jobId)
|
|
257
|
+
return undefined;
|
|
258
|
+
const enabled = await this.isTokenUsageCheckEnabled();
|
|
259
|
+
if (!enabled)
|
|
260
|
+
return undefined;
|
|
261
|
+
const tokenPath = path.join(this.workspace.mcodaDir, "token_usage.json");
|
|
262
|
+
let raw;
|
|
263
|
+
try {
|
|
264
|
+
raw = await fs.readFile(tokenPath, "utf8");
|
|
265
|
+
}
|
|
266
|
+
catch {
|
|
267
|
+
return undefined;
|
|
268
|
+
}
|
|
269
|
+
let entries;
|
|
270
|
+
try {
|
|
271
|
+
const parsed = JSON.parse(raw);
|
|
272
|
+
entries = Array.isArray(parsed) ? parsed : [];
|
|
273
|
+
}
|
|
274
|
+
catch {
|
|
275
|
+
return undefined;
|
|
276
|
+
}
|
|
277
|
+
if (!entries.length)
|
|
278
|
+
return undefined;
|
|
279
|
+
const relevant = entries.filter((entry) => {
|
|
280
|
+
if (!entry)
|
|
281
|
+
return false;
|
|
282
|
+
if (entry.jobId === jobId)
|
|
283
|
+
return true;
|
|
284
|
+
if (commandRunId && entry.commandRunId === commandRunId)
|
|
285
|
+
return true;
|
|
286
|
+
return false;
|
|
287
|
+
});
|
|
288
|
+
if (relevant.length === 0)
|
|
289
|
+
return undefined;
|
|
290
|
+
const total = relevant.reduce((sum, entry) => {
|
|
291
|
+
const prompt = Number(entry.tokensPrompt ?? entry.tokens_prompt ?? 0);
|
|
292
|
+
const completion = Number(entry.tokensCompletion ?? entry.tokens_completion ?? 0);
|
|
293
|
+
const rawTotal = entry.tokensTotal ?? entry.tokens_total;
|
|
294
|
+
const entryTotal = Number.isFinite(rawTotal) ? Number(rawTotal) : prompt + completion;
|
|
295
|
+
return sum + (Number.isFinite(entryTotal) ? entryTotal : 0);
|
|
296
|
+
}, 0);
|
|
297
|
+
return total <= 0;
|
|
298
|
+
}
|
|
299
|
+
async backoffZeroTokens(attempts) {
|
|
300
|
+
const backoffMs = ZERO_TOKEN_BACKOFF_MS * Math.max(1, Math.min(attempts, 2));
|
|
301
|
+
await new Promise((resolve) => setTimeout(resolve, backoffMs));
|
|
302
|
+
}
|
|
101
303
|
async cleanupExpiredTaskLocks(warnings) {
|
|
102
304
|
const cleared = await this.deps.workspaceRepo.cleanupExpiredTaskLocks();
|
|
103
305
|
if (cleared.length > 0) {
|
|
@@ -178,6 +380,12 @@ export class GatewayTrioService {
|
|
|
178
380
|
existing.chosenAgents = {};
|
|
179
381
|
if (!existing.failureHistory)
|
|
180
382
|
existing.failureHistory = [];
|
|
383
|
+
if (!existing.escalationAttempts)
|
|
384
|
+
existing.escalationAttempts = {};
|
|
385
|
+
if (!existing.stepOutcomes)
|
|
386
|
+
existing.stepOutcomes = {};
|
|
387
|
+
if (!existing.decisionHistory)
|
|
388
|
+
existing.decisionHistory = [];
|
|
181
389
|
return existing;
|
|
182
390
|
}
|
|
183
391
|
const created = {
|
|
@@ -186,10 +394,81 @@ export class GatewayTrioService {
|
|
|
186
394
|
status: "pending",
|
|
187
395
|
chosenAgents: {},
|
|
188
396
|
failureHistory: [],
|
|
397
|
+
escalationAttempts: {},
|
|
398
|
+
stepOutcomes: {},
|
|
399
|
+
decisionHistory: [],
|
|
189
400
|
};
|
|
190
401
|
state.tasks[taskKey] = created;
|
|
191
402
|
return created;
|
|
192
403
|
}
|
|
404
|
+
dedupeTaskKeys(keys) {
|
|
405
|
+
const seen = new Set();
|
|
406
|
+
const result = [];
|
|
407
|
+
for (const key of keys) {
|
|
408
|
+
const trimmed = key.trim();
|
|
409
|
+
if (!trimmed || seen.has(trimmed))
|
|
410
|
+
continue;
|
|
411
|
+
seen.add(trimmed);
|
|
412
|
+
result.push(trimmed);
|
|
413
|
+
}
|
|
414
|
+
return result;
|
|
415
|
+
}
|
|
416
|
+
buildRunListFromExplicit(taskKeys, limit, warnings) {
|
|
417
|
+
const filtered = taskKeys.filter((key) => !this.isPseudoTaskKey(key));
|
|
418
|
+
let deduped = this.dedupeTaskKeys(filtered);
|
|
419
|
+
if (typeof limit === "number" && limit > 0 && deduped.length > limit) {
|
|
420
|
+
warnings.push(`Run list limited to ${limit} explicit tasks; skipping ${deduped.length - limit} task(s).`);
|
|
421
|
+
deduped = deduped.slice(0, limit);
|
|
422
|
+
}
|
|
423
|
+
return deduped;
|
|
424
|
+
}
|
|
425
|
+
async buildRunListFromSelection(filters, limit, warnings) {
|
|
426
|
+
const selection = await this.selectionService.selectTasks(filters);
|
|
427
|
+
if (selection.warnings.length)
|
|
428
|
+
warnings.push(...selection.warnings);
|
|
429
|
+
const orderedKeys = selection.ordered.map((entry) => entry.task.key).filter((key) => !this.isPseudoTaskKey(key));
|
|
430
|
+
let deduped = this.dedupeTaskKeys(orderedKeys);
|
|
431
|
+
if (typeof limit === "number" && limit > 0 && deduped.length > limit) {
|
|
432
|
+
warnings.push(`Run list limited to ${limit} tasks; skipping ${deduped.length - limit} task(s).`);
|
|
433
|
+
deduped = deduped.slice(0, limit);
|
|
434
|
+
}
|
|
435
|
+
return deduped;
|
|
436
|
+
}
|
|
437
|
+
async guardMissingContext(step, jobId, taskKey, gateway, warnings, resolvedAgent) {
|
|
438
|
+
const filesMissing = gateway.analysis.filesLikelyTouched.length === 0 && gateway.analysis.filesToCreate.length === 0;
|
|
439
|
+
const docdexMissing = gateway.docdex.length === 0;
|
|
440
|
+
const docdexNotesText = gateway.analysis.docdexNotes.join(" ").toLowerCase();
|
|
441
|
+
const docdexSkipped = DOCDEX_SKIP_PATTERN.test(docdexNotesText);
|
|
442
|
+
const docdexExplicitMissing = DOCDEX_MISSING_PATTERN.test(docdexNotesText);
|
|
443
|
+
const missingTaskSignal = this.hasMissingTaskSignal(gateway.analysis);
|
|
444
|
+
if (!missingTaskSignal && (!filesMissing || !docdexMissing) && !(docdexSkipped && !docdexExplicitMissing)) {
|
|
445
|
+
return undefined;
|
|
446
|
+
}
|
|
447
|
+
const messageLines = [
|
|
448
|
+
missingTaskSignal
|
|
449
|
+
? "Gateway analysis indicates the task details are missing; proceeding with execution anyway."
|
|
450
|
+
: docdexSkipped && !docdexExplicitMissing
|
|
451
|
+
? "Gateway analysis reported docdex work as not executed; proceeding without docdex context."
|
|
452
|
+
: "Gateway analysis returned no file paths and no docdex context; proceeding.",
|
|
453
|
+
gateway.analysis.docdexNotes.length ? `Docdex notes: ${gateway.analysis.docdexNotes.join(" | ")}` : "Docdex notes: (none)",
|
|
454
|
+
gateway.analysis.assumptions.length ? `Assumptions: ${gateway.analysis.assumptions.join(" | ")}` : undefined,
|
|
455
|
+
].filter(Boolean);
|
|
456
|
+
warnings.push(`Task ${taskKey} (${step}) gateway context incomplete.\n${messageLines.join("\n")}`);
|
|
457
|
+
return undefined;
|
|
458
|
+
}
|
|
459
|
+
hasMissingTaskSignal(analysis) {
|
|
460
|
+
const fields = [
|
|
461
|
+
analysis.summary,
|
|
462
|
+
analysis.currentState,
|
|
463
|
+
analysis.todo,
|
|
464
|
+
analysis.understanding,
|
|
465
|
+
...(analysis.assumptions ?? []),
|
|
466
|
+
...(analysis.docdexNotes ?? []),
|
|
467
|
+
]
|
|
468
|
+
.filter(Boolean)
|
|
469
|
+
.join(" ");
|
|
470
|
+
return MISSING_TASK_PATTERN.test(fields);
|
|
471
|
+
}
|
|
193
472
|
hasReachedMaxIterations(progress, maxIterations) {
|
|
194
473
|
if (maxIterations === undefined)
|
|
195
474
|
return false;
|
|
@@ -201,43 +480,102 @@ export class GatewayTrioService {
|
|
|
201
480
|
return true;
|
|
202
481
|
return progress.attempts < maxIterations;
|
|
203
482
|
}
|
|
204
|
-
|
|
483
|
+
shouldReopenFailedTask(progress, taskKey, warnings, continuousMode) {
|
|
484
|
+
if (!progress)
|
|
485
|
+
return true;
|
|
486
|
+
if (continuousMode)
|
|
487
|
+
return true;
|
|
488
|
+
if (progress.lastGuardrailRetryable === false) {
|
|
489
|
+
const lastFailure = progress.failureHistory?.[progress.failureHistory.length - 1];
|
|
490
|
+
const lastReasonRaw = progress.lastError ?? lastFailure?.reason ?? "guardrail_non_retryable";
|
|
491
|
+
warnings.push(`Task ${taskKey} failed with non-retryable reason ${lastReasonRaw}; skipping reopen.`);
|
|
492
|
+
return false;
|
|
493
|
+
}
|
|
494
|
+
const lastFailure = progress.failureHistory?.[progress.failureHistory.length - 1];
|
|
495
|
+
const lastReasonRaw = progress.lastError ?? lastFailure?.reason ?? "";
|
|
496
|
+
const lastReason = normalizeFailureReason(lastReasonRaw);
|
|
497
|
+
if (lastReason) {
|
|
498
|
+
if (NON_RETRYABLE_FAILURE_REASONS.has(lastReason)) {
|
|
499
|
+
warnings.push(`Task ${taskKey} failed with non-retryable reason ${lastReason}; skipping reopen.`);
|
|
500
|
+
return false;
|
|
501
|
+
}
|
|
502
|
+
if (/no eligible agents|missing required capabilities|agent .* missing required capabilities/i.test(lastReasonRaw)) {
|
|
503
|
+
warnings.push(`Task ${taskKey} failed due to agent selection; skipping reopen.`);
|
|
504
|
+
return false;
|
|
505
|
+
}
|
|
506
|
+
const sameReasonCount = progress.failureHistory?.filter((failure) => failure.reason === lastReason).length ?? 0;
|
|
507
|
+
if (sameReasonCount >= MAX_FAILURE_REOPENS_PER_REASON) {
|
|
508
|
+
warnings.push(`Task ${taskKey} hit retry cap for ${lastReason}; skipping reopen.`);
|
|
509
|
+
return false;
|
|
510
|
+
}
|
|
511
|
+
const lastTimestamp = lastFailure?.timestamp ? Date.parse(lastFailure.timestamp) : undefined;
|
|
512
|
+
if (Number.isFinite(lastTimestamp) && Date.now() - lastTimestamp < FAILED_REOPEN_COOLDOWN_MS) {
|
|
513
|
+
warnings.push(`Task ${taskKey} failed recently (${lastReason}); cooling down before reopen.`);
|
|
514
|
+
return false;
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
return true;
|
|
518
|
+
}
|
|
519
|
+
async reopenRetryableFailedTasks(state, explicitTasks, maxIterations, warnings) {
|
|
520
|
+
const continuousMode = maxIterations === undefined;
|
|
205
521
|
const keys = new Set([...explicitTasks, ...Object.keys(state.tasks)]);
|
|
206
522
|
for (const taskKey of keys) {
|
|
207
523
|
const progress = state.tasks[taskKey];
|
|
208
524
|
if (progress?.status === "completed")
|
|
209
525
|
continue;
|
|
210
|
-
if (this.hasReachedMaxIterations(progress, maxIterations))
|
|
211
|
-
continue;
|
|
212
526
|
const task = await this.deps.workspaceRepo.getTaskByKey(taskKey);
|
|
213
527
|
if (!task)
|
|
214
528
|
continue;
|
|
215
529
|
const status = this.normalizeStatus(task.status);
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
530
|
+
if (status === "completed" || status === "cancelled") {
|
|
531
|
+
const terminalProgress = progress ?? this.ensureProgress(state, taskKey);
|
|
532
|
+
terminalProgress.status = status === "completed" ? "completed" : "skipped";
|
|
533
|
+
terminalProgress.lastError = status === "completed" ? "completed_in_db" : "cancelled_in_db";
|
|
534
|
+
state.tasks[taskKey] = terminalProgress;
|
|
535
|
+
warnings.push(`Task ${taskKey} is ${status}; skipping reopen.`);
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
if (this.hasReachedMaxIterations(progress, maxIterations)) {
|
|
539
|
+
if (progress) {
|
|
540
|
+
progress.status = "failed";
|
|
541
|
+
if (!progress.lastError) {
|
|
542
|
+
progress.lastError = "max_iterations_reached";
|
|
543
|
+
}
|
|
544
|
+
state.tasks[taskKey] = progress;
|
|
545
|
+
}
|
|
546
|
+
if (maxIterations !== undefined) {
|
|
547
|
+
warnings.push(`Task ${taskKey} hit max iterations (${maxIterations}); skipping reopen.`);
|
|
225
548
|
}
|
|
226
|
-
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
const metadata = task.metadata ?? {};
|
|
552
|
+
const failedReason = typeof metadata.failed_reason === "string" ? metadata.failed_reason : undefined;
|
|
553
|
+
if (status === "failed") {
|
|
554
|
+
if (failedReason === "dependency_not_ready") {
|
|
227
555
|
const depsReady = await this.dependenciesReady(task.id, warnings);
|
|
228
556
|
if (!depsReady)
|
|
229
557
|
continue;
|
|
230
558
|
}
|
|
231
|
-
|
|
559
|
+
if (!continuousMode &&
|
|
560
|
+
(progress?.lastGuardrailRetryable === false ||
|
|
561
|
+
(failedReason && NON_RETRYABLE_FAILURE_REASONS.has(failedReason)))) {
|
|
562
|
+
const reasonLabel = failedReason ?? progress?.lastError ?? progress?.lastEscalationReason ?? "guardrail_non_retryable";
|
|
563
|
+
warnings.push(`Task ${taskKey} failed with non-retryable reason ${reasonLabel}; skipping reopen.`);
|
|
564
|
+
continue;
|
|
565
|
+
}
|
|
566
|
+
if (!continuousMode && progress && !this.shouldReopenFailedTask(progress, taskKey, warnings, continuousMode)) {
|
|
232
567
|
continue;
|
|
233
568
|
}
|
|
234
569
|
}
|
|
235
570
|
else if (progress?.status !== "failed") {
|
|
236
571
|
continue;
|
|
237
572
|
}
|
|
573
|
+
else if (!continuousMode && !this.shouldReopenFailedTask(progress, taskKey, warnings, continuousMode)) {
|
|
574
|
+
continue;
|
|
575
|
+
}
|
|
238
576
|
const nextMetadata = { ...metadata };
|
|
239
|
-
delete nextMetadata.
|
|
240
|
-
if (status === "
|
|
577
|
+
delete nextMetadata.failed_reason;
|
|
578
|
+
if (status === "failed") {
|
|
241
579
|
await this.deps.workspaceRepo.updateTask(task.id, {
|
|
242
580
|
status: "in_progress",
|
|
243
581
|
metadata: nextMetadata,
|
|
@@ -248,9 +586,7 @@ export class GatewayTrioService {
|
|
|
248
586
|
progress.lastError = undefined;
|
|
249
587
|
state.tasks[taskKey] = progress;
|
|
250
588
|
}
|
|
251
|
-
warnings.push(
|
|
252
|
-
? `Reopened blocked task ${taskKey} (reason=${blockedReason}) for retry.`
|
|
253
|
-
: `Reopened failed task ${taskKey} for retry (attempts=${progress?.attempts ?? 0}${maxIterations !== undefined ? `/${maxIterations}` : ""}).`);
|
|
589
|
+
warnings.push(`Reopened failed task ${taskKey} (reason=${failedReason ?? progress?.lastError ?? "unknown"}) for retry (attempts=${progress?.attempts ?? 0}${maxIterations !== undefined ? `/${maxIterations}` : ""}).`);
|
|
254
590
|
}
|
|
255
591
|
}
|
|
256
592
|
async dependenciesReady(taskId, warnings) {
|
|
@@ -278,11 +614,14 @@ export class GatewayTrioService {
|
|
|
278
614
|
return status ? status.toLowerCase().trim() : undefined;
|
|
279
615
|
}
|
|
280
616
|
resolveRequest(request, payload) {
|
|
281
|
-
if (!payload)
|
|
282
|
-
return request;
|
|
617
|
+
if (!payload) {
|
|
618
|
+
return { ...request, rateAgents: request.rateAgents ?? true };
|
|
619
|
+
}
|
|
283
620
|
const raw = payload;
|
|
284
621
|
const payloadTasks = Array.isArray(raw.tasks) ? raw.tasks : undefined;
|
|
285
622
|
const payloadStatuses = Array.isArray(raw.statusFilter) ? raw.statusFilter : undefined;
|
|
623
|
+
const resolvedQaResult = (request.qaResult ?? raw.qaResult);
|
|
624
|
+
const normalizedQaResult = resolvedQaResult === "blocked" ? "fail" : resolvedQaResult;
|
|
286
625
|
return {
|
|
287
626
|
...request,
|
|
288
627
|
projectKey: request.projectKey ?? raw.projectKey,
|
|
@@ -300,6 +639,7 @@ export class GatewayTrioService {
|
|
|
300
639
|
qaAgentName: request.qaAgentName ?? raw.qaAgentName,
|
|
301
640
|
maxDocs: request.maxDocs ?? raw.maxDocs,
|
|
302
641
|
agentStream: request.agentStream ?? raw.agentStream,
|
|
642
|
+
rateAgents: request.rateAgents ?? raw.rateAgents ?? true,
|
|
303
643
|
noCommit: request.noCommit ?? raw.noCommit,
|
|
304
644
|
dryRun: request.dryRun ?? raw.dryRun,
|
|
305
645
|
reviewBase: request.reviewBase ?? raw.reviewBase,
|
|
@@ -309,11 +649,15 @@ export class GatewayTrioService {
|
|
|
309
649
|
qaTestCommand: request.qaTestCommand ?? raw.qaTestCommand,
|
|
310
650
|
qaMode: request.qaMode ?? raw.qaMode,
|
|
311
651
|
qaFollowups: request.qaFollowups ?? raw.qaFollowups,
|
|
312
|
-
|
|
652
|
+
reviewFollowups: request.reviewFollowups ?? raw.reviewFollowups,
|
|
653
|
+
qaResult: normalizedQaResult,
|
|
313
654
|
qaNotes: request.qaNotes ?? raw.qaNotes,
|
|
314
655
|
qaEvidenceUrl: request.qaEvidenceUrl ?? raw.qaEvidenceUrl,
|
|
315
656
|
qaAllowDirty: request.qaAllowDirty ?? raw.qaAllowDirty,
|
|
316
657
|
escalateOnNoChange: request.escalateOnNoChange ?? raw.escalateOnNoChange,
|
|
658
|
+
workRunner: request.workRunner ?? raw.workRunner,
|
|
659
|
+
useCodali: request.useCodali ?? raw.useCodali,
|
|
660
|
+
agentAdapterOverride: request.agentAdapterOverride ?? raw.agentAdapterOverride,
|
|
317
661
|
};
|
|
318
662
|
}
|
|
319
663
|
async buildStatusFilter(request, warnings) {
|
|
@@ -351,16 +695,26 @@ export class GatewayTrioService {
|
|
|
351
695
|
if (!entry) {
|
|
352
696
|
return { step: "work", status: "failed", error: "Task not processed by work-on-tasks" };
|
|
353
697
|
}
|
|
698
|
+
const guardrail = parseGuardrailReason(entry.notes);
|
|
354
699
|
if (entry.status === "succeeded") {
|
|
355
|
-
if (entry.notes === "no_changes")
|
|
356
|
-
return { step: "work", status: "failed", error: "no_changes" };
|
|
357
700
|
return { step: "work", status: "succeeded" };
|
|
358
701
|
}
|
|
359
|
-
if (entry.status === "
|
|
360
|
-
return {
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
702
|
+
if (entry.status === "skipped") {
|
|
703
|
+
return {
|
|
704
|
+
step: "work",
|
|
705
|
+
status: "skipped",
|
|
706
|
+
error: guardrail?.reason ?? entry.notes,
|
|
707
|
+
guardrailReason: guardrail?.reason,
|
|
708
|
+
guardrailRetryable: guardrail?.retryable,
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
return {
|
|
712
|
+
step: "work",
|
|
713
|
+
status: "failed",
|
|
714
|
+
error: guardrail?.reason ?? entry.notes,
|
|
715
|
+
guardrailReason: guardrail?.reason,
|
|
716
|
+
guardrailRetryable: guardrail?.retryable,
|
|
717
|
+
};
|
|
364
718
|
}
|
|
365
719
|
parseReviewResult(taskKey, result) {
|
|
366
720
|
const entry = result.tasks.find((t) => t.taskKey === taskKey);
|
|
@@ -368,13 +722,22 @@ export class GatewayTrioService {
|
|
|
368
722
|
return { step: "review", status: "failed", error: "Task not processed by code-review" };
|
|
369
723
|
}
|
|
370
724
|
if (entry.error) {
|
|
371
|
-
|
|
725
|
+
const guardrail = parseGuardrailReason(entry.error);
|
|
726
|
+
return {
|
|
727
|
+
step: "review",
|
|
728
|
+
status: "failed",
|
|
729
|
+
error: guardrail?.reason ?? entry.error,
|
|
730
|
+
guardrailReason: guardrail?.reason,
|
|
731
|
+
guardrailRetryable: guardrail?.retryable,
|
|
732
|
+
};
|
|
372
733
|
}
|
|
373
734
|
const decision = entry.decision ?? "error";
|
|
374
735
|
if (decision === "approve" || decision === "info_only")
|
|
375
736
|
return { step: "review", status: "succeeded", decision };
|
|
737
|
+
if (decision === "changes_requested")
|
|
738
|
+
return { step: "review", status: "succeeded", decision };
|
|
376
739
|
if (decision === "block")
|
|
377
|
-
return { step: "review", status: "
|
|
740
|
+
return { step: "review", status: "failed", decision };
|
|
378
741
|
return { step: "review", status: "failed", decision };
|
|
379
742
|
}
|
|
380
743
|
parseQaResult(taskKey, result) {
|
|
@@ -382,17 +745,56 @@ export class GatewayTrioService {
|
|
|
382
745
|
if (!entry) {
|
|
383
746
|
return { step: "qa", status: "failed", error: "Task not processed by qa-tasks" };
|
|
384
747
|
}
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
748
|
+
const guardrail = parseGuardrailReason(entry.outcome);
|
|
749
|
+
if (entry.outcome === "pass") {
|
|
750
|
+
return {
|
|
751
|
+
step: "qa",
|
|
752
|
+
status: "succeeded",
|
|
753
|
+
outcome: entry.outcome,
|
|
754
|
+
notes: entry.notes,
|
|
755
|
+
artifacts: entry.artifacts,
|
|
756
|
+
};
|
|
757
|
+
}
|
|
758
|
+
if (entry.outcome === "infra_issue") {
|
|
759
|
+
return {
|
|
760
|
+
step: "qa",
|
|
761
|
+
status: "failed",
|
|
762
|
+
outcome: entry.outcome,
|
|
763
|
+
notes: entry.notes,
|
|
764
|
+
artifacts: entry.artifacts,
|
|
765
|
+
};
|
|
766
|
+
}
|
|
389
767
|
if (entry.outcome === "fix_required" || entry.outcome === "unclear") {
|
|
390
|
-
return {
|
|
768
|
+
return {
|
|
769
|
+
step: "qa",
|
|
770
|
+
status: "failed",
|
|
771
|
+
outcome: entry.outcome,
|
|
772
|
+
notes: entry.notes,
|
|
773
|
+
artifacts: entry.artifacts,
|
|
774
|
+
};
|
|
391
775
|
}
|
|
392
|
-
return {
|
|
776
|
+
return {
|
|
777
|
+
step: "qa",
|
|
778
|
+
status: "failed",
|
|
779
|
+
outcome: guardrail?.reason ?? entry.outcome,
|
|
780
|
+
notes: entry.notes,
|
|
781
|
+
artifacts: entry.artifacts,
|
|
782
|
+
guardrailReason: guardrail?.reason,
|
|
783
|
+
guardrailRetryable: guardrail?.retryable,
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
isAuthFailure(reason) {
|
|
787
|
+
return normalizeFailureReason(reason) === AUTH_ERROR_REASON;
|
|
393
788
|
}
|
|
394
789
|
shouldRetryAfter(step) {
|
|
395
|
-
if (step.status === "
|
|
790
|
+
if (step.status === "skipped")
|
|
791
|
+
return false;
|
|
792
|
+
if (typeof step.guardrailRetryable === "boolean")
|
|
793
|
+
return step.guardrailRetryable;
|
|
794
|
+
const reason = normalizeFailureReason(step.error ?? step.decision ?? step.outcome ?? "");
|
|
795
|
+
if (reason === "infra_issue" || reason === "block")
|
|
796
|
+
return false;
|
|
797
|
+
if (reason && NON_RETRYABLE_FAILURE_REASONS.has(reason))
|
|
396
798
|
return false;
|
|
397
799
|
return step.status !== "succeeded";
|
|
398
800
|
}
|
|
@@ -402,17 +804,196 @@ export class GatewayTrioService {
|
|
|
402
804
|
reasons.add(NO_CHANGE_REASON);
|
|
403
805
|
return reasons;
|
|
404
806
|
}
|
|
807
|
+
resolveTierFromComplexity(complexity) {
|
|
808
|
+
if (typeof complexity !== "number" || !Number.isFinite(complexity))
|
|
809
|
+
return "unknown";
|
|
810
|
+
const normalized = Math.round(complexity);
|
|
811
|
+
if (normalized >= SPECIALIST_TIER_MIN_COMPLEXITY)
|
|
812
|
+
return "specialist";
|
|
813
|
+
if (normalized >= STRONG_TIER_MIN_COMPLEXITY)
|
|
814
|
+
return "strong";
|
|
815
|
+
return "cheap";
|
|
816
|
+
}
|
|
817
|
+
nextTier(current) {
|
|
818
|
+
if (current === "strong")
|
|
819
|
+
return "specialist";
|
|
820
|
+
if (current === "specialist")
|
|
821
|
+
return undefined;
|
|
822
|
+
return "strong";
|
|
823
|
+
}
|
|
405
824
|
recordFailure(progress, step, attempt) {
|
|
406
|
-
if (step.status !== "failed"
|
|
825
|
+
if (step.status !== "failed")
|
|
407
826
|
return;
|
|
408
|
-
const reason = step.error ?? step.decision ?? step.outcome;
|
|
827
|
+
const reason = step.guardrailReason ?? step.error ?? step.decision ?? step.outcome;
|
|
409
828
|
const agent = step.chosenAgent;
|
|
410
|
-
if (
|
|
829
|
+
if (typeof step.guardrailRetryable === "boolean") {
|
|
830
|
+
progress.lastGuardrailRetryable = step.guardrailRetryable;
|
|
831
|
+
}
|
|
832
|
+
if (!reason)
|
|
833
|
+
return;
|
|
834
|
+
const normalized = normalizeFailureReason(reason) ?? reason;
|
|
835
|
+
progress.lastEscalationReason = normalized;
|
|
836
|
+
const attempts = progress.escalationAttempts ?? {};
|
|
837
|
+
attempts[normalized] = (attempts[normalized] ?? 0) + 1;
|
|
838
|
+
progress.escalationAttempts = attempts;
|
|
839
|
+
if (!agent)
|
|
411
840
|
return;
|
|
412
841
|
const history = progress.failureHistory ?? [];
|
|
413
842
|
history.push({ step: step.step, agent, reason, attempt, timestamp: new Date().toISOString() });
|
|
414
843
|
progress.failureHistory = history;
|
|
415
844
|
}
|
|
845
|
+
mergeGatewayFiles(outcome) {
|
|
846
|
+
const values = [...(outcome.gatewayFiles ?? [])];
|
|
847
|
+
const unique = new Set();
|
|
848
|
+
const merged = [];
|
|
849
|
+
for (const value of values) {
|
|
850
|
+
const trimmed = value.trim();
|
|
851
|
+
if (!trimmed || unique.has(trimmed))
|
|
852
|
+
continue;
|
|
853
|
+
unique.add(trimmed);
|
|
854
|
+
merged.push(trimmed);
|
|
855
|
+
}
|
|
856
|
+
return merged;
|
|
857
|
+
}
|
|
858
|
+
handoffContextForProgress(progress) {
|
|
859
|
+
const qaFailureSummary = progress.lastQaFailureSummary?.trim();
|
|
860
|
+
const learningSummary = progress.pendingLearningSummary?.trim();
|
|
861
|
+
if (!qaFailureSummary && !learningSummary)
|
|
862
|
+
return undefined;
|
|
863
|
+
return {
|
|
864
|
+
qaFailureSummary,
|
|
865
|
+
learningSummary,
|
|
866
|
+
};
|
|
867
|
+
}
|
|
868
|
+
appendFallbackHandoffContext(baseLines, context) {
|
|
869
|
+
if (!context?.qaFailureSummary?.trim() && !context?.learningSummary?.trim()) {
|
|
870
|
+
return baseLines;
|
|
871
|
+
}
|
|
872
|
+
const lines = [...baseLines];
|
|
873
|
+
if (context.qaFailureSummary?.trim()) {
|
|
874
|
+
lines.push("", "## QA Failure Summary", context.qaFailureSummary.trim());
|
|
875
|
+
}
|
|
876
|
+
if (context.learningSummary?.trim()) {
|
|
877
|
+
lines.push("", "## Revert Learning", context.learningSummary.trim());
|
|
878
|
+
}
|
|
879
|
+
return lines;
|
|
880
|
+
}
|
|
881
|
+
recordDecision(progress, step, attempt, kind, status, detail) {
|
|
882
|
+
const stepOutcomes = progress.stepOutcomes ?? {};
|
|
883
|
+
stepOutcomes[step] = detail ? `${status}:${detail}` : status;
|
|
884
|
+
progress.stepOutcomes = stepOutcomes;
|
|
885
|
+
const history = progress.decisionHistory ?? [];
|
|
886
|
+
history.push({
|
|
887
|
+
step,
|
|
888
|
+
kind,
|
|
889
|
+
status,
|
|
890
|
+
detail,
|
|
891
|
+
attempt,
|
|
892
|
+
timestamp: new Date().toISOString(),
|
|
893
|
+
});
|
|
894
|
+
progress.decisionHistory = history.slice(Math.max(0, history.length - DECISION_HISTORY_MAX));
|
|
895
|
+
}
|
|
896
|
+
buildQaFailureSummary(taskKey, outcome) {
|
|
897
|
+
const fragments = [];
|
|
898
|
+
fragments.push(`Task ${taskKey} QA outcome: ${outcome.outcome ?? outcome.status}.`);
|
|
899
|
+
if (outcome.error) {
|
|
900
|
+
fragments.push(`Error: ${outcome.error}.`);
|
|
901
|
+
}
|
|
902
|
+
if (outcome.notes) {
|
|
903
|
+
fragments.push(`Notes: ${outcome.notes}.`);
|
|
904
|
+
}
|
|
905
|
+
if (outcome.artifacts?.length) {
|
|
906
|
+
fragments.push(`Artifacts: ${outcome.artifacts.join(", ")}.`);
|
|
907
|
+
}
|
|
908
|
+
return fragments.join(" ").trim();
|
|
909
|
+
}
|
|
910
|
+
extractRevertLearning(taskKey, metadata) {
|
|
911
|
+
if (!metadata || typeof metadata !== "object")
|
|
912
|
+
return undefined;
|
|
913
|
+
const pending = metadata.revert_event_pending === true;
|
|
914
|
+
const rawEvent = metadata.last_revert_event;
|
|
915
|
+
if (!pending || !rawEvent || typeof rawEvent !== "object")
|
|
916
|
+
return undefined;
|
|
917
|
+
const reason = typeof rawEvent.reason === "string" && rawEvent.reason.trim().length > 0
|
|
918
|
+
? rawEvent.reason.trim()
|
|
919
|
+
: "No explicit reason provided.";
|
|
920
|
+
const from = typeof rawEvent.from_status === "string" ? rawEvent.from_status : "completed";
|
|
921
|
+
const to = typeof rawEvent.to_status === "string" ? rawEvent.to_status : "changes_requested";
|
|
922
|
+
const timestamp = typeof rawEvent.timestamp === "string" ? rawEvent.timestamp : undefined;
|
|
923
|
+
const summary = `Revert detected for ${taskKey}: status moved ${from} -> ${to}. Feedback: ${reason}`;
|
|
924
|
+
return { summary, eventAt: timestamp };
|
|
925
|
+
}
|
|
926
|
+
shouldSavePreference(summary) {
|
|
927
|
+
const lower = summary.toLowerCase();
|
|
928
|
+
return /\b(always|never|prefer|do not|don't|must)\b/.test(lower);
|
|
929
|
+
}
|
|
930
|
+
async persistRevertLearning(taskId, taskKey, metadata, progress, warnings) {
|
|
931
|
+
const learning = this.extractRevertLearning(taskKey, metadata);
|
|
932
|
+
if (!learning)
|
|
933
|
+
return;
|
|
934
|
+
progress.pendingLearningSummary = learning.summary;
|
|
935
|
+
progress.lastRevertEventAt = learning.eventAt ?? new Date().toISOString();
|
|
936
|
+
const gatewayWithLearning = this.deps.gatewayService;
|
|
937
|
+
try {
|
|
938
|
+
if (typeof gatewayWithLearning.saveRepoMemory === "function") {
|
|
939
|
+
await gatewayWithLearning.saveRepoMemory(learning.summary);
|
|
940
|
+
}
|
|
941
|
+
if (this.shouldSavePreference(learning.summary) && typeof gatewayWithLearning.savePreference === "function") {
|
|
942
|
+
await gatewayWithLearning.savePreference("constraint", learning.summary, "codex");
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
catch (error) {
|
|
946
|
+
warnings.push(`Revert learning persistence failed for ${taskKey}: ${error instanceof Error ? error.message : String(error)}`);
|
|
947
|
+
}
|
|
948
|
+
if (metadata) {
|
|
949
|
+
const nextMetadata = { ...metadata, revert_event_pending: false };
|
|
950
|
+
await this.deps.workspaceRepo.updateTask(taskId, { metadata: nextMetadata });
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
sanitizeGoldenText(value) {
|
|
954
|
+
return value
|
|
955
|
+
.replace(/AKIA[0-9A-Z]{16}/g, "[REDACTED_AWS_KEY]")
|
|
956
|
+
.replace(/\bsk-[A-Za-z0-9]{20,}\b/g, "[REDACTED_TOKEN]")
|
|
957
|
+
.replace(/Bearer\s+[A-Za-z0-9._-]+/gi, "Bearer [REDACTED]")
|
|
958
|
+
.trim();
|
|
959
|
+
}
|
|
960
|
+
async appendGoldenExample(entry) {
|
|
961
|
+
const targetPath = path.join(this.workspace.workspaceRoot, GOLDEN_EXAMPLES_REL_PATH);
|
|
962
|
+
await PathHelper.ensureDir(path.dirname(targetPath));
|
|
963
|
+
let existing = [];
|
|
964
|
+
try {
|
|
965
|
+
const raw = await fs.readFile(targetPath, "utf8");
|
|
966
|
+
existing = raw
|
|
967
|
+
.split(/\r?\n/)
|
|
968
|
+
.map((line) => line.trim())
|
|
969
|
+
.filter(Boolean)
|
|
970
|
+
.map((line) => {
|
|
971
|
+
try {
|
|
972
|
+
return JSON.parse(line);
|
|
973
|
+
}
|
|
974
|
+
catch {
|
|
975
|
+
return undefined;
|
|
976
|
+
}
|
|
977
|
+
})
|
|
978
|
+
.filter((item) => Boolean(item));
|
|
979
|
+
}
|
|
980
|
+
catch (error) {
|
|
981
|
+
if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT")) {
|
|
982
|
+
throw error;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
const next = {
|
|
986
|
+
intent: this.sanitizeGoldenText(entry.title ? `${entry.taskKey}: ${entry.title}` : entry.taskKey),
|
|
987
|
+
plan_summary: this.sanitizeGoldenText(entry.planSummary),
|
|
988
|
+
touched_files: entry.touchedFiles.map((file) => file.trim()).filter(Boolean),
|
|
989
|
+
review_notes: entry.reviewNotes ? this.sanitizeGoldenText(entry.reviewNotes) : undefined,
|
|
990
|
+
qa_notes: entry.qaNotes ? this.sanitizeGoldenText(entry.qaNotes) : undefined,
|
|
991
|
+
created_at: new Date().toISOString(),
|
|
992
|
+
};
|
|
993
|
+
const bounded = [...existing, next].slice(Math.max(0, existing.length + 1 - GOLDEN_EXAMPLES_MAX));
|
|
994
|
+
const payload = bounded.map((item) => JSON.stringify(item)).join("\n");
|
|
995
|
+
await fs.writeFile(targetPath, payload.length > 0 ? `${payload}\n` : "", "utf8");
|
|
996
|
+
}
|
|
416
997
|
countFailures(progress, step, reason) {
|
|
417
998
|
if (!progress?.failureHistory?.length)
|
|
418
999
|
return 0;
|
|
@@ -445,17 +1026,16 @@ export class GatewayTrioService {
|
|
|
445
1026
|
}
|
|
446
1027
|
buildAgentOptions(progress, step, request) {
|
|
447
1028
|
const reasons = this.escalationReasons(request.escalateOnNoChange !== false);
|
|
448
|
-
const
|
|
449
|
-
const history = progress.failureHistory ?? [];
|
|
450
|
-
for (const failure of history) {
|
|
1029
|
+
const history = (progress.failureHistory ?? []).filter((failure) => {
|
|
451
1030
|
if (failure.step !== step)
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
const
|
|
458
|
-
|
|
1031
|
+
return false;
|
|
1032
|
+
const normalized = normalizeFailureReason(failure.reason) ?? failure.reason;
|
|
1033
|
+
return reasons.has(normalized);
|
|
1034
|
+
});
|
|
1035
|
+
const forceStronger = history.length > 0;
|
|
1036
|
+
const forceTier = step === "work" && forceStronger ? this.nextTier(progress.lastWorkAgentTier) : undefined;
|
|
1037
|
+
const avoidAgents = history.length > 1 ? Array.from(new Set(history.map((failure) => failure.agent))) : [];
|
|
1038
|
+
return { avoidAgents, forceStronger, forceTier };
|
|
459
1039
|
}
|
|
460
1040
|
recordRating(progress, summary) {
|
|
461
1041
|
if (!summary)
|
|
@@ -469,7 +1049,7 @@ export class GatewayTrioService {
|
|
|
469
1049
|
if (!jobId)
|
|
470
1050
|
return undefined;
|
|
471
1051
|
try {
|
|
472
|
-
const payload = await fs.readFile(path.join(this.workspace.
|
|
1052
|
+
const payload = await fs.readFile(path.join(this.workspace.mcodaDir, "jobs", jobId, "rating.json"), "utf8");
|
|
473
1053
|
const parsed = JSON.parse(payload);
|
|
474
1054
|
const rating = typeof parsed.rating === "number" ? parsed.rating : undefined;
|
|
475
1055
|
const maxComplexity = typeof parsed.maxComplexity === "number" ? parsed.maxComplexity : undefined;
|
|
@@ -481,7 +1061,27 @@ export class GatewayTrioService {
|
|
|
481
1061
|
return undefined;
|
|
482
1062
|
}
|
|
483
1063
|
}
|
|
1064
|
+
async updateJobHeartbeat(params) {
|
|
1065
|
+
const timestamp = new Date().toISOString();
|
|
1066
|
+
const detail = `task:${params.taskKey} step:${params.step} attempt:${params.attempt} last:${timestamp}`;
|
|
1067
|
+
try {
|
|
1068
|
+
await this.deps.jobService.updateJobStatus(params.jobId, "running", {
|
|
1069
|
+
job_state_detail: detail,
|
|
1070
|
+
payload: {
|
|
1071
|
+
current_task: params.taskKey,
|
|
1072
|
+
current_step: params.step,
|
|
1073
|
+
attempt: params.attempt,
|
|
1074
|
+
last_activity: timestamp,
|
|
1075
|
+
activity: params.activity ?? "heartbeat",
|
|
1076
|
+
},
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
catch {
|
|
1080
|
+
// Avoid failing the run if heartbeat updates cannot be persisted.
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
484
1083
|
async runStepWithTimeout(step, jobId, taskKey, attempt, maxAgentSeconds, fn) {
|
|
1084
|
+
await this.updateJobHeartbeat({ jobId, taskKey, step, attempt, activity: "start" });
|
|
485
1085
|
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
486
1086
|
stage: `task:${taskKey}:${step}:start`,
|
|
487
1087
|
timestamp: new Date().toISOString(),
|
|
@@ -491,6 +1091,7 @@ export class GatewayTrioService {
|
|
|
491
1091
|
let timeoutHandle;
|
|
492
1092
|
const controller = new AbortController();
|
|
493
1093
|
const heartbeat = setInterval(() => {
|
|
1094
|
+
void this.updateJobHeartbeat({ jobId, taskKey, step, attempt, activity: "heartbeat" }).catch(() => { });
|
|
494
1095
|
void this.deps.jobService.writeCheckpoint(jobId, {
|
|
495
1096
|
stage: `task:${taskKey}:${step}:heartbeat`,
|
|
496
1097
|
timestamp: new Date().toISOString(),
|
|
@@ -523,7 +1124,16 @@ export class GatewayTrioService {
|
|
|
523
1124
|
}
|
|
524
1125
|
}
|
|
525
1126
|
async runGateway(job, taskKey, projectKey, request, agentOptions) {
|
|
526
|
-
|
|
1127
|
+
const startedAt = new Date().toISOString();
|
|
1128
|
+
const shouldSuppressIo = Boolean(request.onGatewayStart || request.onGatewayEnd || request.onGatewayChunk);
|
|
1129
|
+
let gatewaySlug = request.gatewayAgentName ?? "auto";
|
|
1130
|
+
let chosenSlug;
|
|
1131
|
+
let chosenModel;
|
|
1132
|
+
let chosenAdapter;
|
|
1133
|
+
let status = "failed";
|
|
1134
|
+
let errorMessage;
|
|
1135
|
+
let startEmitted = false;
|
|
1136
|
+
const invoke = () => this.deps.gatewayService.run({
|
|
527
1137
|
workspace: this.workspace,
|
|
528
1138
|
job,
|
|
529
1139
|
projectKey,
|
|
@@ -531,33 +1141,123 @@ export class GatewayTrioService {
|
|
|
531
1141
|
gatewayAgentName: request.gatewayAgentName,
|
|
532
1142
|
maxDocs: request.maxDocs,
|
|
533
1143
|
agentStream: request.agentStream,
|
|
1144
|
+
onStreamChunk: request.onGatewayChunk,
|
|
534
1145
|
rateAgents: request.rateAgents,
|
|
535
1146
|
avoidAgents: agentOptions?.avoidAgents,
|
|
536
1147
|
forceStronger: agentOptions?.forceStronger,
|
|
1148
|
+
forceTier: agentOptions?.forceTier,
|
|
537
1149
|
});
|
|
1150
|
+
try {
|
|
1151
|
+
startEmitted = true;
|
|
1152
|
+
request.onGatewayStart?.({
|
|
1153
|
+
taskKey,
|
|
1154
|
+
job,
|
|
1155
|
+
gatewayAgent: gatewaySlug,
|
|
1156
|
+
startedAt,
|
|
1157
|
+
});
|
|
1158
|
+
const result = shouldSuppressIo ? await this.withGatewayIoSuppressed(invoke) : await invoke();
|
|
1159
|
+
gatewaySlug = result.gatewayAgent.slug ?? result.gatewayAgent.id;
|
|
1160
|
+
chosenSlug = result.chosenAgent.agentSlug ?? result.chosenAgent.agentId;
|
|
1161
|
+
try {
|
|
1162
|
+
const resolved = await this.deps.routingService.resolveAgentForCommand({
|
|
1163
|
+
workspace: this.workspace,
|
|
1164
|
+
commandName: job,
|
|
1165
|
+
overrideAgentSlug: chosenSlug,
|
|
1166
|
+
});
|
|
1167
|
+
chosenModel = resolved.model ?? resolved.agent.defaultModel;
|
|
1168
|
+
chosenAdapter = resolved.agent.adapter;
|
|
1169
|
+
}
|
|
1170
|
+
catch {
|
|
1171
|
+
chosenModel = undefined;
|
|
1172
|
+
chosenAdapter = undefined;
|
|
1173
|
+
}
|
|
1174
|
+
request.onGatewaySelection?.({
|
|
1175
|
+
taskKey,
|
|
1176
|
+
job,
|
|
1177
|
+
gatewayAgent: gatewaySlug,
|
|
1178
|
+
chosenAgent: chosenSlug,
|
|
1179
|
+
chosenModel,
|
|
1180
|
+
chosenAdapter,
|
|
1181
|
+
startedAt,
|
|
1182
|
+
});
|
|
1183
|
+
status = "completed";
|
|
1184
|
+
return result;
|
|
1185
|
+
}
|
|
1186
|
+
catch (error) {
|
|
1187
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1188
|
+
errorMessage = message;
|
|
1189
|
+
const wrapped = message ? `${GATEWAY_FAILED_REASON}:${message}` : GATEWAY_FAILED_REASON;
|
|
1190
|
+
throw new Error(wrapped);
|
|
1191
|
+
}
|
|
1192
|
+
finally {
|
|
1193
|
+
if (startEmitted) {
|
|
1194
|
+
request.onGatewayEnd?.({
|
|
1195
|
+
taskKey,
|
|
1196
|
+
job,
|
|
1197
|
+
gatewayAgent: gatewaySlug,
|
|
1198
|
+
chosenAgent: chosenSlug,
|
|
1199
|
+
chosenModel,
|
|
1200
|
+
chosenAdapter,
|
|
1201
|
+
startedAt,
|
|
1202
|
+
endedAt: new Date().toISOString(),
|
|
1203
|
+
status,
|
|
1204
|
+
error: errorMessage,
|
|
1205
|
+
});
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
async withGatewayIoSuppressed(fn) {
|
|
1210
|
+
const ioEnv = "MCODA_STREAM_IO";
|
|
1211
|
+
const promptEnv = "MCODA_STREAM_IO_PROMPT";
|
|
1212
|
+
const prevIo = process.env[ioEnv];
|
|
1213
|
+
const prevPrompt = process.env[promptEnv];
|
|
1214
|
+
process.env[ioEnv] = "0";
|
|
1215
|
+
process.env[promptEnv] = "0";
|
|
1216
|
+
try {
|
|
1217
|
+
return await fn();
|
|
1218
|
+
}
|
|
1219
|
+
finally {
|
|
1220
|
+
if (prevIo === undefined) {
|
|
1221
|
+
delete process.env[ioEnv];
|
|
1222
|
+
}
|
|
1223
|
+
else {
|
|
1224
|
+
process.env[ioEnv] = prevIo;
|
|
1225
|
+
}
|
|
1226
|
+
if (prevPrompt === undefined) {
|
|
1227
|
+
delete process.env[promptEnv];
|
|
1228
|
+
}
|
|
1229
|
+
else {
|
|
1230
|
+
process.env[promptEnv] = prevPrompt;
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
538
1233
|
}
|
|
539
|
-
async runWorkStep(jobId, attempt, taskKey, projectKey, statusFilter, request, warnings, agentOptions, abortSignal, onResolvedAgent) {
|
|
1234
|
+
async runWorkStep(jobId, attempt, taskKey, projectKey, statusFilter, request, warnings, agentOptions, abortSignal, handoffContext, onResolvedAgent) {
|
|
540
1235
|
let gateway;
|
|
541
1236
|
let handoff;
|
|
542
1237
|
let resolvedAgent;
|
|
1238
|
+
await this.deps.gatewayService.preflightExecutionAgents("work-on-tasks", request.workAgentName);
|
|
543
1239
|
try {
|
|
544
1240
|
gateway = await this.runGateway("work-on-tasks", taskKey, projectKey, request, agentOptions);
|
|
545
|
-
|
|
546
|
-
|
|
1241
|
+
resolvedAgent = request.workAgentName ?? gateway.chosenAgent.agentSlug ?? gateway.chosenAgent.agentId;
|
|
1242
|
+
const missingContext = await this.guardMissingContext("work", jobId, taskKey, gateway, warnings, resolvedAgent);
|
|
1243
|
+
if (missingContext) {
|
|
1244
|
+
return missingContext;
|
|
1245
|
+
}
|
|
1246
|
+
handoff = buildGatewayHandoffContent(gateway, handoffContext);
|
|
547
1247
|
}
|
|
548
1248
|
catch (error) {
|
|
549
1249
|
const message = error instanceof Error ? error.message : String(error);
|
|
550
1250
|
if (!request.workAgentName)
|
|
551
1251
|
throw error;
|
|
552
1252
|
resolvedAgent = request.workAgentName;
|
|
553
|
-
handoff = [
|
|
1253
|
+
handoff = this.appendFallbackHandoffContext([
|
|
554
1254
|
"# Gateway Handoff",
|
|
555
1255
|
"",
|
|
556
1256
|
`Gateway agent failed; proceeding with override agent ${resolvedAgent}.`,
|
|
557
1257
|
`Error: ${message}`,
|
|
558
1258
|
"",
|
|
559
1259
|
buildGatewayHandoffDocdexUsage(),
|
|
560
|
-
].join("\n");
|
|
1260
|
+
], handoffContext).join("\n");
|
|
561
1261
|
warnings.push(`Gateway agent failed for work ${taskKey}; using override ${resolvedAgent}: ${message}`);
|
|
562
1262
|
}
|
|
563
1263
|
if (!resolvedAgent) {
|
|
@@ -566,6 +1266,7 @@ export class GatewayTrioService {
|
|
|
566
1266
|
if (onResolvedAgent) {
|
|
567
1267
|
await onResolvedAgent(resolvedAgent);
|
|
568
1268
|
}
|
|
1269
|
+
const selectedTier = this.resolveTierFromComplexity(gateway?.analysis?.complexity);
|
|
569
1270
|
const handoffPath = await this.prepareHandoff(jobId, taskKey, "work", attempt, handoff);
|
|
570
1271
|
const result = await withGatewayHandoff(handoffPath, async () => this.deps.workService.workOnTasks({
|
|
571
1272
|
workspace: this.workspace,
|
|
@@ -577,6 +1278,9 @@ export class GatewayTrioService {
|
|
|
577
1278
|
dryRun: request.dryRun,
|
|
578
1279
|
agentName: resolvedAgent,
|
|
579
1280
|
agentStream: request.agentStream,
|
|
1281
|
+
workRunner: request.workRunner,
|
|
1282
|
+
useCodali: request.useCodali,
|
|
1283
|
+
agentAdapterOverride: request.agentAdapterOverride,
|
|
580
1284
|
rateAgents: request.rateAgents,
|
|
581
1285
|
abortSignal,
|
|
582
1286
|
maxAgentSeconds: request.maxAgentSeconds,
|
|
@@ -585,30 +1289,55 @@ export class GatewayTrioService {
|
|
|
585
1289
|
const ratingSummary = request.rateAgents
|
|
586
1290
|
? await this.loadRatingSummary(result.jobId, "work", resolvedAgent)
|
|
587
1291
|
: undefined;
|
|
588
|
-
|
|
1292
|
+
const zeroTokens = await this.isZeroTokenRun(result.jobId, result.commandRunId);
|
|
1293
|
+
if (zeroTokens) {
|
|
1294
|
+
return {
|
|
1295
|
+
step: "work",
|
|
1296
|
+
status: "failed",
|
|
1297
|
+
error: ZERO_TOKEN_ERROR,
|
|
1298
|
+
chosenAgent: resolvedAgent,
|
|
1299
|
+
ratingSummary,
|
|
1300
|
+
tier: selectedTier,
|
|
1301
|
+
gatewayPlan: gateway?.analysis?.plan ?? [],
|
|
1302
|
+
gatewayFiles: [...(gateway?.analysis?.filesLikelyTouched ?? []), ...(gateway?.analysis?.filesToCreate ?? [])],
|
|
1303
|
+
};
|
|
1304
|
+
}
|
|
1305
|
+
return {
|
|
1306
|
+
...parsed,
|
|
1307
|
+
chosenAgent: resolvedAgent,
|
|
1308
|
+
ratingSummary,
|
|
1309
|
+
tier: selectedTier,
|
|
1310
|
+
gatewayPlan: gateway?.analysis?.plan ?? [],
|
|
1311
|
+
gatewayFiles: [...(gateway?.analysis?.filesLikelyTouched ?? []), ...(gateway?.analysis?.filesToCreate ?? [])],
|
|
1312
|
+
};
|
|
589
1313
|
}
|
|
590
|
-
async runReviewStep(jobId, attempt, taskKey, projectKey, statusFilter, request, warnings, agentOptions, abortSignal, onResolvedAgent) {
|
|
1314
|
+
async runReviewStep(jobId, attempt, taskKey, projectKey, statusFilter, request, warnings, agentOptions, abortSignal, handoffContext, onResolvedAgent) {
|
|
591
1315
|
let gateway;
|
|
592
1316
|
let handoff;
|
|
593
1317
|
let resolvedAgent;
|
|
1318
|
+
await this.deps.gatewayService.preflightExecutionAgents("code-review", request.reviewAgentName);
|
|
594
1319
|
try {
|
|
595
1320
|
gateway = await this.runGateway("code-review", taskKey, projectKey, request, agentOptions);
|
|
596
|
-
|
|
597
|
-
|
|
1321
|
+
resolvedAgent = request.reviewAgentName ?? gateway.chosenAgent.agentSlug ?? gateway.chosenAgent.agentId;
|
|
1322
|
+
const missingContext = await this.guardMissingContext("review", jobId, taskKey, gateway, warnings, resolvedAgent);
|
|
1323
|
+
if (missingContext) {
|
|
1324
|
+
return missingContext;
|
|
1325
|
+
}
|
|
1326
|
+
handoff = buildGatewayHandoffContent(gateway, handoffContext);
|
|
598
1327
|
}
|
|
599
1328
|
catch (error) {
|
|
600
1329
|
const message = error instanceof Error ? error.message : String(error);
|
|
601
1330
|
if (!request.reviewAgentName)
|
|
602
1331
|
throw error;
|
|
603
1332
|
resolvedAgent = request.reviewAgentName;
|
|
604
|
-
handoff = [
|
|
1333
|
+
handoff = this.appendFallbackHandoffContext([
|
|
605
1334
|
"# Gateway Handoff",
|
|
606
1335
|
"",
|
|
607
|
-
`
|
|
1336
|
+
`Routing failed; proceeding with override agent ${resolvedAgent}.`,
|
|
608
1337
|
`Error: ${message}`,
|
|
609
1338
|
"",
|
|
610
1339
|
buildGatewayHandoffDocdexUsage(),
|
|
611
|
-
].join("\n");
|
|
1340
|
+
], handoffContext).join("\n");
|
|
612
1341
|
warnings.push(`Gateway agent failed for review ${taskKey}; using override ${resolvedAgent}: ${message}`);
|
|
613
1342
|
}
|
|
614
1343
|
if (!resolvedAgent) {
|
|
@@ -628,36 +1357,60 @@ export class GatewayTrioService {
|
|
|
628
1357
|
agentName: resolvedAgent,
|
|
629
1358
|
agentStream: request.agentStream,
|
|
630
1359
|
rateAgents: request.rateAgents,
|
|
1360
|
+
createFollowupTasks: request.reviewFollowups === true,
|
|
631
1361
|
abortSignal,
|
|
632
1362
|
}));
|
|
633
1363
|
const parsed = this.parseReviewResult(taskKey, result);
|
|
634
1364
|
const ratingSummary = request.rateAgents
|
|
635
1365
|
? await this.loadRatingSummary(result.jobId, "review", resolvedAgent)
|
|
636
1366
|
: undefined;
|
|
637
|
-
|
|
1367
|
+
const zeroTokens = await this.isZeroTokenRun(result.jobId, result.commandRunId);
|
|
1368
|
+
if (zeroTokens) {
|
|
1369
|
+
return {
|
|
1370
|
+
step: "review",
|
|
1371
|
+
status: "failed",
|
|
1372
|
+
error: ZERO_TOKEN_ERROR,
|
|
1373
|
+
chosenAgent: resolvedAgent,
|
|
1374
|
+
ratingSummary,
|
|
1375
|
+
gatewayPlan: gateway?.analysis?.plan ?? [],
|
|
1376
|
+
gatewayFiles: [...(gateway?.analysis?.filesLikelyTouched ?? []), ...(gateway?.analysis?.filesToCreate ?? [])],
|
|
1377
|
+
};
|
|
1378
|
+
}
|
|
1379
|
+
return {
|
|
1380
|
+
...parsed,
|
|
1381
|
+
chosenAgent: resolvedAgent,
|
|
1382
|
+
ratingSummary,
|
|
1383
|
+
gatewayPlan: gateway?.analysis?.plan ?? [],
|
|
1384
|
+
gatewayFiles: [...(gateway?.analysis?.filesLikelyTouched ?? []), ...(gateway?.analysis?.filesToCreate ?? [])],
|
|
1385
|
+
};
|
|
638
1386
|
}
|
|
639
|
-
async runQaStep(jobId, attempt, taskKey, projectKey, statusFilter, request, warnings, agentOptions, abortSignal, onResolvedAgent) {
|
|
1387
|
+
async runQaStep(jobId, attempt, taskKey, projectKey, statusFilter, request, warnings, agentOptions, abortSignal, handoffContext, onResolvedAgent) {
|
|
640
1388
|
let gateway;
|
|
641
1389
|
let handoff;
|
|
642
1390
|
let resolvedAgent;
|
|
1391
|
+
await this.deps.gatewayService.preflightExecutionAgents("qa-tasks", request.qaAgentName);
|
|
643
1392
|
try {
|
|
644
1393
|
gateway = await this.runGateway("qa-tasks", taskKey, projectKey, request, agentOptions);
|
|
645
|
-
|
|
646
|
-
|
|
1394
|
+
resolvedAgent = request.qaAgentName ?? gateway.chosenAgent.agentSlug ?? gateway.chosenAgent.agentId;
|
|
1395
|
+
const missingContext = await this.guardMissingContext("qa", jobId, taskKey, gateway, warnings, resolvedAgent);
|
|
1396
|
+
if (missingContext) {
|
|
1397
|
+
return missingContext;
|
|
1398
|
+
}
|
|
1399
|
+
handoff = buildGatewayHandoffContent(gateway, handoffContext);
|
|
647
1400
|
}
|
|
648
1401
|
catch (error) {
|
|
649
1402
|
const message = error instanceof Error ? error.message : String(error);
|
|
650
1403
|
if (!request.qaAgentName)
|
|
651
1404
|
throw error;
|
|
652
1405
|
resolvedAgent = request.qaAgentName;
|
|
653
|
-
handoff = [
|
|
1406
|
+
handoff = this.appendFallbackHandoffContext([
|
|
654
1407
|
"# Gateway Handoff",
|
|
655
1408
|
"",
|
|
656
|
-
`
|
|
1409
|
+
`Routing failed; proceeding with override agent ${resolvedAgent}.`,
|
|
657
1410
|
`Error: ${message}`,
|
|
658
1411
|
"",
|
|
659
1412
|
buildGatewayHandoffDocdexUsage(),
|
|
660
|
-
].join("\n");
|
|
1413
|
+
], handoffContext).join("\n");
|
|
661
1414
|
warnings.push(`Gateway agent failed for QA ${taskKey}; using override ${resolvedAgent}: ${message}`);
|
|
662
1415
|
}
|
|
663
1416
|
if (!resolvedAgent) {
|
|
@@ -691,7 +1444,25 @@ export class GatewayTrioService {
|
|
|
691
1444
|
const ratingSummary = request.rateAgents
|
|
692
1445
|
? await this.loadRatingSummary(result.jobId, "qa", resolvedAgent)
|
|
693
1446
|
: undefined;
|
|
694
|
-
|
|
1447
|
+
const zeroTokens = await this.isZeroTokenRun(result.jobId, result.commandRunId);
|
|
1448
|
+
if (zeroTokens) {
|
|
1449
|
+
return {
|
|
1450
|
+
step: "qa",
|
|
1451
|
+
status: "failed",
|
|
1452
|
+
error: ZERO_TOKEN_ERROR,
|
|
1453
|
+
chosenAgent: resolvedAgent,
|
|
1454
|
+
ratingSummary,
|
|
1455
|
+
gatewayPlan: gateway?.analysis?.plan ?? [],
|
|
1456
|
+
gatewayFiles: [...(gateway?.analysis?.filesLikelyTouched ?? []), ...(gateway?.analysis?.filesToCreate ?? [])],
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
return {
|
|
1460
|
+
...parsed,
|
|
1461
|
+
chosenAgent: resolvedAgent,
|
|
1462
|
+
ratingSummary,
|
|
1463
|
+
gatewayPlan: gateway?.analysis?.plan ?? [],
|
|
1464
|
+
gatewayFiles: [...(gateway?.analysis?.filesLikelyTouched ?? []), ...(gateway?.analysis?.filesToCreate ?? [])],
|
|
1465
|
+
};
|
|
695
1466
|
}
|
|
696
1467
|
toSummary(state) {
|
|
697
1468
|
return Object.values(state.tasks).map((task) => ({
|
|
@@ -723,25 +1494,76 @@ export class GatewayTrioService {
|
|
|
723
1494
|
}
|
|
724
1495
|
const resolvedRequest = this.resolveRequest(request, resumeJob?.payload);
|
|
725
1496
|
const maxIterations = resolvedRequest.maxIterations;
|
|
1497
|
+
const continuousMode = maxIterations === undefined;
|
|
726
1498
|
const maxCycles = resolvedRequest.maxCycles;
|
|
727
1499
|
const maxAgentSeconds = resolvedRequest.maxAgentSeconds;
|
|
728
1500
|
if (!resolvedRequest.rateAgents) {
|
|
729
1501
|
warnings.push("Agent rating disabled; use --rate-agents to track rating/complexity updates.");
|
|
730
1502
|
}
|
|
731
1503
|
const statusFilter = await this.buildStatusFilter(resolvedRequest, warnings);
|
|
732
|
-
const
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
1504
|
+
const explicitTaskKeys = resolvedRequest.taskKeys ? this.dedupeTaskKeys(resolvedRequest.taskKeys) : [];
|
|
1505
|
+
const pseudoTaskKeys = explicitTaskKeys.filter((key) => this.isPseudoTaskKey(key));
|
|
1506
|
+
const filteredTaskKeys = explicitTaskKeys.filter((key) => !this.isPseudoTaskKey(key));
|
|
1507
|
+
const explicitTaskKeysProvided = explicitTaskKeys.length > 0;
|
|
1508
|
+
const includeTypes = resolvedRequest.includeTypes?.length ? resolvedRequest.includeTypes : undefined;
|
|
1509
|
+
let excludeTypes = resolvedRequest.excludeTypes;
|
|
1510
|
+
if (!excludeTypes && !includeTypes?.length) {
|
|
1511
|
+
excludeTypes = ["qa_followup"];
|
|
1512
|
+
}
|
|
1513
|
+
const explicitTaskFilterEmpty = explicitTaskKeysProvided && filteredTaskKeys.length === 0;
|
|
1514
|
+
if (pseudoTaskKeys.length) {
|
|
1515
|
+
warnings.push(`Skipping pseudo tasks: ${pseudoTaskKeys.join(", ")}.`);
|
|
1516
|
+
}
|
|
1517
|
+
if (explicitTaskFilterEmpty) {
|
|
1518
|
+
warnings.push("All requested tasks were pseudo entries; nothing to run.");
|
|
1519
|
+
}
|
|
1520
|
+
const baseTaskKeys = explicitTaskKeysProvided ? filteredTaskKeys : resolvedRequest.taskKeys;
|
|
1521
|
+
const shouldFixRunList = explicitTaskKeysProvided || (typeof resolvedRequest.limit === "number" && resolvedRequest.limit > 0);
|
|
736
1522
|
let jobId = request.resumeJobId;
|
|
737
1523
|
let state;
|
|
1524
|
+
let runList;
|
|
738
1525
|
if (request.resumeJobId) {
|
|
739
1526
|
state = await this.loadState(request.resumeJobId);
|
|
740
1527
|
if (!state)
|
|
741
1528
|
throw new Error(`Missing gateway-trio state for job ${request.resumeJobId}`);
|
|
1529
|
+
if (shouldFixRunList) {
|
|
1530
|
+
runList = state.run_list;
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
if (shouldFixRunList && !runList) {
|
|
1534
|
+
if (explicitTaskFilterEmpty) {
|
|
1535
|
+
runList = [];
|
|
1536
|
+
}
|
|
1537
|
+
else if (explicitTaskKeysProvided) {
|
|
1538
|
+
runList = this.buildRunListFromExplicit(filteredTaskKeys, resolvedRequest.limit, warnings);
|
|
1539
|
+
}
|
|
1540
|
+
else {
|
|
1541
|
+
runList = await this.buildRunListFromSelection({
|
|
1542
|
+
projectKey: resolvedRequest.projectKey,
|
|
1543
|
+
epicKey: resolvedRequest.epicKey,
|
|
1544
|
+
storyKey: resolvedRequest.storyKey,
|
|
1545
|
+
taskKeys: baseTaskKeys,
|
|
1546
|
+
statusFilter,
|
|
1547
|
+
includeTypes,
|
|
1548
|
+
excludeTypes,
|
|
1549
|
+
limit: resolvedRequest.limit,
|
|
1550
|
+
parallel: resolvedRequest.parallel,
|
|
1551
|
+
}, resolvedRequest.limit, warnings);
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
const taskKeysForRun = runList ?? baseTaskKeys;
|
|
1555
|
+
const commandRun = await this.deps.jobService.startCommandRun("gateway-trio", resolvedRequest.projectKey, {
|
|
1556
|
+
taskIds: taskKeysForRun,
|
|
1557
|
+
jobId: request.resumeJobId,
|
|
1558
|
+
});
|
|
1559
|
+
if (request.resumeJobId) {
|
|
742
1560
|
await this.deps.jobService.updateJobStatus(request.resumeJobId, "running", {
|
|
743
1561
|
job_state_detail: "resuming",
|
|
744
1562
|
});
|
|
1563
|
+
if (shouldFixRunList && runList && state && !state.run_list) {
|
|
1564
|
+
state.run_list = runList;
|
|
1565
|
+
await this.writeState(state);
|
|
1566
|
+
}
|
|
745
1567
|
}
|
|
746
1568
|
else {
|
|
747
1569
|
const job = await this.deps.jobService.startJob("gateway-trio", commandRun.id, resolvedRequest.projectKey, {
|
|
@@ -750,7 +1572,7 @@ export class GatewayTrioService {
|
|
|
750
1572
|
projectKey: resolvedRequest.projectKey,
|
|
751
1573
|
epicKey: resolvedRequest.epicKey,
|
|
752
1574
|
storyKey: resolvedRequest.storyKey,
|
|
753
|
-
tasks:
|
|
1575
|
+
tasks: taskKeysForRun,
|
|
754
1576
|
statusFilter,
|
|
755
1577
|
maxIterations,
|
|
756
1578
|
maxCycles,
|
|
@@ -771,6 +1593,7 @@ export class GatewayTrioService {
|
|
|
771
1593
|
qaTestCommand: resolvedRequest.qaTestCommand,
|
|
772
1594
|
qaMode: resolvedRequest.qaMode,
|
|
773
1595
|
qaFollowups: resolvedRequest.qaFollowups,
|
|
1596
|
+
reviewFollowups: resolvedRequest.reviewFollowups,
|
|
774
1597
|
qaResult: resolvedRequest.qaResult,
|
|
775
1598
|
qaNotes: resolvedRequest.qaNotes,
|
|
776
1599
|
qaEvidenceUrl: resolvedRequest.qaEvidenceUrl,
|
|
@@ -778,18 +1601,20 @@ export class GatewayTrioService {
|
|
|
778
1601
|
escalateOnNoChange: resolvedRequest.escalateOnNoChange,
|
|
779
1602
|
resumeSupported: true,
|
|
780
1603
|
},
|
|
781
|
-
totalItems: 0,
|
|
782
|
-
processedItems: 0,
|
|
783
1604
|
});
|
|
784
1605
|
jobId = job.id;
|
|
785
1606
|
state = {
|
|
786
1607
|
schema_version: 1,
|
|
787
1608
|
job_id: job.id,
|
|
788
1609
|
command_run_id: commandRun.id,
|
|
1610
|
+
run_list: runList,
|
|
789
1611
|
cycle: 0,
|
|
790
1612
|
tasks: {},
|
|
791
1613
|
};
|
|
792
1614
|
await this.writeState(state);
|
|
1615
|
+
await this.deps.jobService.updateJobStatus(jobId, "running", {
|
|
1616
|
+
job_state_detail: "loading_tasks",
|
|
1617
|
+
});
|
|
793
1618
|
}
|
|
794
1619
|
if (!jobId || !state) {
|
|
795
1620
|
throw new Error("gateway-trio job initialization failed");
|
|
@@ -797,309 +1622,852 @@ export class GatewayTrioService {
|
|
|
797
1622
|
if (resolvedRequest.onJobStart) {
|
|
798
1623
|
resolvedRequest.onJobStart(jobId, commandRun.id);
|
|
799
1624
|
}
|
|
1625
|
+
await this.runDocdexPreflight(jobId, warnings);
|
|
800
1626
|
await this.cleanupExpiredTaskLocks(warnings);
|
|
801
|
-
const explicitTasks = new Set(
|
|
1627
|
+
const explicitTasks = new Set(explicitTaskKeysProvided ? (runList ?? filteredTaskKeys) : []);
|
|
1628
|
+
if (pseudoTaskKeys.length) {
|
|
1629
|
+
for (const key of pseudoTaskKeys) {
|
|
1630
|
+
this.skipPseudoTask(state, key, warnings);
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
802
1633
|
await this.seedExplicitTasks(state, explicitTasks, warnings);
|
|
803
1634
|
await this.writeState(state);
|
|
804
1635
|
let cycle = state.cycle ?? 0;
|
|
1636
|
+
const taskLimit = resolvedRequest.limit;
|
|
1637
|
+
let activeTaskKey = null;
|
|
1638
|
+
let abortRemainingReason = null;
|
|
1639
|
+
const resolveJobState = (job) => job?.jobState ?? job?.state;
|
|
1640
|
+
const checkCancellation = async (label) => {
|
|
1641
|
+
const job = await this.deps.jobService.getJob(jobId);
|
|
1642
|
+
const stateNow = resolveJobState(job);
|
|
1643
|
+
if (stateNow === "cancelled") {
|
|
1644
|
+
abortRemainingReason = "cancelled";
|
|
1645
|
+
warnings.push(`Job ${jobId} cancelled${label ? ` (${label})` : ""}; stopping.`);
|
|
1646
|
+
return true;
|
|
1647
|
+
}
|
|
1648
|
+
return false;
|
|
1649
|
+
};
|
|
805
1650
|
try {
|
|
806
|
-
while (maxCycles === undefined || cycle < maxCycles) {
|
|
807
|
-
await
|
|
1651
|
+
while ((maxCycles === undefined || cycle < maxCycles) && !abortRemainingReason) {
|
|
1652
|
+
if (await checkCancellation("cycle_start"))
|
|
1653
|
+
break;
|
|
1654
|
+
await this.reopenRetryableFailedTasks(state, explicitTasks, maxIterations, warnings);
|
|
808
1655
|
await this.writeState(state);
|
|
809
|
-
const selection =
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
1656
|
+
const selection = explicitTaskFilterEmpty || (Array.isArray(runList) && runList.length === 0)
|
|
1657
|
+
? {
|
|
1658
|
+
ordered: [],
|
|
1659
|
+
warnings: [],
|
|
1660
|
+
filters: { effectiveStatuses: [] },
|
|
1661
|
+
}
|
|
1662
|
+
: await this.selectionService.selectTasks({
|
|
1663
|
+
projectKey: resolvedRequest.projectKey,
|
|
1664
|
+
epicKey: resolvedRequest.epicKey,
|
|
1665
|
+
storyKey: resolvedRequest.storyKey,
|
|
1666
|
+
taskKeys: Array.isArray(runList) ? runList : taskKeysForRun,
|
|
1667
|
+
statusFilter,
|
|
1668
|
+
limit: resolvedRequest.limit,
|
|
1669
|
+
parallel: resolvedRequest.parallel,
|
|
1670
|
+
});
|
|
819
1671
|
if (selection.warnings.length)
|
|
820
1672
|
warnings.push(...selection.warnings);
|
|
821
|
-
const
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
1673
|
+
const completedKeys = new Set(Object.values(state.tasks)
|
|
1674
|
+
.filter((task) => task.status === "completed")
|
|
1675
|
+
.map((task) => task.taskKey));
|
|
1676
|
+
let orderedCandidates = this.prioritizeFeedbackTasks(selection.ordered, state);
|
|
1677
|
+
if (activeTaskKey) {
|
|
1678
|
+
const activeIndex = orderedCandidates.findIndex((entry) => entry.task.key === activeTaskKey);
|
|
1679
|
+
if (activeIndex >= 0) {
|
|
1680
|
+
const activeEntry = orderedCandidates[activeIndex];
|
|
1681
|
+
orderedCandidates = [
|
|
1682
|
+
activeEntry,
|
|
1683
|
+
...orderedCandidates.slice(0, activeIndex),
|
|
1684
|
+
...orderedCandidates.slice(activeIndex + 1),
|
|
1685
|
+
];
|
|
1686
|
+
}
|
|
1687
|
+
else {
|
|
1688
|
+
const activeSelection = await this.selectionService.selectTasks({
|
|
1689
|
+
taskKeys: [activeTaskKey],
|
|
1690
|
+
ignoreStatusFilter: true,
|
|
1691
|
+
limit: 1,
|
|
1692
|
+
parallel: resolvedRequest.parallel,
|
|
1693
|
+
});
|
|
1694
|
+
const activeEntry = activeSelection.ordered[0];
|
|
1695
|
+
if (activeEntry) {
|
|
1696
|
+
orderedCandidates = [activeEntry, ...orderedCandidates];
|
|
1697
|
+
}
|
|
1698
|
+
else {
|
|
1699
|
+
activeTaskKey = null;
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
const ordered = [];
|
|
1704
|
+
const seenOrdered = new Set();
|
|
1705
|
+
for (const entry of orderedCandidates) {
|
|
1706
|
+
const taskKey = entry.task.key;
|
|
1707
|
+
if (seenOrdered.has(taskKey)) {
|
|
1708
|
+
warnings.push(`Task ${taskKey} appears multiple times in this cycle; skipping duplicate entry.`);
|
|
825
1709
|
continue;
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
1710
|
+
}
|
|
1711
|
+
seenOrdered.add(taskKey);
|
|
1712
|
+
const statusNow = this.normalizeStatus(entry.task.status);
|
|
1713
|
+
if (statusNow === "cancelled") {
|
|
1714
|
+
const progress = this.ensureProgress(state, taskKey);
|
|
1715
|
+
progress.status = "skipped";
|
|
1716
|
+
progress.lastError = "cancelled_in_db";
|
|
1717
|
+
state.tasks[taskKey] = progress;
|
|
1718
|
+
warnings.push(`Task ${taskKey} is cancelled; skipping.`);
|
|
1719
|
+
if (activeTaskKey === taskKey) {
|
|
1720
|
+
activeTaskKey = null;
|
|
1721
|
+
}
|
|
1722
|
+
continue;
|
|
1723
|
+
}
|
|
1724
|
+
if (completedKeys.has(taskKey)) {
|
|
1725
|
+
warnings.push(`Task ${taskKey} already completed earlier in this run; skipping.`);
|
|
1726
|
+
if (activeTaskKey === taskKey) {
|
|
1727
|
+
activeTaskKey = null;
|
|
1728
|
+
}
|
|
1729
|
+
continue;
|
|
1730
|
+
}
|
|
1731
|
+
if (this.isPseudoTaskKey(taskKey)) {
|
|
1732
|
+
this.skipPseudoTask(state, taskKey, warnings);
|
|
1733
|
+
if (activeTaskKey === taskKey) {
|
|
1734
|
+
activeTaskKey = null;
|
|
1735
|
+
}
|
|
1736
|
+
continue;
|
|
1737
|
+
}
|
|
1738
|
+
ordered.push(entry);
|
|
830
1739
|
}
|
|
831
1740
|
await this.writeState(state);
|
|
832
1741
|
await this.deps.jobService.updateJobStatus(jobId, "running", {
|
|
833
1742
|
totalItems: ordered.length,
|
|
834
1743
|
processedItems: 0,
|
|
1744
|
+
job_state_detail: ordered.length === 0 ? "no_tasks" : "processing",
|
|
835
1745
|
});
|
|
836
1746
|
let completedThisCycle = 0;
|
|
837
1747
|
let processedThisCycle = 0;
|
|
838
1748
|
let attemptedThisCycle = 0;
|
|
1749
|
+
const seenThisCycle = new Set();
|
|
839
1750
|
for (const entry of ordered) {
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
continue;
|
|
1751
|
+
if (abortRemainingReason)
|
|
1752
|
+
break;
|
|
1753
|
+
if (await checkCancellation(`task:${entry.task.key}`))
|
|
1754
|
+
break;
|
|
1755
|
+
if (typeof taskLimit === "number" && taskLimit > 0 && completedKeys.size >= taskLimit) {
|
|
1756
|
+
warnings.push(`Completed task limit ${taskLimit} reached; stopping run.`);
|
|
1757
|
+
abortRemainingReason = "limit_reached";
|
|
1758
|
+
break;
|
|
849
1759
|
}
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
if (progress.status === "skipped") {
|
|
857
|
-
progress.status = "pending";
|
|
858
|
-
progress.lastError = undefined;
|
|
859
|
-
}
|
|
860
|
-
if (progress.status === "completed" || progress.status === "blocked") {
|
|
861
|
-
continue;
|
|
1760
|
+
if (activeTaskKey && entry.task.key !== activeTaskKey) {
|
|
1761
|
+
const activeProgress = state.tasks[activeTaskKey];
|
|
1762
|
+
if (activeProgress && activeProgress.status !== "completed") {
|
|
1763
|
+
break;
|
|
1764
|
+
}
|
|
1765
|
+
activeTaskKey = null;
|
|
862
1766
|
}
|
|
863
|
-
|
|
864
|
-
|
|
1767
|
+
let attempted = false;
|
|
1768
|
+
let holdAfterTask = false;
|
|
1769
|
+
let currentTaskKey;
|
|
1770
|
+
try {
|
|
1771
|
+
const taskKey = entry.task.key;
|
|
1772
|
+
currentTaskKey = taskKey;
|
|
1773
|
+
if (seenThisCycle.has(taskKey)) {
|
|
1774
|
+
warnings.push(`Task ${taskKey} appears multiple times in this cycle; skipping duplicate entry.`);
|
|
865
1775
|
continue;
|
|
866
1776
|
}
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
if (this.hasReachedMaxIterations(progress, maxIterations)) {
|
|
872
|
-
progress.status = "failed";
|
|
873
|
-
progress.lastError = "max_iterations_reached";
|
|
874
|
-
state.tasks[taskKey] = progress;
|
|
875
|
-
if (maxIterations !== undefined) {
|
|
876
|
-
warnings.push(`Task ${taskKey} hit max iterations (${maxIterations}).`);
|
|
1777
|
+
seenThisCycle.add(taskKey);
|
|
1778
|
+
if (completedKeys.has(taskKey)) {
|
|
1779
|
+
warnings.push(`Task ${taskKey} already completed earlier in this run; skipping.`);
|
|
1780
|
+
continue;
|
|
877
1781
|
}
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
if (!workWasSkipped) {
|
|
891
|
-
const workAgentOptions = this.buildAgentOptions(progress, "work", resolvedRequest);
|
|
892
|
-
progress.lastStep = "work";
|
|
1782
|
+
if (typeof taskLimit === "number" && taskLimit > 0 && completedKeys.size >= taskLimit) {
|
|
1783
|
+
warnings.push(`Completed task limit ${taskLimit} reached; stopping run.`);
|
|
1784
|
+
abortRemainingReason = "limit_reached";
|
|
1785
|
+
break;
|
|
1786
|
+
}
|
|
1787
|
+
const normalizedStatus = this.normalizeStatus(entry.task.status);
|
|
1788
|
+
if (normalizedStatus && TERMINAL_STATUSES.has(normalizedStatus)) {
|
|
1789
|
+
warnings.push(`Skipping terminal task ${taskKey} (${normalizedStatus}).`);
|
|
1790
|
+
continue;
|
|
1791
|
+
}
|
|
1792
|
+
const progress = this.ensureProgress(state, taskKey);
|
|
1793
|
+
await this.persistRevertLearning(entry.task.id, taskKey, entry.task.metadata ?? undefined, progress, warnings);
|
|
893
1794
|
state.tasks[taskKey] = progress;
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
progress.
|
|
1795
|
+
if (progress.status === "skipped") {
|
|
1796
|
+
progress.status = "pending";
|
|
1797
|
+
progress.lastError = undefined;
|
|
1798
|
+
}
|
|
1799
|
+
if (progress.status === "completed") {
|
|
1800
|
+
continue;
|
|
1801
|
+
}
|
|
1802
|
+
if (progress.status === "failed") {
|
|
1803
|
+
if (progress.lastError === ZERO_TOKEN_ERROR && !continuousMode) {
|
|
1804
|
+
warnings.push(`Task ${taskKey} failed after repeated zero-token runs.`);
|
|
1805
|
+
continue;
|
|
1806
|
+
}
|
|
1807
|
+
const normalizedReason = normalizeFailureReason(progress.lastError ?? progress.failureHistory?.[progress.failureHistory.length - 1]?.reason ?? "");
|
|
1808
|
+
if (!continuousMode &&
|
|
1809
|
+
(progress.lastGuardrailRetryable === false ||
|
|
1810
|
+
(normalizedReason && NON_RETRYABLE_FAILURE_REASONS.has(normalizedReason)))) {
|
|
1811
|
+
const reasonLabel = normalizedReason ?? progress.lastError ?? progress.lastEscalationReason ?? "guardrail_non_retryable";
|
|
1812
|
+
warnings.push(`Task ${taskKey} failed with non-retryable reason ${reasonLabel}; skipping.`);
|
|
1813
|
+
continue;
|
|
1814
|
+
}
|
|
1815
|
+
if (this.hasReachedMaxIterations(progress, maxIterations)) {
|
|
1816
|
+
continue;
|
|
1817
|
+
}
|
|
1818
|
+
if (!this.shouldReopenFailedTask(progress, taskKey, warnings, continuousMode)) {
|
|
1819
|
+
continue;
|
|
1820
|
+
}
|
|
1821
|
+
progress.status = "pending";
|
|
1822
|
+
progress.lastError = undefined;
|
|
1823
|
+
progress.lastGuardrailRetryable = undefined;
|
|
897
1824
|
state.tasks[taskKey] = progress;
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
const
|
|
914
|
-
if (
|
|
915
|
-
progress.status = "
|
|
916
|
-
progress.lastError = "
|
|
1825
|
+
}
|
|
1826
|
+
const projectKey = await this.projectKeyForTask(entry.task.projectId);
|
|
1827
|
+
let currentStatus = this.normalizeStatus(entry.task.status);
|
|
1828
|
+
const readStatus = async () => {
|
|
1829
|
+
if (resolvedRequest.dryRun)
|
|
1830
|
+
return currentStatus;
|
|
1831
|
+
return await this.refreshTaskStatus(taskKey, warnings);
|
|
1832
|
+
};
|
|
1833
|
+
const setDryRunStatus = (next) => {
|
|
1834
|
+
if (resolvedRequest.dryRun) {
|
|
1835
|
+
currentStatus = next;
|
|
1836
|
+
}
|
|
1837
|
+
};
|
|
1838
|
+
let taskCompleted = false;
|
|
1839
|
+
while (!taskCompleted) {
|
|
1840
|
+
const statusNow = await readStatus();
|
|
1841
|
+
if (!statusNow) {
|
|
1842
|
+
progress.status = "failed";
|
|
1843
|
+
progress.lastError = "status_unknown";
|
|
1844
|
+
state.tasks[taskKey] = progress;
|
|
1845
|
+
await this.writeState(state);
|
|
1846
|
+
warnings.push(`Task ${taskKey} status unknown; skipping.`);
|
|
1847
|
+
break;
|
|
1848
|
+
}
|
|
1849
|
+
if (TERMINAL_STATUSES.has(statusNow)) {
|
|
1850
|
+
progress.status = statusNow === "completed" ? "completed" : "skipped";
|
|
1851
|
+
progress.lastError = statusNow === "completed" ? "completed_in_db" : "cancelled_in_db";
|
|
1852
|
+
state.tasks[taskKey] = progress;
|
|
1853
|
+
await this.writeState(state);
|
|
1854
|
+
if (statusNow === "completed") {
|
|
1855
|
+
completedKeys.add(taskKey);
|
|
1856
|
+
completedThisCycle += 1;
|
|
1857
|
+
}
|
|
1858
|
+
break;
|
|
1859
|
+
}
|
|
1860
|
+
if (statusNow === "blocked") {
|
|
1861
|
+
progress.status = "failed";
|
|
1862
|
+
progress.lastError = "legacy_blocked";
|
|
1863
|
+
state.tasks[taskKey] = progress;
|
|
1864
|
+
if (!resolvedRequest.dryRun) {
|
|
1865
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
1866
|
+
status: "failed",
|
|
1867
|
+
metadata: {
|
|
1868
|
+
...entry.task.metadata,
|
|
1869
|
+
failed_reason: "legacy_blocked",
|
|
1870
|
+
},
|
|
1871
|
+
});
|
|
1872
|
+
}
|
|
1873
|
+
await this.writeState(state);
|
|
1874
|
+
warnings.push(`Task ${taskKey} had legacy blocked status; marked failed.`);
|
|
1875
|
+
break;
|
|
1876
|
+
}
|
|
1877
|
+
if (!attempted) {
|
|
1878
|
+
attemptedThisCycle += 1;
|
|
1879
|
+
attempted = true;
|
|
1880
|
+
}
|
|
1881
|
+
if (isReadyToReviewStatus(statusNow)) {
|
|
1882
|
+
const attemptIndex = Math.max(progress.attempts, 1);
|
|
1883
|
+
const reviewAgentOptions = this.buildAgentOptions(progress, "review", resolvedRequest);
|
|
1884
|
+
progress.lastStep = "review";
|
|
1885
|
+
state.tasks[taskKey] = progress;
|
|
1886
|
+
await this.writeState(state);
|
|
1887
|
+
const reviewOutcome = await this.runStepWithTimeout("review", jobId, taskKey, attemptIndex, maxAgentSeconds, (signal) => this.runReviewStep(jobId, attemptIndex, taskKey, projectKey, normalizeReviewStatuses([READY_TO_CODE_REVIEW]), resolvedRequest, warnings, reviewAgentOptions, signal, this.handoffContextForProgress(progress), async (agent) => {
|
|
1888
|
+
progress.chosenAgents.review = agent;
|
|
1889
|
+
state.tasks[taskKey] = progress;
|
|
1890
|
+
await this.writeState(state);
|
|
1891
|
+
}));
|
|
1892
|
+
progress.lastStep = "review";
|
|
1893
|
+
progress.lastDecision = reviewOutcome.decision;
|
|
1894
|
+
progress.lastError = reviewOutcome.error;
|
|
1895
|
+
progress.chosenAgents.review = reviewOutcome.chosenAgent ?? progress.chosenAgents.review;
|
|
1896
|
+
if (reviewOutcome.gatewayPlan?.length) {
|
|
1897
|
+
progress.lastOutcome = reviewOutcome.gatewayPlan.join(" | ");
|
|
1898
|
+
}
|
|
1899
|
+
this.recordDecision(progress, "review", attemptIndex, "result", reviewOutcome.status, reviewOutcome.decision ?? reviewOutcome.error ?? reviewOutcome.outcome);
|
|
1900
|
+
if (this.isAuthFailure(reviewOutcome.error)) {
|
|
1901
|
+
const message = reviewOutcome.error ?? AUTH_ERROR_REASON;
|
|
1902
|
+
reviewOutcome.error = AUTH_ERROR_REASON;
|
|
1903
|
+
progress.lastError = AUTH_ERROR_REASON;
|
|
1904
|
+
this.recordFailure(progress, reviewOutcome, attemptIndex);
|
|
1905
|
+
progress.status = "failed";
|
|
1906
|
+
state.tasks[taskKey] = progress;
|
|
1907
|
+
await this.writeState(state);
|
|
1908
|
+
if (!resolvedRequest.dryRun) {
|
|
1909
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
1910
|
+
status: "failed",
|
|
1911
|
+
metadata: {
|
|
1912
|
+
...entry.task.metadata,
|
|
1913
|
+
failed_reason: AUTH_ERROR_REASON,
|
|
1914
|
+
},
|
|
1915
|
+
});
|
|
1916
|
+
}
|
|
1917
|
+
warnings.push(`Task ${taskKey} failed due to auth/rate limit during review; continuing run. ${message}`);
|
|
1918
|
+
break;
|
|
1919
|
+
}
|
|
1920
|
+
this.recordFailure(progress, reviewOutcome, attemptIndex);
|
|
1921
|
+
this.recordRating(progress, reviewOutcome.ratingSummary);
|
|
1922
|
+
state.tasks[taskKey] = progress;
|
|
1923
|
+
await this.writeState(state);
|
|
1924
|
+
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
1925
|
+
stage: `task:${taskKey}:review`,
|
|
1926
|
+
timestamp: new Date().toISOString(),
|
|
1927
|
+
details: { taskKey, attempt: attemptIndex, outcome: reviewOutcome },
|
|
1928
|
+
});
|
|
1929
|
+
if (reviewOutcome.error === ZERO_TOKEN_ERROR) {
|
|
1930
|
+
if (!resolvedRequest.dryRun) {
|
|
1931
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, { status: READY_TO_CODE_REVIEW });
|
|
1932
|
+
}
|
|
1933
|
+
const zeroTokenCount = this.countFailures(progress, "review", ZERO_TOKEN_ERROR);
|
|
1934
|
+
if (zeroTokenCount >= 2) {
|
|
1935
|
+
progress.status = "failed";
|
|
1936
|
+
progress.lastError = ZERO_TOKEN_ERROR;
|
|
1937
|
+
state.tasks[taskKey] = progress;
|
|
1938
|
+
await this.writeState(state);
|
|
1939
|
+
warnings.push(`Task ${taskKey} failed after repeated zero-token review runs.`);
|
|
1940
|
+
break;
|
|
1941
|
+
}
|
|
1942
|
+
warnings.push(`Retrying ${taskKey} after zero-token review run.`);
|
|
1943
|
+
state.tasks[taskKey] = progress;
|
|
1944
|
+
await this.writeState(state);
|
|
1945
|
+
await this.backoffZeroTokens(zeroTokenCount);
|
|
1946
|
+
continue;
|
|
1947
|
+
}
|
|
1948
|
+
const reviewGatewayFailure = reviewOutcome.status === "failed" &&
|
|
1949
|
+
normalizeFailureReason(reviewOutcome.error ?? reviewOutcome.decision) === GATEWAY_FAILED_REASON;
|
|
1950
|
+
if (reviewGatewayFailure) {
|
|
1951
|
+
progress.status = "failed";
|
|
1952
|
+
progress.lastError = GATEWAY_FAILED_REASON;
|
|
1953
|
+
state.tasks[taskKey] = progress;
|
|
1954
|
+
await this.writeState(state);
|
|
1955
|
+
if (!resolvedRequest.dryRun) {
|
|
1956
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
1957
|
+
status: "failed",
|
|
1958
|
+
metadata: {
|
|
1959
|
+
...entry.task.metadata,
|
|
1960
|
+
failed_reason: GATEWAY_FAILED_REASON,
|
|
1961
|
+
},
|
|
1962
|
+
});
|
|
1963
|
+
}
|
|
1964
|
+
warnings.push(`Task ${taskKey} failed due to gateway failure during review.`);
|
|
1965
|
+
break;
|
|
1966
|
+
}
|
|
1967
|
+
const retryReview = this.shouldRetryAfter(reviewOutcome);
|
|
1968
|
+
if (retryReview) {
|
|
1969
|
+
this.recordDecision(progress, "review", attemptIndex, "retry", "retry", reviewOutcome.decision ?? reviewOutcome.status);
|
|
1970
|
+
warnings.push(`Retrying ${taskKey} after review (${reviewOutcome.decision ?? reviewOutcome.status}).`);
|
|
1971
|
+
setDryRunStatus("in_progress");
|
|
1972
|
+
state.tasks[taskKey] = progress;
|
|
1973
|
+
await this.writeState(state);
|
|
1974
|
+
continue;
|
|
1975
|
+
}
|
|
1976
|
+
if (reviewOutcome.status === "failed" && reviewOutcome.guardrailRetryable === false) {
|
|
1977
|
+
const reason = normalizeFailureReason(reviewOutcome.guardrailReason ?? reviewOutcome.error ?? reviewOutcome.decision ?? "failed") ??
|
|
1978
|
+
reviewOutcome.guardrailReason ??
|
|
1979
|
+
reviewOutcome.error ??
|
|
1980
|
+
reviewOutcome.decision ??
|
|
1981
|
+
"failed";
|
|
1982
|
+
progress.status = "failed";
|
|
1983
|
+
progress.lastError = reason;
|
|
1984
|
+
this.recordDecision(progress, "review", attemptIndex, "terminal", "failed", reason);
|
|
1985
|
+
state.tasks[taskKey] = progress;
|
|
1986
|
+
await this.writeState(state);
|
|
1987
|
+
if (!resolvedRequest.dryRun) {
|
|
1988
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
1989
|
+
status: "failed",
|
|
1990
|
+
metadata: {
|
|
1991
|
+
...entry.task.metadata,
|
|
1992
|
+
failed_reason: reason,
|
|
1993
|
+
},
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1996
|
+
warnings.push(`Task ${taskKey} failed with non-retryable review reason ${reason}.`);
|
|
1997
|
+
break;
|
|
1998
|
+
}
|
|
1999
|
+
if (resolvedRequest.dryRun) {
|
|
2000
|
+
if (reviewOutcome.decision === "changes_requested") {
|
|
2001
|
+
setDryRunStatus("changes_requested");
|
|
2002
|
+
}
|
|
2003
|
+
else {
|
|
2004
|
+
setDryRunStatus(reviewOutcome.status === "succeeded" ? "ready_to_qa" : "in_progress");
|
|
2005
|
+
}
|
|
2006
|
+
continue;
|
|
2007
|
+
}
|
|
2008
|
+
const statusAfterReview = await this.refreshTaskStatus(taskKey, warnings);
|
|
2009
|
+
if (statusAfterReview && statusAfterReview === "completed") {
|
|
2010
|
+
progress.status = "completed";
|
|
2011
|
+
state.tasks[taskKey] = progress;
|
|
2012
|
+
completedKeys.add(taskKey);
|
|
2013
|
+
completedThisCycle += 1;
|
|
2014
|
+
await this.writeState(state);
|
|
2015
|
+
break;
|
|
2016
|
+
}
|
|
2017
|
+
if (statusAfterReview && statusAfterReview === "failed") {
|
|
2018
|
+
progress.status = "failed";
|
|
2019
|
+
progress.lastError = progress.lastError ?? "failed";
|
|
2020
|
+
state.tasks[taskKey] = progress;
|
|
2021
|
+
await this.writeState(state);
|
|
2022
|
+
break;
|
|
2023
|
+
}
|
|
2024
|
+
if (statusAfterReview && statusAfterReview === "changes_requested") {
|
|
2025
|
+
continue;
|
|
2026
|
+
}
|
|
2027
|
+
if (statusAfterReview && !["ready_to_qa"].includes(statusAfterReview)) {
|
|
2028
|
+
warnings.push(`Task ${taskKey} status ${statusAfterReview} after review; retrying work step.`);
|
|
2029
|
+
continue;
|
|
2030
|
+
}
|
|
2031
|
+
continue;
|
|
2032
|
+
}
|
|
2033
|
+
if (statusNow === "ready_to_qa") {
|
|
2034
|
+
const attemptIndex = Math.max(progress.attempts, 1);
|
|
2035
|
+
progress.lastStep = "qa";
|
|
2036
|
+
state.tasks[taskKey] = progress;
|
|
2037
|
+
await this.writeState(state);
|
|
2038
|
+
const qaOutcome = await this.runStepWithTimeout("qa", jobId, taskKey, attemptIndex, maxAgentSeconds, (signal) => this.runQaStep(jobId, attemptIndex, taskKey, projectKey, ["ready_to_qa"], resolvedRequest, warnings, this.buildAgentOptions(progress, "qa", resolvedRequest), signal, this.handoffContextForProgress(progress), async (agent) => {
|
|
2039
|
+
progress.chosenAgents.qa = agent;
|
|
2040
|
+
state.tasks[taskKey] = progress;
|
|
2041
|
+
await this.writeState(state);
|
|
2042
|
+
}));
|
|
2043
|
+
progress.lastStep = "qa";
|
|
2044
|
+
progress.lastOutcome = qaOutcome.outcome;
|
|
2045
|
+
progress.lastError = qaOutcome.error;
|
|
2046
|
+
progress.chosenAgents.qa = qaOutcome.chosenAgent ?? progress.chosenAgents.qa;
|
|
2047
|
+
this.recordDecision(progress, "qa", attemptIndex, "result", qaOutcome.status, qaOutcome.outcome ?? qaOutcome.error ?? qaOutcome.notes);
|
|
2048
|
+
if (this.isAuthFailure(qaOutcome.error)) {
|
|
2049
|
+
const message = qaOutcome.error ?? AUTH_ERROR_REASON;
|
|
2050
|
+
qaOutcome.error = AUTH_ERROR_REASON;
|
|
2051
|
+
progress.lastError = AUTH_ERROR_REASON;
|
|
2052
|
+
this.recordFailure(progress, qaOutcome, attemptIndex);
|
|
2053
|
+
progress.status = "failed";
|
|
2054
|
+
state.tasks[taskKey] = progress;
|
|
2055
|
+
await this.writeState(state);
|
|
2056
|
+
if (!resolvedRequest.dryRun) {
|
|
2057
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
2058
|
+
status: "failed",
|
|
2059
|
+
metadata: {
|
|
2060
|
+
...entry.task.metadata,
|
|
2061
|
+
failed_reason: AUTH_ERROR_REASON,
|
|
2062
|
+
},
|
|
2063
|
+
});
|
|
2064
|
+
}
|
|
2065
|
+
warnings.push(`Task ${taskKey} failed due to auth/rate limit during QA; continuing run. ${message}`);
|
|
2066
|
+
break;
|
|
2067
|
+
}
|
|
2068
|
+
this.recordFailure(progress, qaOutcome, attemptIndex);
|
|
2069
|
+
this.recordRating(progress, qaOutcome.ratingSummary);
|
|
2070
|
+
state.tasks[taskKey] = progress;
|
|
2071
|
+
await this.writeState(state);
|
|
2072
|
+
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
2073
|
+
stage: `task:${taskKey}:qa`,
|
|
2074
|
+
timestamp: new Date().toISOString(),
|
|
2075
|
+
details: { taskKey, attempt: attemptIndex, outcome: qaOutcome },
|
|
2076
|
+
});
|
|
2077
|
+
if (qaOutcome.error === ZERO_TOKEN_ERROR) {
|
|
2078
|
+
if (!resolvedRequest.dryRun) {
|
|
2079
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, { status: "ready_to_qa" });
|
|
2080
|
+
}
|
|
2081
|
+
const zeroTokenCount = this.countFailures(progress, "qa", ZERO_TOKEN_ERROR);
|
|
2082
|
+
if (zeroTokenCount >= 2) {
|
|
2083
|
+
progress.status = "failed";
|
|
2084
|
+
progress.lastError = ZERO_TOKEN_ERROR;
|
|
2085
|
+
state.tasks[taskKey] = progress;
|
|
2086
|
+
await this.writeState(state);
|
|
2087
|
+
warnings.push(`Task ${taskKey} failed after repeated zero-token QA runs.`);
|
|
2088
|
+
break;
|
|
2089
|
+
}
|
|
2090
|
+
warnings.push(`Retrying ${taskKey} after zero-token QA run.`);
|
|
2091
|
+
state.tasks[taskKey] = progress;
|
|
2092
|
+
await this.writeState(state);
|
|
2093
|
+
await this.backoffZeroTokens(zeroTokenCount);
|
|
2094
|
+
continue;
|
|
2095
|
+
}
|
|
2096
|
+
const qaGatewayFailure = qaOutcome.status === "failed" &&
|
|
2097
|
+
normalizeFailureReason(qaOutcome.error ?? qaOutcome.outcome) === GATEWAY_FAILED_REASON;
|
|
2098
|
+
if (qaGatewayFailure) {
|
|
2099
|
+
progress.status = "failed";
|
|
2100
|
+
progress.lastError = GATEWAY_FAILED_REASON;
|
|
2101
|
+
state.tasks[taskKey] = progress;
|
|
2102
|
+
await this.writeState(state);
|
|
2103
|
+
if (!resolvedRequest.dryRun) {
|
|
2104
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
2105
|
+
status: "failed",
|
|
2106
|
+
metadata: {
|
|
2107
|
+
...entry.task.metadata,
|
|
2108
|
+
failed_reason: GATEWAY_FAILED_REASON,
|
|
2109
|
+
},
|
|
2110
|
+
});
|
|
2111
|
+
}
|
|
2112
|
+
warnings.push(`Task ${taskKey} failed due to gateway failure during QA.`);
|
|
2113
|
+
break;
|
|
2114
|
+
}
|
|
2115
|
+
const retryQa = this.shouldRetryAfter(qaOutcome);
|
|
2116
|
+
if (retryQa) {
|
|
2117
|
+
progress.lastQaFailureSummary = this.buildQaFailureSummary(taskKey, qaOutcome);
|
|
2118
|
+
this.recordDecision(progress, "qa", attemptIndex, "retry", "retry", qaOutcome.outcome ?? qaOutcome.status);
|
|
2119
|
+
warnings.push(`Retrying ${taskKey} after QA (${qaOutcome.outcome ?? qaOutcome.status}).`);
|
|
2120
|
+
warnings.push(`Task ${taskKey} queued for gateway re-analysis after QA failure.`);
|
|
2121
|
+
setDryRunStatus("in_progress");
|
|
2122
|
+
state.tasks[taskKey] = progress;
|
|
2123
|
+
await this.writeState(state);
|
|
2124
|
+
continue;
|
|
2125
|
+
}
|
|
2126
|
+
if (qaOutcome.status === "failed" && qaOutcome.guardrailRetryable === false) {
|
|
2127
|
+
const reason = normalizeFailureReason(qaOutcome.guardrailReason ?? qaOutcome.error ?? qaOutcome.outcome ?? "failed") ??
|
|
2128
|
+
qaOutcome.guardrailReason ??
|
|
2129
|
+
qaOutcome.error ??
|
|
2130
|
+
qaOutcome.outcome ??
|
|
2131
|
+
"failed";
|
|
2132
|
+
progress.status = "failed";
|
|
2133
|
+
progress.lastError = reason;
|
|
2134
|
+
progress.lastQaFailureSummary = this.buildQaFailureSummary(taskKey, qaOutcome);
|
|
2135
|
+
this.recordDecision(progress, "qa", attemptIndex, "terminal", "failed", reason);
|
|
2136
|
+
state.tasks[taskKey] = progress;
|
|
2137
|
+
await this.writeState(state);
|
|
2138
|
+
if (!resolvedRequest.dryRun) {
|
|
2139
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
2140
|
+
status: "failed",
|
|
2141
|
+
metadata: {
|
|
2142
|
+
...entry.task.metadata,
|
|
2143
|
+
failed_reason: reason,
|
|
2144
|
+
},
|
|
2145
|
+
});
|
|
2146
|
+
}
|
|
2147
|
+
warnings.push(`Task ${taskKey} failed with non-retryable QA reason ${reason}.`);
|
|
2148
|
+
break;
|
|
2149
|
+
}
|
|
2150
|
+
if (resolvedRequest.dryRun) {
|
|
2151
|
+
setDryRunStatus(qaOutcome.status === "succeeded" ? "completed" : "in_progress");
|
|
2152
|
+
}
|
|
2153
|
+
else {
|
|
2154
|
+
const statusAfterQa = await this.refreshTaskStatus(taskKey, warnings);
|
|
2155
|
+
if (statusAfterQa && statusAfterQa === "failed") {
|
|
2156
|
+
progress.status = "failed";
|
|
2157
|
+
progress.lastError = progress.lastError ?? "failed";
|
|
2158
|
+
state.tasks[taskKey] = progress;
|
|
2159
|
+
await this.writeState(state);
|
|
2160
|
+
break;
|
|
2161
|
+
}
|
|
2162
|
+
if (statusAfterQa && statusAfterQa !== "completed") {
|
|
2163
|
+
warnings.push(`Task ${taskKey} status ${statusAfterQa} after QA; retrying work step.`);
|
|
2164
|
+
continue;
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
progress.status = "completed";
|
|
2168
|
+
progress.lastQaFailureSummary = undefined;
|
|
2169
|
+
const touchedFiles = this.mergeGatewayFiles(qaOutcome).length > 0
|
|
2170
|
+
? this.mergeGatewayFiles(qaOutcome)
|
|
2171
|
+
: Array.isArray(entry.task.metadata?.last_review_changed_paths)
|
|
2172
|
+
? (entry.task.metadata.last_review_changed_paths || [])
|
|
2173
|
+
.filter((item) => typeof item === "string" && item.trim().length > 0)
|
|
2174
|
+
.map((item) => item.trim())
|
|
2175
|
+
: [];
|
|
2176
|
+
const planSummary = (qaOutcome.gatewayPlan ?? []).join(" | ") ||
|
|
2177
|
+
progress.stepOutcomes?.work ||
|
|
2178
|
+
"Work-review-QA loop completed";
|
|
2179
|
+
try {
|
|
2180
|
+
await this.appendGoldenExample({
|
|
2181
|
+
taskKey,
|
|
2182
|
+
title: entry.task.title,
|
|
2183
|
+
planSummary,
|
|
2184
|
+
touchedFiles,
|
|
2185
|
+
reviewNotes: progress.lastDecision,
|
|
2186
|
+
qaNotes: qaOutcome.notes ?? qaOutcome.outcome ?? qaOutcome.status,
|
|
2187
|
+
});
|
|
2188
|
+
}
|
|
2189
|
+
catch (error) {
|
|
2190
|
+
warnings.push(`Golden example capture failed for ${taskKey}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2191
|
+
}
|
|
917
2192
|
state.tasks[taskKey] = progress;
|
|
2193
|
+
completedKeys.add(taskKey);
|
|
918
2194
|
await this.writeState(state);
|
|
919
|
-
|
|
2195
|
+
completedThisCycle += 1;
|
|
2196
|
+
taskCompleted = true;
|
|
920
2197
|
continue;
|
|
921
2198
|
}
|
|
922
|
-
|
|
2199
|
+
if (this.hasReachedMaxIterations(progress, maxIterations)) {
|
|
2200
|
+
progress.status = "failed";
|
|
2201
|
+
progress.lastError = "max_iterations_reached";
|
|
2202
|
+
state.tasks[taskKey] = progress;
|
|
2203
|
+
if (maxIterations !== undefined) {
|
|
2204
|
+
warnings.push(`Task ${taskKey} hit max iterations (${maxIterations}).`);
|
|
2205
|
+
}
|
|
2206
|
+
await this.writeState(state);
|
|
2207
|
+
break;
|
|
2208
|
+
}
|
|
2209
|
+
const attemptIndex = progress.attempts + 1;
|
|
2210
|
+
progress.attempts = attemptIndex;
|
|
2211
|
+
progress.lastError = undefined;
|
|
2212
|
+
progress.lastGuardrailRetryable = undefined;
|
|
923
2213
|
state.tasks[taskKey] = progress;
|
|
924
2214
|
await this.writeState(state);
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
if (workOutcome.status === "blocked") {
|
|
928
|
-
progress.status = "blocked";
|
|
929
|
-
progress.lastError = workOutcome.error ?? "blocked";
|
|
2215
|
+
const workAgentOptions = this.buildAgentOptions(progress, "work", resolvedRequest);
|
|
2216
|
+
progress.lastStep = "work";
|
|
930
2217
|
state.tasks[taskKey] = progress;
|
|
931
2218
|
await this.writeState(state);
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
2219
|
+
const workOutcome = await this.runStepWithTimeout("work", jobId, taskKey, attemptIndex, maxAgentSeconds, (signal) => this.runWorkStep(jobId, attemptIndex, taskKey, projectKey, ["not_started", "in_progress", "changes_requested"], resolvedRequest, warnings, workAgentOptions, signal, this.handoffContextForProgress(progress), async (agent) => {
|
|
2220
|
+
progress.chosenAgents.work = agent;
|
|
2221
|
+
if (!progress.lastWorkAgentTier) {
|
|
2222
|
+
progress.lastWorkAgentTier = "unknown";
|
|
2223
|
+
}
|
|
2224
|
+
state.tasks[taskKey] = progress;
|
|
2225
|
+
await this.writeState(state);
|
|
2226
|
+
}));
|
|
2227
|
+
progress.lastStep = "work";
|
|
2228
|
+
progress.lastError = workOutcome.error;
|
|
2229
|
+
progress.chosenAgents.work = workOutcome.chosenAgent ?? progress.chosenAgents.work;
|
|
2230
|
+
if (workOutcome.tier) {
|
|
2231
|
+
progress.lastWorkAgentTier = workOutcome.tier;
|
|
2232
|
+
}
|
|
2233
|
+
else if (!progress.lastWorkAgentTier) {
|
|
2234
|
+
progress.lastWorkAgentTier = "unknown";
|
|
2235
|
+
}
|
|
2236
|
+
this.recordDecision(progress, "work", attemptIndex, "result", workOutcome.status, workOutcome.error ?? workOutcome.outcome ?? workOutcome.decision);
|
|
2237
|
+
if (this.isAuthFailure(workOutcome.error)) {
|
|
2238
|
+
const message = workOutcome.error ?? AUTH_ERROR_REASON;
|
|
2239
|
+
workOutcome.error = AUTH_ERROR_REASON;
|
|
2240
|
+
progress.lastError = AUTH_ERROR_REASON;
|
|
2241
|
+
this.recordFailure(progress, workOutcome, attemptIndex);
|
|
2242
|
+
progress.status = "failed";
|
|
2243
|
+
state.tasks[taskKey] = progress;
|
|
2244
|
+
await this.writeState(state);
|
|
2245
|
+
if (!resolvedRequest.dryRun) {
|
|
2246
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
2247
|
+
status: "failed",
|
|
2248
|
+
metadata: {
|
|
2249
|
+
...entry.task.metadata,
|
|
2250
|
+
failed_reason: AUTH_ERROR_REASON,
|
|
2251
|
+
},
|
|
2252
|
+
});
|
|
2253
|
+
}
|
|
2254
|
+
warnings.push(`Task ${taskKey} failed due to auth/rate limit during work; continuing run. ${message}`);
|
|
2255
|
+
break;
|
|
2256
|
+
}
|
|
2257
|
+
this.recordFailure(progress, workOutcome, attemptIndex);
|
|
2258
|
+
this.recordRating(progress, workOutcome.ratingSummary);
|
|
937
2259
|
state.tasks[taskKey] = progress;
|
|
938
2260
|
await this.writeState(state);
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
2261
|
+
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
2262
|
+
stage: `task:${taskKey}:work`,
|
|
2263
|
+
timestamp: new Date().toISOString(),
|
|
2264
|
+
details: { taskKey, attempt: attemptIndex, outcome: workOutcome },
|
|
2265
|
+
});
|
|
2266
|
+
if (workOutcome.error === ZERO_TOKEN_ERROR) {
|
|
2267
|
+
if (!resolvedRequest.dryRun) {
|
|
2268
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, { status: "in_progress" });
|
|
2269
|
+
}
|
|
2270
|
+
const zeroTokenCount = this.countFailures(progress, "work", ZERO_TOKEN_ERROR);
|
|
2271
|
+
if (zeroTokenCount >= 2) {
|
|
2272
|
+
progress.status = "failed";
|
|
2273
|
+
progress.lastError = ZERO_TOKEN_ERROR;
|
|
2274
|
+
state.tasks[taskKey] = progress;
|
|
2275
|
+
await this.writeState(state);
|
|
2276
|
+
warnings.push(`Task ${taskKey} failed after repeated zero-token work runs.`);
|
|
2277
|
+
break;
|
|
2278
|
+
}
|
|
2279
|
+
warnings.push(`Retrying ${taskKey} after zero-token work run.`);
|
|
2280
|
+
state.tasks[taskKey] = progress;
|
|
2281
|
+
await this.writeState(state);
|
|
2282
|
+
await this.backoffZeroTokens(zeroTokenCount);
|
|
2283
|
+
continue;
|
|
2284
|
+
}
|
|
2285
|
+
if (workOutcome.error === "tests_failed") {
|
|
2286
|
+
const testsFailedCount = this.countFailures(progress, "work", "tests_failed");
|
|
2287
|
+
if (testsFailedCount >= 2) {
|
|
2288
|
+
progress.status = "failed";
|
|
2289
|
+
progress.lastError = "tests_failed";
|
|
2290
|
+
state.tasks[taskKey] = progress;
|
|
2291
|
+
await this.writeState(state);
|
|
2292
|
+
if (!resolvedRequest.dryRun) {
|
|
2293
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
2294
|
+
status: "failed",
|
|
2295
|
+
metadata: {
|
|
2296
|
+
...entry.task.metadata,
|
|
2297
|
+
failed_reason: "tests_failed",
|
|
2298
|
+
},
|
|
2299
|
+
});
|
|
2300
|
+
}
|
|
2301
|
+
warnings.push(`Task ${taskKey} failed after repeated tests_failed.`);
|
|
2302
|
+
break;
|
|
2303
|
+
}
|
|
2304
|
+
warnings.push(`Retrying ${taskKey} after tests_failed with stronger agent.`);
|
|
2305
|
+
state.tasks[taskKey] = progress;
|
|
2306
|
+
await this.writeState(state);
|
|
2307
|
+
continue;
|
|
2308
|
+
}
|
|
2309
|
+
const workGatewayFailure = workOutcome.status === "failed" &&
|
|
2310
|
+
normalizeFailureReason(workOutcome.error ?? workOutcome.status) === GATEWAY_FAILED_REASON;
|
|
2311
|
+
if (workGatewayFailure) {
|
|
2312
|
+
progress.status = "failed";
|
|
2313
|
+
progress.lastError = GATEWAY_FAILED_REASON;
|
|
2314
|
+
state.tasks[taskKey] = progress;
|
|
2315
|
+
await this.writeState(state);
|
|
2316
|
+
if (!resolvedRequest.dryRun) {
|
|
2317
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
2318
|
+
status: "failed",
|
|
2319
|
+
metadata: {
|
|
2320
|
+
...entry.task.metadata,
|
|
2321
|
+
failed_reason: GATEWAY_FAILED_REASON,
|
|
2322
|
+
},
|
|
2323
|
+
});
|
|
2324
|
+
}
|
|
2325
|
+
warnings.push(`Task ${taskKey} failed due to gateway failure during work.`);
|
|
2326
|
+
break;
|
|
2327
|
+
}
|
|
2328
|
+
if (workOutcome.status === "skipped") {
|
|
2329
|
+
progress.status = "skipped";
|
|
2330
|
+
progress.lastError = workOutcome.error ?? "skipped";
|
|
2331
|
+
this.recordDecision(progress, "work", attemptIndex, "terminal", "skipped", progress.lastError);
|
|
2332
|
+
state.tasks[taskKey] = progress;
|
|
2333
|
+
await this.writeState(state);
|
|
2334
|
+
break;
|
|
2335
|
+
}
|
|
2336
|
+
const retryWork = this.shouldRetryAfter(workOutcome);
|
|
2337
|
+
if (workOutcome.status !== "succeeded" && retryWork) {
|
|
2338
|
+
this.recordDecision(progress, "work", attemptIndex, "retry", "retry", workOutcome.error ?? workOutcome.status);
|
|
943
2339
|
warnings.push(`Retrying ${taskKey} after work step (${workOutcome.status}).`);
|
|
944
2340
|
state.tasks[taskKey] = progress;
|
|
945
2341
|
await this.writeState(state);
|
|
946
2342
|
continue;
|
|
947
2343
|
}
|
|
948
|
-
|
|
949
|
-
|
|
2344
|
+
if (workOutcome.status === "failed" && workOutcome.guardrailRetryable === false) {
|
|
2345
|
+
const reason = normalizeFailureReason(workOutcome.guardrailReason ?? workOutcome.error ?? workOutcome.status) ??
|
|
2346
|
+
workOutcome.guardrailReason ??
|
|
2347
|
+
workOutcome.error ??
|
|
2348
|
+
"failed";
|
|
2349
|
+
progress.status = "failed";
|
|
2350
|
+
progress.lastError = reason;
|
|
2351
|
+
this.recordDecision(progress, "work", attemptIndex, "terminal", "failed", reason);
|
|
2352
|
+
state.tasks[taskKey] = progress;
|
|
2353
|
+
await this.writeState(state);
|
|
2354
|
+
if (!resolvedRequest.dryRun) {
|
|
2355
|
+
await this.deps.workspaceRepo.updateTask(entry.task.id, {
|
|
2356
|
+
status: "failed",
|
|
2357
|
+
metadata: {
|
|
2358
|
+
...entry.task.metadata,
|
|
2359
|
+
failed_reason: reason,
|
|
2360
|
+
},
|
|
2361
|
+
});
|
|
2362
|
+
}
|
|
2363
|
+
warnings.push(`Task ${taskKey} failed with non-retryable work reason ${reason}.`);
|
|
2364
|
+
break;
|
|
2365
|
+
}
|
|
2366
|
+
if (resolvedRequest.dryRun) {
|
|
2367
|
+
setDryRunStatus(READY_TO_CODE_REVIEW);
|
|
2368
|
+
continue;
|
|
2369
|
+
}
|
|
950
2370
|
const statusAfterWork = await this.refreshTaskStatus(taskKey, warnings);
|
|
951
|
-
if (statusAfterWork &&
|
|
2371
|
+
if (statusAfterWork &&
|
|
2372
|
+
(isReadyToReviewStatus(statusAfterWork) || ["ready_to_qa", "completed"].includes(statusAfterWork))) {
|
|
2373
|
+
currentStatus = statusAfterWork;
|
|
2374
|
+
continue;
|
|
2375
|
+
}
|
|
2376
|
+
if (statusAfterWork && statusAfterWork === "failed") {
|
|
2377
|
+
progress.status = "failed";
|
|
2378
|
+
progress.lastError = "failed";
|
|
2379
|
+
state.tasks[taskKey] = progress;
|
|
2380
|
+
await this.writeState(state);
|
|
2381
|
+
break;
|
|
2382
|
+
}
|
|
2383
|
+
if (statusAfterWork && statusAfterWork === "blocked") {
|
|
2384
|
+
progress.status = "failed";
|
|
2385
|
+
progress.lastError = "legacy_blocked";
|
|
2386
|
+
state.tasks[taskKey] = progress;
|
|
2387
|
+
await this.writeState(state);
|
|
2388
|
+
warnings.push(`Task ${taskKey} had legacy blocked status after work; marked failed.`);
|
|
2389
|
+
break;
|
|
2390
|
+
}
|
|
2391
|
+
if (statusAfterWork) {
|
|
952
2392
|
warnings.push(`Task ${taskKey} status ${statusAfterWork} after work; retrying work step.`);
|
|
2393
|
+
progress.lastError = "work_status_not_ready";
|
|
2394
|
+
const statusFailure = {
|
|
2395
|
+
step: "work",
|
|
2396
|
+
status: "failed",
|
|
2397
|
+
error: "work_status_not_ready",
|
|
2398
|
+
chosenAgent: progress.chosenAgents.work,
|
|
2399
|
+
};
|
|
2400
|
+
this.recordFailure(progress, statusFailure, attemptIndex);
|
|
2401
|
+
state.tasks[taskKey] = progress;
|
|
2402
|
+
await this.writeState(state);
|
|
953
2403
|
continue;
|
|
954
2404
|
}
|
|
955
|
-
|
|
956
|
-
}
|
|
957
|
-
else {
|
|
958
|
-
progress.lastStep = "work";
|
|
959
|
-
progress.lastError = undefined;
|
|
960
|
-
state.tasks[taskKey] = progress;
|
|
961
|
-
await this.writeState(state);
|
|
962
|
-
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
963
|
-
stage: `task:${taskKey}:work:skipped`,
|
|
964
|
-
timestamp: new Date().toISOString(),
|
|
965
|
-
details: { taskKey, attempt: attemptIndex, reason: statusBefore ?? "ready" },
|
|
966
|
-
});
|
|
967
|
-
}
|
|
968
|
-
if (!reviewWasSkipped) {
|
|
969
|
-
const reviewAgentOptions = this.buildAgentOptions(progress, "review", resolvedRequest);
|
|
970
|
-
progress.lastStep = "review";
|
|
971
|
-
state.tasks[taskKey] = progress;
|
|
972
|
-
await this.writeState(state);
|
|
973
|
-
const reviewOutcome = await this.runStepWithTimeout("review", jobId, taskKey, attemptIndex, maxAgentSeconds, (signal) => this.runReviewStep(jobId, attemptIndex, taskKey, projectKey, ["ready_to_review", ...statusFilter], resolvedRequest, warnings, reviewAgentOptions, signal, async (agent) => {
|
|
974
|
-
progress.chosenAgents.review = agent;
|
|
975
|
-
state.tasks[taskKey] = progress;
|
|
976
|
-
await this.writeState(state);
|
|
977
|
-
}));
|
|
978
|
-
progress.lastStep = "review";
|
|
979
|
-
progress.lastDecision = reviewOutcome.decision;
|
|
980
|
-
progress.lastError = reviewOutcome.error;
|
|
981
|
-
progress.chosenAgents.review = reviewOutcome.chosenAgent ?? progress.chosenAgents.review;
|
|
982
|
-
this.recordFailure(progress, reviewOutcome, attemptIndex);
|
|
983
|
-
this.recordRating(progress, reviewOutcome.ratingSummary);
|
|
984
|
-
state.tasks[taskKey] = progress;
|
|
985
|
-
await this.writeState(state);
|
|
986
|
-
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
987
|
-
stage: `task:${taskKey}:review`,
|
|
988
|
-
timestamp: new Date().toISOString(),
|
|
989
|
-
details: { taskKey, attempt: attemptIndex, outcome: reviewOutcome },
|
|
990
|
-
});
|
|
991
|
-
if (reviewOutcome.status === "blocked") {
|
|
992
|
-
progress.status = "blocked";
|
|
993
|
-
progress.lastError = reviewOutcome.error ?? "blocked";
|
|
994
|
-
state.tasks[taskKey] = progress;
|
|
995
|
-
await this.writeState(state);
|
|
996
|
-
continue;
|
|
997
|
-
}
|
|
998
|
-
if (this.shouldRetryAfter(reviewOutcome)) {
|
|
999
|
-
warnings.push(`Retrying ${taskKey} after review (${reviewOutcome.decision ?? reviewOutcome.status}).`);
|
|
1000
|
-
state.tasks[taskKey] = progress;
|
|
1001
|
-
await this.writeState(state);
|
|
2405
|
+
warnings.push(`Task ${taskKey} status missing after work; retrying work step.`);
|
|
1002
2406
|
continue;
|
|
1003
2407
|
}
|
|
1004
|
-
if (!
|
|
1005
|
-
const
|
|
1006
|
-
if (
|
|
1007
|
-
|
|
1008
|
-
|
|
2408
|
+
if (attempted && !abortRemainingReason) {
|
|
2409
|
+
const finalProgress = currentTaskKey ? state.tasks[currentTaskKey] : undefined;
|
|
2410
|
+
if (finalProgress) {
|
|
2411
|
+
if (finalProgress.status === "completed") {
|
|
2412
|
+
activeTaskKey = null;
|
|
2413
|
+
}
|
|
2414
|
+
else {
|
|
2415
|
+
activeTaskKey = currentTaskKey ?? activeTaskKey;
|
|
2416
|
+
holdAfterTask = true;
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
if (typeof taskLimit === "number" && taskLimit > 0 && completedKeys.size >= taskLimit) {
|
|
2420
|
+
warnings.push(`Completed task limit ${taskLimit} reached; stopping run.`);
|
|
2421
|
+
abortRemainingReason = "limit_reached";
|
|
1009
2422
|
}
|
|
1010
2423
|
}
|
|
1011
2424
|
}
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
1019
|
-
stage: `task:${taskKey}:review:skipped`,
|
|
1020
|
-
timestamp: new Date().toISOString(),
|
|
1021
|
-
details: { taskKey, attempt: attemptIndex, reason: statusBefore ?? "ready_to_qa" },
|
|
1022
|
-
});
|
|
1023
|
-
}
|
|
1024
|
-
progress.lastStep = "qa";
|
|
1025
|
-
state.tasks[taskKey] = progress;
|
|
1026
|
-
await this.writeState(state);
|
|
1027
|
-
const qaOutcome = await this.runStepWithTimeout("qa", jobId, taskKey, attemptIndex, maxAgentSeconds, (signal) => this.runQaStep(jobId, attemptIndex, taskKey, projectKey, ["ready_to_qa", ...statusFilter], resolvedRequest, warnings, this.buildAgentOptions(progress, "qa", resolvedRequest), signal, async (agent) => {
|
|
1028
|
-
progress.chosenAgents.qa = agent;
|
|
1029
|
-
state.tasks[taskKey] = progress;
|
|
1030
|
-
await this.writeState(state);
|
|
1031
|
-
}));
|
|
1032
|
-
progress.lastStep = "qa";
|
|
1033
|
-
progress.lastOutcome = qaOutcome.outcome;
|
|
1034
|
-
progress.lastError = qaOutcome.error;
|
|
1035
|
-
progress.chosenAgents.qa = qaOutcome.chosenAgent ?? progress.chosenAgents.qa;
|
|
1036
|
-
this.recordFailure(progress, qaOutcome, attemptIndex);
|
|
1037
|
-
this.recordRating(progress, qaOutcome.ratingSummary);
|
|
1038
|
-
state.tasks[taskKey] = progress;
|
|
1039
|
-
await this.writeState(state);
|
|
1040
|
-
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
1041
|
-
stage: `task:${taskKey}:qa`,
|
|
1042
|
-
timestamp: new Date().toISOString(),
|
|
1043
|
-
details: { taskKey, attempt: attemptIndex, outcome: qaOutcome },
|
|
1044
|
-
});
|
|
1045
|
-
if (qaOutcome.status === "blocked") {
|
|
1046
|
-
progress.status = "blocked";
|
|
1047
|
-
progress.lastError = qaOutcome.error ?? "blocked";
|
|
1048
|
-
state.tasks[taskKey] = progress;
|
|
1049
|
-
await this.writeState(state);
|
|
1050
|
-
continue;
|
|
1051
|
-
}
|
|
1052
|
-
if (this.shouldRetryAfter(qaOutcome)) {
|
|
1053
|
-
warnings.push(`Retrying ${taskKey} after QA (${qaOutcome.outcome ?? qaOutcome.status}).`);
|
|
1054
|
-
state.tasks[taskKey] = progress;
|
|
1055
|
-
await this.writeState(state);
|
|
1056
|
-
continue;
|
|
1057
|
-
}
|
|
1058
|
-
if (!resolvedRequest.dryRun) {
|
|
1059
|
-
const statusAfterQa = await this.refreshTaskStatus(taskKey, warnings);
|
|
1060
|
-
if (statusAfterQa && statusAfterQa !== "completed") {
|
|
1061
|
-
warnings.push(`Task ${taskKey} status ${statusAfterQa} after QA; retrying work step.`);
|
|
1062
|
-
state.tasks[taskKey] = progress;
|
|
1063
|
-
await this.writeState(state);
|
|
1064
|
-
continue;
|
|
2425
|
+
finally {
|
|
2426
|
+
if (attempted) {
|
|
2427
|
+
processedThisCycle += 1;
|
|
2428
|
+
await this.deps.jobService.updateJobStatus(jobId, "running", {
|
|
2429
|
+
processedItems: processedThisCycle,
|
|
2430
|
+
});
|
|
1065
2431
|
}
|
|
1066
2432
|
}
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
await this.writeState(state);
|
|
1070
|
-
completedThisCycle += 1;
|
|
1071
|
-
processedThisCycle += 1;
|
|
1072
|
-
await this.deps.jobService.updateJobStatus(jobId, "running", {
|
|
1073
|
-
processedItems: processedThisCycle,
|
|
1074
|
-
});
|
|
2433
|
+
if (abortRemainingReason || holdAfterTask)
|
|
2434
|
+
break;
|
|
1075
2435
|
}
|
|
2436
|
+
if (abortRemainingReason)
|
|
2437
|
+
break;
|
|
1076
2438
|
cycle += 1;
|
|
1077
2439
|
state.cycle = cycle;
|
|
1078
2440
|
await this.writeState(state);
|
|
1079
2441
|
if (attemptedThisCycle === 0) {
|
|
1080
|
-
const
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
2442
|
+
const hasRemaining = Object.values(state.tasks).some((task) => {
|
|
2443
|
+
if (task.status === "completed" || task.status === "skipped")
|
|
2444
|
+
return false;
|
|
2445
|
+
if (maxIterations !== undefined && task.attempts >= maxIterations)
|
|
2446
|
+
return false;
|
|
2447
|
+
return true;
|
|
2448
|
+
});
|
|
2449
|
+
if (hasRemaining) {
|
|
2450
|
+
warnings.push("No tasks attempted in this cycle; tasks remain, continuing.");
|
|
2451
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
1087
2452
|
continue;
|
|
1088
2453
|
}
|
|
1089
|
-
warnings.push("No tasks attempted in this cycle; stopping
|
|
2454
|
+
warnings.push("No tasks attempted in this cycle; stopping.");
|
|
1090
2455
|
break;
|
|
1091
2456
|
}
|
|
1092
2457
|
}
|
|
1093
2458
|
const summaries = this.toSummary(state);
|
|
1094
|
-
const blocked = summaries.filter((t) => t.status === "blocked").map((t) => t.taskKey);
|
|
1095
2459
|
const failed = summaries.filter((t) => t.status === "failed").map((t) => t.taskKey);
|
|
1096
2460
|
const skipped = summaries.filter((t) => t.status === "skipped").map((t) => t.taskKey);
|
|
1097
2461
|
const pending = summaries.filter((t) => t.status === "pending").map((t) => t.taskKey);
|
|
1098
|
-
const failureCount = failed.length +
|
|
1099
|
-
const
|
|
1100
|
-
const
|
|
2462
|
+
const failureCount = failed.length + skipped.length + pending.length;
|
|
2463
|
+
const jobSnapshot = await this.deps.jobService.getJob(jobId);
|
|
2464
|
+
const jobStateNow = resolveJobState(jobSnapshot);
|
|
2465
|
+
const cancelled = jobStateNow === "cancelled" || abortRemainingReason === "cancelled";
|
|
2466
|
+
const endState = cancelled ? "cancelled" : failureCount === 0 ? "completed" : "partial";
|
|
2467
|
+
const errorSummary = cancelled ? "Job cancelled by user" : failureCount ? `${failureCount} task(s) not fully completed` : undefined;
|
|
1101
2468
|
await this.deps.jobService.updateJobStatus(jobId, endState, { errorSummary });
|
|
1102
|
-
|
|
2469
|
+
const commandStatus = endState === "completed" ? "succeeded" : endState === "cancelled" ? "cancelled" : "failed";
|
|
2470
|
+
await this.deps.jobService.finishCommandRun(commandRun.id, commandStatus, errorSummary);
|
|
1103
2471
|
await this.deps.jobService.writeCheckpoint(jobId, {
|
|
1104
2472
|
stage: "completed",
|
|
1105
2473
|
timestamp: new Date().toISOString(),
|
|
@@ -1110,7 +2478,6 @@ export class GatewayTrioService {
|
|
|
1110
2478
|
commandRunId: commandRun.id,
|
|
1111
2479
|
tasks: summaries,
|
|
1112
2480
|
warnings,
|
|
1113
|
-
blocked,
|
|
1114
2481
|
failed,
|
|
1115
2482
|
skipped,
|
|
1116
2483
|
};
|