@tea-agent/loop-agent 0.31.1 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +46 -0
- package/README.md +1 -1
- package/dist/shared/operator/capabilities.js +22 -1
- package/dist/shared/operator/command-lifecycle.js +94 -0
- package/dist/shared/operator/index.js +1 -0
- package/dist/worker/cli.js +10 -24
- package/dist/worker/console/doctor.js +11 -4
- package/dist/worker/console/observe-health-match.js +18 -15
- package/dist/worker/console/operator-actions.js +2 -1
- package/dist/worker/console/static/assets/index-Bpa2qrc-.js +29 -0
- package/dist/worker/console/static/assets/index-BqfFDdnG.css +1 -0
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/console/static-src/app/console-types.js +140 -0
- package/dist/worker/console/static-src/app/useConsoleShell.js +101 -0
- package/dist/worker/console/static-src/app/useOperatorActions.js +304 -0
- package/dist/worker/console/static-src/app/usePrdImport.js +171 -0
- package/dist/worker/console/static-src/app/useRecoveryActions.js +257 -0
- package/dist/worker/console/static-src/app/useRecoveryConsole.js +334 -0
- package/dist/worker/console/static-src/app/useTaskWizard.js +229 -0
- package/dist/worker/console/static-src/chat-view-types.js +2 -0
- package/dist/worker/console/static-src/night/night-types.js +24 -0
- package/dist/worker/console/static-src/night/useNightBoard.js +125 -0
- package/dist/worker/console/static-src/night/useNightWizard.js +171 -0
- package/dist/worker/console/static-src/operator-chat/chat-sse-events.js +253 -0
- package/dist/worker/console/static-src/operator-chat/format.js +71 -0
- package/dist/worker/console/static-src/operator-chat/refs.js +47 -0
- package/dist/worker/console/static-src/operator-chat/tools-catalog.js +77 -0
- package/dist/worker/console/static-src/operator-chat/types.js +1 -0
- package/dist/worker/console/static-src/operator-chat/useChatSessions.js +320 -0
- package/dist/worker/console/static-src/operator-chat/useChatStream.js +209 -0
- package/dist/worker/console/static-src/operator-chat/useChatThread.js +218 -0
- package/dist/worker/console/static-src/operator-chat/useComposer.js +125 -0
- package/dist/worker/console/static-src/operator-chat/useInterview.js +108 -0
- package/dist/worker/console/static-src/operator-chat/useRepoBrowser.js +123 -0
- package/dist/worker/console/static-src/operator-chat/useRuntimeControls.js +207 -0
- package/dist/worker/feature/acceptance-policy.js +227 -0
- package/dist/worker/feature/decision-loader.js +1 -1
- package/dist/worker/feature/next-action.js +56 -6
- package/dist/worker/feature/profile-schema.js +3 -0
- package/dist/worker/feature/reducer.js +1 -0
- package/dist/worker/feature/review.js +72 -8
- package/dist/worker/feature/scaffold.js +14 -0
- package/dist/worker/loop-agent/controller-protocol.js +143 -0
- package/dist/worker/materialize/harness-task-lifecycle-probe.js +126 -0
- package/dist/worker/materialize/harness-task-lineage.js +220 -0
- package/dist/worker/materialize/harness-task-materializer.js +350 -80
- package/dist/worker/observability/progress-composite.js +1 -0
- package/dist/worker/observability/read-model.js +66 -19
- package/dist/worker/pool/attempt-identity.js +41 -0
- package/dist/worker/pool/attempt-lease.js +184 -0
- package/dist/worker/pool/attempt-transition.js +210 -0
- package/dist/worker/pool/begin-attempt-with-lease.js +26 -0
- package/dist/worker/pool/begin-attempt.js +35 -0
- package/dist/worker/pool/failure-routing.js +49 -0
- package/dist/worker/pool/recovery-decision.js +163 -0
- package/dist/worker/pool/run-owner-store.js +126 -0
- package/dist/worker/pool/run-store.js +32 -46
- package/dist/worker/pool/runtime-reconcile-inventory.js +127 -0
- package/dist/worker/pool/state-projection.js +57 -0
- package/dist/worker/run-task/run-task.js +31 -4
- package/dist/worker/runner/run-ready.js +64 -14
- package/dist/worker/runner/single-task-attempt.js +42 -13
- package/dist/worker/task-graph/acceptance-schema.js +3 -0
- package/docs/README.md +3 -1
- package/docs/architecture/evolution.md +1 -1
- package/docs/operations/README.md +1 -0
- package/docs/templates/README.md +1 -0
- package/docs/templates/agent-worker-production-readiness-checklist.md +47 -0
- package/docs/templates/evaluation/agents-map-slim-v1.md +1 -1
- package/docs/templates/evaluation/agents-map-verbose-v0.md +1 -1
- package/docs/templates/init-managed-agents.md +1 -1
- package/docs/templates/product-line/scaffold-samples/backend-only/acceptance.yaml +3 -0
- package/docs/templates/product-line/scaffold-samples/backend-only/feature.yaml +11 -0
- package/docs/templates/product-line/scaffold-samples/fe-with-api/acceptance.yaml +3 -0
- package/docs/templates/product-line/scaffold-samples/fe-with-api/feature.yaml +11 -0
- package/docs/templates/product-line/scaffold-samples/frontend-only/acceptance.yaml +3 -0
- package/docs/templates/product-line/scaffold-samples/frontend-only/feature.yaml +11 -0
- package/package.json +5 -4
- package/skills/agent-worker/SKILL.md +1 -1
- package/skills/agent-worker/references/agent-worker-operator.md +1 -1
- package/skills/loop-agent/references/command-reference.md +2 -2
- package/skills/loop-agent/references/harness-policy.md +1 -1
- package/dist/worker/console/static/assets/index-D9gnJn_l.js +0 -29
- package/dist/worker/console/static/assets/index-rajoXwkM.css +0 -1
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { createWorkerAttemptIdentity } from "./attempt-identity.js";
|
|
2
|
+
import { registerRunOwner } from "./run-owner-store.js";
|
|
3
|
+
import { writeTaskPoolState } from "./run-store.js";
|
|
4
|
+
/**
|
|
5
|
+
* Attempt begin transaction (Phase 1/4 foundation):
|
|
6
|
+
* register immutable owner → write Running state.
|
|
7
|
+
* Callers then emit required lifecycle events and launch DAG.
|
|
8
|
+
*/
|
|
9
|
+
export async function beginWorkerAttempt(input) {
|
|
10
|
+
const now = input.now ?? new Date();
|
|
11
|
+
const identity = createWorkerAttemptIdentity({
|
|
12
|
+
featureId: input.featureId,
|
|
13
|
+
taskId: input.taskId,
|
|
14
|
+
workerRunId: input.workerRunId,
|
|
15
|
+
batchRunId: input.batchRunId,
|
|
16
|
+
});
|
|
17
|
+
const ownerRegistration = await registerRunOwner({
|
|
18
|
+
repoRoot: input.repoRoot,
|
|
19
|
+
identity,
|
|
20
|
+
now,
|
|
21
|
+
...(input.controllerAnchor ? { controllerAnchor: input.controllerAnchor } : {}),
|
|
22
|
+
...(input.revisionId ? { revisionId: input.revisionId } : {}),
|
|
23
|
+
});
|
|
24
|
+
const state = {
|
|
25
|
+
schemaVersion: 2,
|
|
26
|
+
featureId: identity.featureId,
|
|
27
|
+
taskId: identity.taskId,
|
|
28
|
+
status: "Running",
|
|
29
|
+
updatedAt: now.toISOString(),
|
|
30
|
+
workerRunId: identity.workerRunId,
|
|
31
|
+
...(input.stateExtras ?? {}),
|
|
32
|
+
};
|
|
33
|
+
await writeTaskPoolState(input.repoRoot, state);
|
|
34
|
+
return { identity, ownerRegistration, state };
|
|
35
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { defaultRetryabilityForCategory, recommendedActionForRetryability, } from "./recovery-decision.js";
|
|
1
2
|
const PRODUCT_CATEGORIES = new Set([
|
|
2
3
|
"SpecUnclear",
|
|
3
4
|
"ContractMismatch",
|
|
@@ -82,6 +83,54 @@ export function deriveFailureRouteFromError(message, businessId) {
|
|
|
82
83
|
source: "runner-error",
|
|
83
84
|
};
|
|
84
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* Phase-aware V2 failure for new writers. V1 FailureRoute remains available.
|
|
88
|
+
*/
|
|
89
|
+
export function deriveFailureRouteV2(result, phase = "execute") {
|
|
90
|
+
const base = deriveFailureRoute(result);
|
|
91
|
+
if (!base)
|
|
92
|
+
return undefined;
|
|
93
|
+
const retryability = defaultRetryabilityForCategory(base.category, phase);
|
|
94
|
+
return {
|
|
95
|
+
...base,
|
|
96
|
+
schemaVersion: 2,
|
|
97
|
+
phase,
|
|
98
|
+
retryability,
|
|
99
|
+
recommendedAction: recommendedActionForRetryability(retryability),
|
|
100
|
+
message: result.reportDecision.reason,
|
|
101
|
+
evidence: [
|
|
102
|
+
result.runRecordPath,
|
|
103
|
+
result.dagPath,
|
|
104
|
+
...(result.failureArtifacts
|
|
105
|
+
? Object.values(result.failureArtifacts).filter((value) => typeof value === "string")
|
|
106
|
+
: []),
|
|
107
|
+
],
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
export function deriveFailureRouteV2FromError(message, businessId, phase = "prepare") {
|
|
111
|
+
const base = deriveFailureRouteFromError(message, businessId);
|
|
112
|
+
const retryability = defaultRetryabilityForCategory(base.category, phase);
|
|
113
|
+
return {
|
|
114
|
+
...base,
|
|
115
|
+
schemaVersion: 2,
|
|
116
|
+
phase,
|
|
117
|
+
retryability,
|
|
118
|
+
recommendedAction: recommendedActionForRetryability(retryability),
|
|
119
|
+
message,
|
|
120
|
+
evidence: [],
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
export function toWorkerAttemptFailureV2(route) {
|
|
124
|
+
return {
|
|
125
|
+
schemaVersion: 2,
|
|
126
|
+
phase: route.phase,
|
|
127
|
+
category: route.category,
|
|
128
|
+
message: route.message ?? route.recommendedFollowUpKind,
|
|
129
|
+
retryability: route.retryability,
|
|
130
|
+
recommendedAction: route.recommendedAction,
|
|
131
|
+
evidence: route.evidence ?? [],
|
|
132
|
+
};
|
|
133
|
+
}
|
|
85
134
|
function readProductCategory(value, key) {
|
|
86
135
|
const candidate = readString(value, key);
|
|
87
136
|
return candidate && PRODUCT_CATEGORIES.has(candidate)
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Single recovery decision for CLI / Feature Review / Console / Scheduler.
|
|
3
|
+
* Uncertain facts → reconcile or decide; never guess retry.
|
|
4
|
+
*/
|
|
5
|
+
export function resolveWorkerRecoveryDecision(facts) {
|
|
6
|
+
if (facts.factsIncomplete) {
|
|
7
|
+
return {
|
|
8
|
+
recommendedAction: "reconcile",
|
|
9
|
+
reason: "attempt facts are incomplete or conflicting",
|
|
10
|
+
retryability: "requires-reconcile",
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
const failureV2 = facts.state?.failureV2 ?? facts.terminalRun?.failureV2 ?? undefined;
|
|
14
|
+
if (failureV2) {
|
|
15
|
+
return decisionFromFailureV2(failureV2);
|
|
16
|
+
}
|
|
17
|
+
const status = facts.state?.status;
|
|
18
|
+
if (status === "Done") {
|
|
19
|
+
return { recommendedAction: "none", reason: "attempt already succeeded" };
|
|
20
|
+
}
|
|
21
|
+
if (status === "Running" || status === "Queued") {
|
|
22
|
+
if (facts.dagCanResume) {
|
|
23
|
+
return {
|
|
24
|
+
recommendedAction: "resume",
|
|
25
|
+
reason: "DAG is paused or recoverable; resume same attempt",
|
|
26
|
+
retryability: "requires-resume",
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
if (facts.hasActiveOrRecoverableDag) {
|
|
30
|
+
return {
|
|
31
|
+
recommendedAction: "none",
|
|
32
|
+
reason: "attempt is still active",
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
recommendedAction: "reconcile",
|
|
37
|
+
reason: "Running state without live DAG or terminal evidence",
|
|
38
|
+
retryability: "requires-reconcile",
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
if (status === "Blocked" || status === "HumanReview") {
|
|
42
|
+
const category = facts.state?.failure?.category;
|
|
43
|
+
if (category === "ContractMismatch" ||
|
|
44
|
+
category === "SpecUnclear" ||
|
|
45
|
+
category === "Unknown") {
|
|
46
|
+
return {
|
|
47
|
+
recommendedAction: "revise-contract",
|
|
48
|
+
reason: "contract/risk prerequisite blocks execution",
|
|
49
|
+
retryability: "requires-contract-revision",
|
|
50
|
+
category,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
recommendedAction: "decide",
|
|
55
|
+
reason: "human gate or non-retryable block",
|
|
56
|
+
retryability: "requires-human",
|
|
57
|
+
category,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
if (status === "Failed" || status === "Abandoned") {
|
|
61
|
+
const category = facts.state?.failure?.category ?? facts.terminalRun?.failure?.category;
|
|
62
|
+
if (category === "EnvFailure" ||
|
|
63
|
+
category === "FlakyTest" ||
|
|
64
|
+
category === "ProductBug" ||
|
|
65
|
+
category === "TestBug" ||
|
|
66
|
+
facts.terminalRun?.status === "run-error" ||
|
|
67
|
+
facts.terminalRun?.status === "failed") {
|
|
68
|
+
return {
|
|
69
|
+
recommendedAction: "retry",
|
|
70
|
+
reason: "terminal failure is safe to retry as a new attempt",
|
|
71
|
+
retryability: "safe",
|
|
72
|
+
category,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
return {
|
|
76
|
+
recommendedAction: "decide",
|
|
77
|
+
reason: "terminal failure requires human review before retry",
|
|
78
|
+
retryability: "requires-human",
|
|
79
|
+
category,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
recommendedAction: "none",
|
|
84
|
+
reason: "no recovery action for current facts",
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
function decisionFromFailureV2(failure) {
|
|
88
|
+
return {
|
|
89
|
+
recommendedAction: failure.recommendedAction,
|
|
90
|
+
reason: failure.message,
|
|
91
|
+
retryability: failure.retryability,
|
|
92
|
+
phase: failure.phase,
|
|
93
|
+
category: failure.category,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
export function isSafeRetryDecision(decision) {
|
|
97
|
+
return (decision.recommendedAction === "retry" &&
|
|
98
|
+
(decision.retryability === "safe" || decision.retryability === undefined));
|
|
99
|
+
}
|
|
100
|
+
export function defaultRetryabilityForCategory(category, phase) {
|
|
101
|
+
if (phase === "prepare" || phase === "materialize") {
|
|
102
|
+
if (category === "ContractMismatch" ||
|
|
103
|
+
category === "SpecUnclear" ||
|
|
104
|
+
category === "Unknown") {
|
|
105
|
+
return "requires-contract-revision";
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
switch (category) {
|
|
109
|
+
case "EnvFailure":
|
|
110
|
+
case "FlakyTest":
|
|
111
|
+
case "ProductBug":
|
|
112
|
+
case "TestBug":
|
|
113
|
+
return "safe";
|
|
114
|
+
case "DependencyFailure":
|
|
115
|
+
return "requires-human";
|
|
116
|
+
case "RiskyChange":
|
|
117
|
+
case "NeedsHuman":
|
|
118
|
+
return "requires-human";
|
|
119
|
+
case "ContractMismatch":
|
|
120
|
+
case "SpecUnclear":
|
|
121
|
+
return "requires-contract-revision";
|
|
122
|
+
case "Unknown":
|
|
123
|
+
return phase === "prepare" || phase === "materialize"
|
|
124
|
+
? "requires-contract-revision"
|
|
125
|
+
: "requires-human";
|
|
126
|
+
default:
|
|
127
|
+
return "requires-human";
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
export function recommendedActionForRetryability(retryability, _status) {
|
|
131
|
+
switch (retryability) {
|
|
132
|
+
case "safe":
|
|
133
|
+
return "retry";
|
|
134
|
+
case "requires-resume":
|
|
135
|
+
return "resume";
|
|
136
|
+
case "requires-reconcile":
|
|
137
|
+
return "reconcile";
|
|
138
|
+
case "requires-contract-revision":
|
|
139
|
+
return "revise-contract";
|
|
140
|
+
case "requires-human":
|
|
141
|
+
case "never":
|
|
142
|
+
return "decide";
|
|
143
|
+
default:
|
|
144
|
+
return "decide";
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/** @deprecated Use recommendedActionForRetryability */
|
|
148
|
+
export const actionFromRetryability = recommendedActionForRetryability;
|
|
149
|
+
export function inferPhaseFromRunStatus(runStatus) {
|
|
150
|
+
if (runStatus === "run-error")
|
|
151
|
+
return "prepare";
|
|
152
|
+
if (runStatus === "failed")
|
|
153
|
+
return "execute";
|
|
154
|
+
if (runStatus === "succeeded")
|
|
155
|
+
return "record";
|
|
156
|
+
return "execute";
|
|
157
|
+
}
|
|
158
|
+
export function inferRetryability(category, phase, runStatus) {
|
|
159
|
+
if (category === "EnvFailure" && runStatus === "failed") {
|
|
160
|
+
return "requires-human";
|
|
161
|
+
}
|
|
162
|
+
return defaultRetryabilityForCategory(category, phase);
|
|
163
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { mkdir, readFile, readdir } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
4
|
+
import { assertWorkerAttemptIdentity, createWorkerAttemptIdentity, ownerMatchesIdentity, } from "./attempt-identity.js";
|
|
5
|
+
import { assertSafeTaskPoolId, getTaskPoolRoot } from "./run-store.js";
|
|
6
|
+
import { TASK_POOL_STATE_ERROR_CODES, TaskPoolStateError } from "./types.js";
|
|
7
|
+
export const RUN_OWNERS_RELATIVE = "run-owners";
|
|
8
|
+
export function getRunOwnersRoot(repoRoot) {
|
|
9
|
+
return path.join(getTaskPoolRoot(repoRoot), RUN_OWNERS_RELATIVE);
|
|
10
|
+
}
|
|
11
|
+
export function getRunOwnerPath(repoRoot, workerRunId) {
|
|
12
|
+
assertSafeTaskPoolId(workerRunId, "workerRunId");
|
|
13
|
+
const ownerPath = path.join(getRunOwnersRoot(repoRoot), `${workerRunId}.json`);
|
|
14
|
+
const root = path.resolve(getRunOwnersRoot(repoRoot));
|
|
15
|
+
const resolved = path.resolve(ownerPath);
|
|
16
|
+
const relative = path.relative(root, resolved);
|
|
17
|
+
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
18
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.PATH_MISMATCH, `run owner path escapes run-owners root: ${ownerPath}`);
|
|
19
|
+
}
|
|
20
|
+
return ownerPath;
|
|
21
|
+
}
|
|
22
|
+
function isOwnerRecord(value) {
|
|
23
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
const record = value;
|
|
27
|
+
return (record.schemaVersion === 1 &&
|
|
28
|
+
typeof record.featureId === "string" &&
|
|
29
|
+
record.featureId.length > 0 &&
|
|
30
|
+
typeof record.taskId === "string" &&
|
|
31
|
+
record.taskId.length > 0 &&
|
|
32
|
+
typeof record.workerRunId === "string" &&
|
|
33
|
+
record.workerRunId.length > 0 &&
|
|
34
|
+
typeof record.batchRunId === "string" &&
|
|
35
|
+
record.batchRunId.length > 0 &&
|
|
36
|
+
typeof record.registeredAt === "string" &&
|
|
37
|
+
!Number.isNaN(Date.parse(record.registeredAt)));
|
|
38
|
+
}
|
|
39
|
+
export async function readRunOwner(repoRoot, workerRunId) {
|
|
40
|
+
const ownerPath = getRunOwnerPath(repoRoot, workerRunId);
|
|
41
|
+
try {
|
|
42
|
+
const raw = await readFile(ownerPath, "utf-8");
|
|
43
|
+
const parsed = JSON.parse(raw);
|
|
44
|
+
return isOwnerRecord(parsed) ? parsed : undefined;
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
const code = error.code;
|
|
48
|
+
if (code === "ENOENT")
|
|
49
|
+
return undefined;
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Register an immutable run owner before Task-scoped events / DAG start.
|
|
55
|
+
* Same identity → idempotent reuse. Conflicting identity → fail-closed.
|
|
56
|
+
*/
|
|
57
|
+
export async function registerRunOwner(input) {
|
|
58
|
+
assertWorkerAttemptIdentity(input.identity);
|
|
59
|
+
assertSafeTaskPoolId(input.identity.featureId, "featureId");
|
|
60
|
+
assertSafeTaskPoolId(input.identity.taskId, "taskId");
|
|
61
|
+
assertSafeTaskPoolId(input.identity.workerRunId, "workerRunId");
|
|
62
|
+
const existing = await readRunOwner(input.repoRoot, input.identity.workerRunId);
|
|
63
|
+
if (existing) {
|
|
64
|
+
if (ownerMatchesIdentity(existing, input.identity)) {
|
|
65
|
+
return { status: "idempotent-reuse", owner: existing };
|
|
66
|
+
}
|
|
67
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.CONFLICT, `run owner conflict for ${input.identity.workerRunId}: existing ` +
|
|
68
|
+
`${existing.featureId}/${existing.taskId}/${existing.batchRunId} vs ` +
|
|
69
|
+
`${input.identity.featureId}/${input.identity.taskId}/${input.identity.batchRunId}`);
|
|
70
|
+
}
|
|
71
|
+
const owner = {
|
|
72
|
+
schemaVersion: 1,
|
|
73
|
+
featureId: input.identity.featureId,
|
|
74
|
+
taskId: input.identity.taskId,
|
|
75
|
+
workerRunId: input.identity.workerRunId,
|
|
76
|
+
batchRunId: input.identity.batchRunId,
|
|
77
|
+
registeredAt: (input.now ?? new Date()).toISOString(),
|
|
78
|
+
...(input.controllerAnchor ? { controllerAnchor: input.controllerAnchor } : {}),
|
|
79
|
+
...(input.revisionId ? { revisionId: input.revisionId } : {}),
|
|
80
|
+
};
|
|
81
|
+
await mkdir(getRunOwnersRoot(input.repoRoot), { recursive: true });
|
|
82
|
+
const ownerPath = getRunOwnerPath(input.repoRoot, input.identity.workerRunId);
|
|
83
|
+
await writeJsonAtomic(ownerPath, owner, { repoRoot: input.repoRoot });
|
|
84
|
+
const confirmed = await readRunOwner(input.repoRoot, input.identity.workerRunId);
|
|
85
|
+
if (!confirmed) {
|
|
86
|
+
throw new Error(`failed to persist run owner for ${input.identity.workerRunId}`);
|
|
87
|
+
}
|
|
88
|
+
if (!ownerMatchesIdentity(confirmed, input.identity)) {
|
|
89
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.CONFLICT, `run owner race conflict for ${input.identity.workerRunId}`);
|
|
90
|
+
}
|
|
91
|
+
// If another writer won first with the same identity, treat as idempotent.
|
|
92
|
+
if (confirmed.registeredAt !== owner.registeredAt) {
|
|
93
|
+
return { status: "idempotent-reuse", owner: confirmed };
|
|
94
|
+
}
|
|
95
|
+
return { status: "created", owner: confirmed };
|
|
96
|
+
}
|
|
97
|
+
export async function listRunOwners(repoRoot) {
|
|
98
|
+
const root = getRunOwnersRoot(repoRoot);
|
|
99
|
+
let entries;
|
|
100
|
+
try {
|
|
101
|
+
entries = await readdir(root);
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
if (error.code === "ENOENT")
|
|
105
|
+
return [];
|
|
106
|
+
throw error;
|
|
107
|
+
}
|
|
108
|
+
const owners = [];
|
|
109
|
+
for (const entry of entries) {
|
|
110
|
+
if (!entry.endsWith(".json"))
|
|
111
|
+
continue;
|
|
112
|
+
const workerRunId = entry.slice(0, -".json".length);
|
|
113
|
+
const owner = await readRunOwner(repoRoot, workerRunId);
|
|
114
|
+
if (owner)
|
|
115
|
+
owners.push(owner);
|
|
116
|
+
}
|
|
117
|
+
return owners;
|
|
118
|
+
}
|
|
119
|
+
export function identityFromOwner(owner) {
|
|
120
|
+
return createWorkerAttemptIdentity({
|
|
121
|
+
featureId: owner.featureId,
|
|
122
|
+
taskId: owner.taskId,
|
|
123
|
+
workerRunId: owner.workerRunId,
|
|
124
|
+
batchRunId: owner.batchRunId,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
@@ -2,6 +2,10 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
2
2
|
import { appendFile, mkdir, readFile, readdir, rename, stat, unlink, writeFile, } from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { TASK_POOL_STATE_ERROR_CODES, TaskPoolStateError, } from "./types.js";
|
|
5
|
+
import { reduceAttemptTransition } from "./attempt-transition.js";
|
|
6
|
+
import { createWorkerAttemptIdentity } from "./attempt-identity.js";
|
|
7
|
+
import { stateFromRun as stateFromRunProjection } from "./state-projection.js";
|
|
8
|
+
export { projectTaskPoolStatusFromRun } from "./state-projection.js";
|
|
5
9
|
export const TASK_POOL_RELATIVE_ROOT = ".harness/task-pool";
|
|
6
10
|
const SAFE_TASK_POOL_ID = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
|
|
7
11
|
export function getTaskPoolRoot(repoRoot) {
|
|
@@ -52,7 +56,7 @@ export function getLegacyTaskStatePath(repoRoot, taskId) {
|
|
|
52
56
|
export async function recordTaskPoolRun(input) {
|
|
53
57
|
await ensurePoolDirs(input.repoRoot);
|
|
54
58
|
await appendJsonlFile(getRunsJsonlPath(input.repoRoot), input.run);
|
|
55
|
-
await writeTaskPoolState(input.repoRoot,
|
|
59
|
+
await writeTaskPoolState(input.repoRoot, stateFromRunProjection(input.run));
|
|
56
60
|
await appendJsonlFile(getEventsJsonlPath(input.repoRoot), eventFromRun(input.run));
|
|
57
61
|
}
|
|
58
62
|
export async function appendTaskPoolEvent(repoRoot, event) {
|
|
@@ -74,13 +78,34 @@ export async function prepareTaskPoolRetry(input) {
|
|
|
74
78
|
if (!current) {
|
|
75
79
|
throw new Error(`cannot retry ${input.taskId}: Task Pool state does not exist`);
|
|
76
80
|
}
|
|
81
|
+
if (current.status === "Done") {
|
|
82
|
+
throw new Error(`cannot retry ${input.taskId}: task is Done`);
|
|
83
|
+
}
|
|
84
|
+
if (current.status === "Running") {
|
|
85
|
+
throw new Error(`cannot retry ${input.taskId}: active attempt still Running; reconcile first`);
|
|
86
|
+
}
|
|
77
87
|
if (current.status !== "Failed") {
|
|
78
88
|
throw new Error(`cannot retry ${input.taskId}: expected Failed state, found ${current.status}`);
|
|
79
89
|
}
|
|
80
90
|
if (!current.workerRunId) {
|
|
81
91
|
throw new Error(`cannot retry ${input.taskId}: failed state is missing workerRunId`);
|
|
82
92
|
}
|
|
83
|
-
const
|
|
93
|
+
const identity = createWorkerAttemptIdentity({
|
|
94
|
+
featureId,
|
|
95
|
+
taskId: input.taskId,
|
|
96
|
+
workerRunId: current.workerRunId,
|
|
97
|
+
batchRunId: "retry",
|
|
98
|
+
});
|
|
99
|
+
const transition = reduceAttemptTransition({
|
|
100
|
+
identity,
|
|
101
|
+
from: current,
|
|
102
|
+
event: {
|
|
103
|
+
type: "retry-requested",
|
|
104
|
+
...(input.reason ? { reason: input.reason } : {}),
|
|
105
|
+
now: input.now ?? new Date(),
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
const retryRequestedAt = transition.nextState.retryRequestedAt ?? transition.nextState.updatedAt;
|
|
84
109
|
const result = {
|
|
85
110
|
taskId: input.taskId,
|
|
86
111
|
featureId,
|
|
@@ -89,26 +114,11 @@ export async function prepareTaskPoolRetry(input) {
|
|
|
89
114
|
retryRequestedAt,
|
|
90
115
|
...(input.reason ? { reason: input.reason } : {}),
|
|
91
116
|
};
|
|
92
|
-
await writeTaskPoolState(input.repoRoot,
|
|
93
|
-
schemaVersion: 2,
|
|
94
|
-
featureId,
|
|
95
|
-
taskId: current.taskId,
|
|
96
|
-
status: "Ready",
|
|
97
|
-
updatedAt: retryRequestedAt,
|
|
98
|
-
retryOfWorkerRunId: current.workerRunId,
|
|
99
|
-
retryRequestedAt,
|
|
100
|
-
...(input.reason ? { retryReason: input.reason } : {}),
|
|
101
|
-
});
|
|
117
|
+
await writeTaskPoolState(input.repoRoot, transition.nextState);
|
|
102
118
|
await ensurePoolDirs(input.repoRoot);
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
type: "task-retry-requested",
|
|
107
|
-
taskId: input.taskId,
|
|
108
|
-
featureId,
|
|
109
|
-
previousWorkerRunId: current.workerRunId,
|
|
110
|
-
...(input.reason ? { reason: input.reason } : {}),
|
|
111
|
-
});
|
|
119
|
+
if (transition.auditEvent) {
|
|
120
|
+
await appendJsonlFile(getEventsJsonlPath(input.repoRoot), transition.auditEvent);
|
|
121
|
+
}
|
|
112
122
|
return result;
|
|
113
123
|
}
|
|
114
124
|
async function resolveRetryFeatureId(repoRoot, taskId, featureId) {
|
|
@@ -328,31 +338,7 @@ async function ensurePoolDirs(repoRoot) {
|
|
|
328
338
|
await mkdir(getStatesRoot(repoRoot), { recursive: true });
|
|
329
339
|
}
|
|
330
340
|
export function stateFromRun(run) {
|
|
331
|
-
return
|
|
332
|
-
schemaVersion: 2,
|
|
333
|
-
featureId: run.featureId,
|
|
334
|
-
taskId: run.taskId,
|
|
335
|
-
status: stateStatusFromRun(run),
|
|
336
|
-
updatedAt: run.recordedAt,
|
|
337
|
-
workerRunId: run.workerRunId,
|
|
338
|
-
...(run.runRecordPath ? { lastRunRecordPath: run.runRecordPath } : {}),
|
|
339
|
-
...(run.failure ? { failure: run.failure } : {}),
|
|
340
|
-
...(run.retryOfWorkerRunId
|
|
341
|
-
? { retryOfWorkerRunId: run.retryOfWorkerRunId }
|
|
342
|
-
: {}),
|
|
343
|
-
};
|
|
344
|
-
}
|
|
345
|
-
function stateStatusFromRun(run) {
|
|
346
|
-
if (run.status === "succeeded")
|
|
347
|
-
return "Done";
|
|
348
|
-
// A pre-DAG provider/executor configuration failure is retryable after the
|
|
349
|
-
// operator corrects the environment; do not bury it in a non-retryable Blocked state.
|
|
350
|
-
if (run.status === "run-error" && run.failure?.category === "EnvFailure") {
|
|
351
|
-
return "Failed";
|
|
352
|
-
}
|
|
353
|
-
if (run.status === "run-error")
|
|
354
|
-
return "Blocked";
|
|
355
|
-
return "Failed";
|
|
341
|
+
return stateFromRunProjection(run);
|
|
356
342
|
}
|
|
357
343
|
function eventFromRun(run) {
|
|
358
344
|
return {
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Phase 7 Track A: runtime reconcile inventory facts + dry-run builder.
|
|
3
|
+
* Apply/quarantine orchestration remains intentionally out of scope.
|
|
4
|
+
*/
|
|
5
|
+
export function classifyRuntimeCandidate(signals) {
|
|
6
|
+
const evidence = [];
|
|
7
|
+
if (signals.ownerConflict || signals.taskOwnerMismatch) {
|
|
8
|
+
evidence.push("owner-or-lease conflict detected");
|
|
9
|
+
return {
|
|
10
|
+
liveness: "conflict",
|
|
11
|
+
recommendedAction: "human-review",
|
|
12
|
+
evidence,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
if (signals.lifecycle === "completed" || signals.hasTerminalEvidence) {
|
|
16
|
+
evidence.push("terminal run evidence present");
|
|
17
|
+
return {
|
|
18
|
+
liveness: "healthy",
|
|
19
|
+
recommendedAction: "wait",
|
|
20
|
+
evidence,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
const progress = signals.sessionEventsGrowing === true || signals.providerActive === true;
|
|
24
|
+
if (signals.heartbeatFresh && progress) {
|
|
25
|
+
evidence.push("heartbeat fresh with meaningful progress");
|
|
26
|
+
return {
|
|
27
|
+
liveness: "healthy",
|
|
28
|
+
recommendedAction: signals.lifecycle === "paused" ? "resume" : "wait",
|
|
29
|
+
evidence,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
if (signals.heartbeatFresh && !progress) {
|
|
33
|
+
evidence.push("heartbeat fresh but no recent session/provider progress");
|
|
34
|
+
return {
|
|
35
|
+
liveness: "recoverable",
|
|
36
|
+
recommendedAction: "wait",
|
|
37
|
+
evidence,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
if (signals.processAlive && progress) {
|
|
41
|
+
evidence.push("process alive with progress despite stale heartbeat");
|
|
42
|
+
return {
|
|
43
|
+
liveness: "recoverable",
|
|
44
|
+
recommendedAction: "resume",
|
|
45
|
+
evidence,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
if (signals.lifecycle === "partial" && signals.processAlive !== true) {
|
|
49
|
+
evidence.push("partial lifecycle without live process");
|
|
50
|
+
return {
|
|
51
|
+
liveness: "orphaned",
|
|
52
|
+
recommendedAction: "reconcile",
|
|
53
|
+
evidence,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
evidence.push("no heartbeat or progress signals");
|
|
57
|
+
return {
|
|
58
|
+
liveness: "stale",
|
|
59
|
+
recommendedAction: signals.lifecycle === "active" ? "quarantine" : "abandon",
|
|
60
|
+
evidence,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
export function projectRuntimeRecoveryCommand(candidate) {
|
|
64
|
+
const featureArgs = candidate.featureId && candidate.taskId
|
|
65
|
+
? `--feature-id ${JSON.stringify(candidate.featureId)} --task-id ${JSON.stringify(candidate.taskId)}`
|
|
66
|
+
: undefined;
|
|
67
|
+
switch (candidate.recommendedAction) {
|
|
68
|
+
case "wait":
|
|
69
|
+
return undefined;
|
|
70
|
+
case "resume":
|
|
71
|
+
return candidate.dagRunId
|
|
72
|
+
? `loop-agent dag resume --run-id ${JSON.stringify(candidate.dagRunId)}`
|
|
73
|
+
: "loop-agent dag resume --run-id <dagRunId>";
|
|
74
|
+
case "reconcile":
|
|
75
|
+
return featureArgs
|
|
76
|
+
? `agent-worker task reconcile ${featureArgs}`
|
|
77
|
+
: "agent-worker task reconcile --feature-id <featureId> --task-id <taskId>";
|
|
78
|
+
case "quarantine":
|
|
79
|
+
return featureArgs
|
|
80
|
+
? `agent-worker pool doctor --json ${featureArgs}`
|
|
81
|
+
: "agent-worker pool doctor --json";
|
|
82
|
+
case "abandon":
|
|
83
|
+
return featureArgs
|
|
84
|
+
? `agent-worker task mark-failed ${featureArgs} --reason reconcile-abandon`
|
|
85
|
+
: "agent-worker task mark-failed --feature-id <featureId> --task-id <taskId> --reason reconcile-abandon";
|
|
86
|
+
case "human-review":
|
|
87
|
+
return featureArgs
|
|
88
|
+
? `agent-worker feature review --feature-id ${JSON.stringify(candidate.featureId)}`
|
|
89
|
+
: "agent-worker feature review";
|
|
90
|
+
default:
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Pure dry-run inventory: classify candidates and attach a single recovery command.
|
|
96
|
+
* Does not mutate Task Pool / lease / DAG facts.
|
|
97
|
+
*/
|
|
98
|
+
export function buildRuntimeReconcileInventoryDryRun(input) {
|
|
99
|
+
const candidates = input.items.map((item) => {
|
|
100
|
+
const classified = classifyRuntimeCandidate(item.signals);
|
|
101
|
+
const base = {
|
|
102
|
+
lifecycle: item.signals.lifecycle,
|
|
103
|
+
...classified,
|
|
104
|
+
...(item.workerAttemptId ? { workerAttemptId: item.workerAttemptId } : {}),
|
|
105
|
+
...(item.dagRunId ? { dagRunId: item.dagRunId } : {}),
|
|
106
|
+
...(item.featureId ? { featureId: item.featureId } : {}),
|
|
107
|
+
...(item.taskId ? { taskId: item.taskId } : {}),
|
|
108
|
+
};
|
|
109
|
+
const recoveryCommand = projectRuntimeRecoveryCommand(base);
|
|
110
|
+
return recoveryCommand ? { ...base, recoveryCommand } : base;
|
|
111
|
+
});
|
|
112
|
+
const summary = {
|
|
113
|
+
total: candidates.length,
|
|
114
|
+
healthy: candidates.filter((c) => c.liveness === "healthy").length,
|
|
115
|
+
recoverable: candidates.filter((c) => c.liveness === "recoverable").length,
|
|
116
|
+
stale: candidates.filter((c) => c.liveness === "stale").length,
|
|
117
|
+
orphaned: candidates.filter((c) => c.liveness === "orphaned").length,
|
|
118
|
+
conflict: candidates.filter((c) => c.liveness === "conflict").length,
|
|
119
|
+
};
|
|
120
|
+
return {
|
|
121
|
+
schemaVersion: 1,
|
|
122
|
+
mode: "dry-run",
|
|
123
|
+
generatedAt: (input.now ?? new Date()).toISOString(),
|
|
124
|
+
candidates,
|
|
125
|
+
summary,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { inferRetryability, inferPhaseFromRunStatus } from "./recovery-decision.js";
|
|
2
|
+
/**
|
|
3
|
+
* Project Task Pool status from an immutable terminal run record.
|
|
4
|
+
* Blocked is reserved for contract/risk/dependency/human gates.
|
|
5
|
+
*/
|
|
6
|
+
export function projectTaskPoolStatusFromRun(run) {
|
|
7
|
+
if (run.status === "succeeded")
|
|
8
|
+
return "Done";
|
|
9
|
+
if (run.status === "run-error") {
|
|
10
|
+
const failureV2 = run.failureV2;
|
|
11
|
+
if (failureV2?.retryability === "safe")
|
|
12
|
+
return "Failed";
|
|
13
|
+
if (failureV2?.retryability === "requires-contract-revision")
|
|
14
|
+
return "Blocked";
|
|
15
|
+
const category = failureV2?.category ?? run.failure?.category;
|
|
16
|
+
const phase = failureV2?.phase ?? inferPhaseFromRunStatus(run.status);
|
|
17
|
+
if (category) {
|
|
18
|
+
const retryability = inferRetryability(category, phase, run.status);
|
|
19
|
+
if (retryability === "safe")
|
|
20
|
+
return "Failed";
|
|
21
|
+
if (retryability === "requires-contract-revision")
|
|
22
|
+
return "Blocked";
|
|
23
|
+
if (retryability === "requires-human" ||
|
|
24
|
+
category === "RiskyChange" ||
|
|
25
|
+
category === "NeedsHuman" ||
|
|
26
|
+
category === "DependencyFailure") {
|
|
27
|
+
return "Blocked";
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
// EnvFailure pre-DAG errors remain retryable execution failures.
|
|
31
|
+
if (run.failure?.category === "EnvFailure")
|
|
32
|
+
return "Failed";
|
|
33
|
+
const hasCompleteEvidence = Boolean(run.error || run.failure || run.failureV2 || run.runRecordPath);
|
|
34
|
+
if (hasCompleteEvidence && category === "ProductBug")
|
|
35
|
+
return "Failed";
|
|
36
|
+
if (hasCompleteEvidence && category === "TestBug")
|
|
37
|
+
return "Failed";
|
|
38
|
+
return "Blocked";
|
|
39
|
+
}
|
|
40
|
+
return "Failed";
|
|
41
|
+
}
|
|
42
|
+
export function stateFromRun(run) {
|
|
43
|
+
return {
|
|
44
|
+
schemaVersion: 2,
|
|
45
|
+
featureId: run.featureId,
|
|
46
|
+
taskId: run.taskId,
|
|
47
|
+
status: projectTaskPoolStatusFromRun(run),
|
|
48
|
+
updatedAt: run.recordedAt,
|
|
49
|
+
workerRunId: run.workerRunId,
|
|
50
|
+
...(run.runRecordPath ? { lastRunRecordPath: run.runRecordPath } : {}),
|
|
51
|
+
...(run.failure ? { failure: run.failure } : {}),
|
|
52
|
+
...(run.failureV2 ? { failureV2: run.failureV2 } : {}),
|
|
53
|
+
...(run.retryOfWorkerRunId
|
|
54
|
+
? { retryOfWorkerRunId: run.retryOfWorkerRunId }
|
|
55
|
+
: {}),
|
|
56
|
+
};
|
|
57
|
+
}
|