@tea-agent/loop-agent 0.31.1 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +32 -0
- package/README.md +1 -1
- package/dist/shared/operator/capabilities.js +22 -1
- package/dist/shared/operator/command-lifecycle.js +94 -0
- package/dist/shared/operator/index.js +1 -0
- package/dist/worker/cli.js +10 -24
- package/dist/worker/console/doctor.js +11 -4
- package/dist/worker/console/observe-health-match.js +18 -15
- package/dist/worker/console/operator-actions.js +2 -1
- package/dist/worker/feature/acceptance-policy.js +227 -0
- package/dist/worker/feature/decision-loader.js +1 -1
- package/dist/worker/feature/next-action.js +56 -6
- package/dist/worker/feature/profile-schema.js +3 -0
- package/dist/worker/feature/reducer.js +1 -0
- package/dist/worker/feature/review.js +72 -8
- package/dist/worker/feature/scaffold.js +14 -0
- package/dist/worker/loop-agent/controller-protocol.js +143 -0
- package/dist/worker/materialize/harness-task-lifecycle-probe.js +126 -0
- package/dist/worker/materialize/harness-task-lineage.js +220 -0
- package/dist/worker/materialize/harness-task-materializer.js +350 -80
- package/dist/worker/observability/progress-composite.js +1 -0
- package/dist/worker/observability/read-model.js +66 -19
- package/dist/worker/pool/attempt-identity.js +41 -0
- package/dist/worker/pool/attempt-lease.js +184 -0
- package/dist/worker/pool/attempt-transition.js +210 -0
- package/dist/worker/pool/begin-attempt-with-lease.js +26 -0
- package/dist/worker/pool/begin-attempt.js +35 -0
- package/dist/worker/pool/failure-routing.js +49 -0
- package/dist/worker/pool/recovery-decision.js +163 -0
- package/dist/worker/pool/run-owner-store.js +126 -0
- package/dist/worker/pool/run-store.js +32 -46
- package/dist/worker/pool/runtime-reconcile-inventory.js +127 -0
- package/dist/worker/pool/state-projection.js +57 -0
- package/dist/worker/run-task/run-task.js +31 -4
- package/dist/worker/runner/run-ready.js +64 -14
- package/dist/worker/runner/single-task-attempt.js +42 -13
- package/dist/worker/task-graph/acceptance-schema.js +3 -0
- package/docs/README.md +3 -1
- package/docs/architecture/evolution.md +1 -1
- package/docs/operations/README.md +1 -0
- package/docs/templates/README.md +1 -0
- package/docs/templates/agent-worker-production-readiness-checklist.md +45 -0
- package/docs/templates/evaluation/agents-map-slim-v1.md +1 -1
- package/docs/templates/evaluation/agents-map-verbose-v0.md +1 -1
- package/docs/templates/init-managed-agents.md +1 -1
- package/docs/templates/product-line/scaffold-samples/backend-only/acceptance.yaml +3 -0
- package/docs/templates/product-line/scaffold-samples/backend-only/feature.yaml +11 -0
- package/docs/templates/product-line/scaffold-samples/fe-with-api/acceptance.yaml +3 -0
- package/docs/templates/product-line/scaffold-samples/fe-with-api/feature.yaml +11 -0
- package/docs/templates/product-line/scaffold-samples/frontend-only/acceptance.yaml +3 -0
- package/docs/templates/product-line/scaffold-samples/frontend-only/feature.yaml +11 -0
- package/package.json +1 -1
- package/skills/agent-worker/SKILL.md +1 -1
- package/skills/agent-worker/references/agent-worker-operator.md +1 -1
- package/skills/loop-agent/references/command-reference.md +2 -2
- package/skills/loop-agent/references/harness-policy.md +1 -1
|
@@ -5,6 +5,7 @@ import { truncateUtf8Preview } from "../../shared/preview.js";
|
|
|
5
5
|
import { parseWorkerEventLine } from "./events.js";
|
|
6
6
|
import { readBoundedJsonlTail } from "./event-history.js";
|
|
7
7
|
import { getTaskPoolRoot, listLegacyStateFiles, listTaskPoolStates, } from "../pool/run-store.js";
|
|
8
|
+
import { listRunOwners } from "../pool/run-owner-store.js";
|
|
8
9
|
import { assessDagRunLiveness, assessDagRunRecoveryEligibility, deriveDagRunEffectiveStatus, } from "../../workflows/dag/lifecycle.js";
|
|
9
10
|
import { parseDagSpec, resolveModelForTask, } from "../../workflows/dag/types.js";
|
|
10
11
|
import { loadFeatureDecisionModels } from "../feature/decision-loader.js";
|
|
@@ -51,13 +52,14 @@ export async function buildGlobalSnapshot(options) {
|
|
|
51
52
|
const staleThresholdMs = options.staleThresholdMs ?? 90_000;
|
|
52
53
|
const quietThresholdMs = options.quietThresholdMs ?? 60_000;
|
|
53
54
|
const taskPoolPresent = existsSync(getTaskPoolRoot(repoRoot));
|
|
54
|
-
const [eventsLoad, ledgerLoad, ledgerBatches, statesLoad, handoffs, dagRuns,] = await Promise.all([
|
|
55
|
+
const [eventsLoad, ledgerLoad, ledgerBatches, statesLoad, handoffs, dagRuns, owners,] = await Promise.all([
|
|
55
56
|
loadObservabilityEvents(repoRoot),
|
|
56
57
|
loadLedgerRuns(repoRoot),
|
|
57
58
|
loadLedgerBatches(repoRoot),
|
|
58
59
|
loadStateRecords(repoRoot),
|
|
59
60
|
loadFailureHandoffs(repoRoot),
|
|
60
61
|
loadDagRuns(repoRoot, now()),
|
|
62
|
+
listRunOwners(repoRoot).catch(() => []),
|
|
61
63
|
]);
|
|
62
64
|
const events = eventsLoad.events;
|
|
63
65
|
const ledgerRuns = ledgerLoad.runs;
|
|
@@ -74,7 +76,7 @@ export async function buildGlobalSnapshot(options) {
|
|
|
74
76
|
for (const state of statesLoad.legacyCandidates) {
|
|
75
77
|
mergeLegacyStateRecord(taskMap, state, identityWarnings);
|
|
76
78
|
}
|
|
77
|
-
const workerRunFeatureIndex = buildWorkerRunFeatureIndex(ledgerRuns, taskMap);
|
|
79
|
+
const workerRunFeatureIndex = buildWorkerRunFeatureIndex(ledgerRuns, taskMap, owners);
|
|
78
80
|
mergeEvents(taskMap, events, workerRunFeatureIndex, identityWarnings);
|
|
79
81
|
mergeHandoffs(taskMap, handoffs);
|
|
80
82
|
applyStaleDetection(taskMap, now(), staleThresholdMs, quietThresholdMs);
|
|
@@ -274,6 +276,15 @@ function toPublicTaskSummary(task) {
|
|
|
274
276
|
const { timeoutMs: _to, commandStartedAt: _cs, ...rest } = task;
|
|
275
277
|
return rest;
|
|
276
278
|
}
|
|
279
|
+
function isLivePoolStatus(status) {
|
|
280
|
+
return (status === "Running" ||
|
|
281
|
+
status === "Queued" ||
|
|
282
|
+
status === "AgentCompleted" ||
|
|
283
|
+
status === "VerificationRunning" ||
|
|
284
|
+
status === "HumanReview" ||
|
|
285
|
+
status === "Ready" ||
|
|
286
|
+
status === "Draft");
|
|
287
|
+
}
|
|
277
288
|
function mergeStateRecord(taskMap, state, warnings) {
|
|
278
289
|
const featureId = state.featureId?.trim();
|
|
279
290
|
if (!featureId) {
|
|
@@ -286,13 +297,19 @@ function mergeStateRecord(taskMap, state, warnings) {
|
|
|
286
297
|
featureId,
|
|
287
298
|
status: "unknown",
|
|
288
299
|
};
|
|
300
|
+
const live = isLivePoolStatus(state.status);
|
|
301
|
+
const mapped = mapStateStatus(state.status);
|
|
289
302
|
taskMap.set(key, {
|
|
290
303
|
...existing,
|
|
291
304
|
featureId,
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
305
|
+
// Live Task Pool state is the current attempt truth; do not keep a stale
|
|
306
|
+
// terminal ledger workerRunId/status over an active Running attempt.
|
|
307
|
+
status: live
|
|
308
|
+
? mapped
|
|
309
|
+
: existing.status !== "unknown"
|
|
310
|
+
? existing.status
|
|
311
|
+
: mapped,
|
|
312
|
+
workerRunId: state.workerRunId ?? existing.workerRunId,
|
|
296
313
|
updatedAt: state.updatedAt ?? existing.updatedAt,
|
|
297
314
|
artifactRefs: mergeArtifactRefs(existing.artifactRefs, {
|
|
298
315
|
runRecordPath: state.lastRunRecordPath,
|
|
@@ -362,10 +379,10 @@ function mergeLedgerRun(taskMap, run, warnings) {
|
|
|
362
379
|
});
|
|
363
380
|
}
|
|
364
381
|
/**
|
|
365
|
-
* Build workerRunId → featureId unique index from ledger + already-owned task rows.
|
|
382
|
+
* Build workerRunId → featureId unique index from ledger + already-owned task rows + run owners.
|
|
366
383
|
* Ambiguous (0 or N) mappings are omitted so callers treat them as unowned.
|
|
367
384
|
*/
|
|
368
|
-
function buildWorkerRunFeatureIndex(ledgerRuns, taskMap) {
|
|
385
|
+
function buildWorkerRunFeatureIndex(ledgerRuns, taskMap, owners) {
|
|
369
386
|
const candidates = new Map();
|
|
370
387
|
const add = (workerRunId, featureId) => {
|
|
371
388
|
const wr = workerRunId?.trim();
|
|
@@ -387,6 +404,11 @@ function buildWorkerRunFeatureIndex(ledgerRuns, taskMap) {
|
|
|
387
404
|
add(task.workerRunId, task.featureId);
|
|
388
405
|
}
|
|
389
406
|
}
|
|
407
|
+
if (owners) {
|
|
408
|
+
for (const owner of owners) {
|
|
409
|
+
add(owner.workerRunId, owner.featureId);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
390
412
|
const unique = new Map();
|
|
391
413
|
for (const [workerRunId, features] of candidates) {
|
|
392
414
|
if (features.size === 1) {
|
|
@@ -397,15 +419,16 @@ function buildWorkerRunFeatureIndex(ledgerRuns, taskMap) {
|
|
|
397
419
|
}
|
|
398
420
|
function mergeEvents(taskMap, events, workerRunFeatureIndex, warnings) {
|
|
399
421
|
const sorted = [...events].sort((a, b) => a.at.localeCompare(b.at));
|
|
422
|
+
const unownedKeys = new Set();
|
|
400
423
|
for (const event of sorted) {
|
|
401
424
|
if (event.taskId) {
|
|
402
|
-
applyTaskEvent(taskMap, event, workerRunFeatureIndex, warnings);
|
|
425
|
+
applyTaskEvent(taskMap, event, workerRunFeatureIndex, warnings, unownedKeys);
|
|
403
426
|
}
|
|
404
427
|
}
|
|
405
428
|
}
|
|
406
|
-
function applyTaskEvent(taskMap, event, workerRunFeatureIndex, warnings) {
|
|
429
|
+
function applyTaskEvent(taskMap, event, workerRunFeatureIndex, warnings, unownedKeys) {
|
|
407
430
|
const taskId = event.taskId;
|
|
408
|
-
const featureId = resolveEventFeatureId(event, taskMap, workerRunFeatureIndex, warnings);
|
|
431
|
+
const featureId = resolveEventFeatureId(event, taskMap, workerRunFeatureIndex, warnings, unownedKeys);
|
|
409
432
|
if (!featureId)
|
|
410
433
|
return;
|
|
411
434
|
const key = taskIdentityKey(featureId, taskId);
|
|
@@ -416,28 +439,46 @@ function applyTaskEvent(taskMap, event, workerRunFeatureIndex, warnings) {
|
|
|
416
439
|
};
|
|
417
440
|
taskMap.set(key, applyEventToSummary(existing, event));
|
|
418
441
|
}
|
|
419
|
-
function resolveEventFeatureId(event, taskMap, workerRunFeatureIndex, warnings) {
|
|
442
|
+
function resolveEventFeatureId(event, taskMap, workerRunFeatureIndex, warnings, unownedKeys) {
|
|
420
443
|
const taskId = event.taskId;
|
|
421
444
|
if (!taskId)
|
|
422
445
|
return undefined;
|
|
446
|
+
const eventFeatureId = event.featureId?.trim();
|
|
423
447
|
if (event.workerRunId) {
|
|
424
448
|
const fromIndex = workerRunFeatureIndex.get(event.workerRunId);
|
|
449
|
+
if (eventFeatureId && fromIndex && eventFeatureId !== fromIndex) {
|
|
450
|
+
pushOwnershipWarning(warnings, unownedKeys, `ownership conflict for workerRunId ${event.workerRunId}: event featureId ${eventFeatureId} vs index ${fromIndex}`, event.workerRunId);
|
|
451
|
+
return undefined;
|
|
452
|
+
}
|
|
453
|
+
if (eventFeatureId)
|
|
454
|
+
return eventFeatureId;
|
|
425
455
|
if (fromIndex)
|
|
426
456
|
return fromIndex;
|
|
427
|
-
// Fall through: maybe a single existing composite row already owns this workerRunId.
|
|
428
457
|
const byWorkerRun = [...taskMap.values()].filter((task) => task.workerRunId === event.workerRunId && Boolean(task.featureId));
|
|
429
458
|
if (byWorkerRun.length === 1)
|
|
430
459
|
return byWorkerRun[0].featureId;
|
|
431
460
|
}
|
|
432
|
-
|
|
461
|
+
else if (eventFeatureId) {
|
|
462
|
+
return eventFeatureId;
|
|
463
|
+
}
|
|
433
464
|
const byTaskId = [...taskMap.values()].filter((task) => task.taskId === taskId && Boolean(task.featureId));
|
|
434
465
|
if (byTaskId.length === 1)
|
|
435
466
|
return byTaskId[0].featureId;
|
|
436
|
-
warnings
|
|
467
|
+
pushOwnershipWarning(warnings, unownedKeys, `unowned event ${event.type} for task ${taskId}` +
|
|
437
468
|
(event.workerRunId ? ` workerRunId ${event.workerRunId}` : "") +
|
|
438
|
-
` cannot uniquely resolve featureId; excluded from task projection`);
|
|
469
|
+
` cannot uniquely resolve featureId; excluded from task projection`, event.workerRunId ?? `${taskId}:${event.type}`);
|
|
439
470
|
return undefined;
|
|
440
471
|
}
|
|
472
|
+
function pushOwnershipWarning(warnings, unownedKeys, message, key) {
|
|
473
|
+
if (!warnings)
|
|
474
|
+
return;
|
|
475
|
+
if (unownedKeys) {
|
|
476
|
+
if (unownedKeys.has(key))
|
|
477
|
+
return;
|
|
478
|
+
unownedKeys.add(key);
|
|
479
|
+
}
|
|
480
|
+
warnings.push(message);
|
|
481
|
+
}
|
|
441
482
|
function applyEventToSummary(existing, event) {
|
|
442
483
|
const updated = {
|
|
443
484
|
...existing,
|
|
@@ -489,6 +530,7 @@ function applyEventToSummary(existing, event) {
|
|
|
489
530
|
}
|
|
490
531
|
function buildHistoricalRunMap(ledgerRuns, events, workerRunFeatureIndex = new Map(), warnings) {
|
|
491
532
|
const runMap = new Map();
|
|
533
|
+
const unownedKeys = new Set();
|
|
492
534
|
for (const run of ledgerRuns) {
|
|
493
535
|
// History map is keyed by workerRunId (run-scoped); featureId still required when present.
|
|
494
536
|
runMap.set(run.workerRunId, {
|
|
@@ -515,13 +557,18 @@ function buildHistoricalRunMap(ledgerRuns, events, workerRunFeatureIndex = new M
|
|
|
515
557
|
continue;
|
|
516
558
|
const existing = runMap.get(event.workerRunId);
|
|
517
559
|
if (existing) {
|
|
560
|
+
if (event.featureId?.trim() &&
|
|
561
|
+
existing.featureId &&
|
|
562
|
+
event.featureId.trim() !== existing.featureId) {
|
|
563
|
+
pushOwnershipWarning(warnings, unownedKeys, `ownership conflict for workerRunId ${event.workerRunId}: event featureId ${event.featureId} vs historical ${existing.featureId}`, event.workerRunId);
|
|
564
|
+
continue;
|
|
565
|
+
}
|
|
518
566
|
runMap.set(event.workerRunId, applyEventToSummary(existing, event));
|
|
519
567
|
continue;
|
|
520
568
|
}
|
|
521
|
-
|
|
522
|
-
const featureId = workerRunFeatureIndex.get(event.workerRunId);
|
|
569
|
+
const featureId = event.featureId?.trim() || workerRunFeatureIndex.get(event.workerRunId);
|
|
523
570
|
if (!featureId) {
|
|
524
|
-
warnings
|
|
571
|
+
pushOwnershipWarning(warnings, unownedKeys, `unowned historical event ${event.type} for task ${event.taskId} workerRunId ${event.workerRunId} cannot uniquely resolve featureId; excluded from historical run map`, event.workerRunId);
|
|
525
572
|
continue;
|
|
526
573
|
}
|
|
527
574
|
runMap.set(event.workerRunId, applyEventToSummary({
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical Worker Attempt identity (Phase 1).
|
|
3
|
+
* All Task-scoped ownership / events / begin transactions must carry this shape.
|
|
4
|
+
*/
|
|
5
|
+
export function assertWorkerAttemptIdentity(value, label = "WorkerAttemptIdentity") {
|
|
6
|
+
if (!value || typeof value !== "object") {
|
|
7
|
+
throw new Error(`${label} is required`);
|
|
8
|
+
}
|
|
9
|
+
for (const key of ["featureId", "taskId", "workerRunId", "batchRunId"]) {
|
|
10
|
+
const field = value[key];
|
|
11
|
+
if (typeof field !== "string" || field.trim().length === 0) {
|
|
12
|
+
throw new Error(`${label}.${key} is required`);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
if (value.schemaVersion !== 1 && value.schemaVersion !== undefined) {
|
|
16
|
+
throw new Error(`${label}.schemaVersion must be 1`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function createWorkerAttemptIdentity(input) {
|
|
20
|
+
const identity = {
|
|
21
|
+
schemaVersion: 1,
|
|
22
|
+
featureId: input.featureId.trim(),
|
|
23
|
+
taskId: input.taskId.trim(),
|
|
24
|
+
workerRunId: input.workerRunId.trim(),
|
|
25
|
+
batchRunId: input.batchRunId.trim(),
|
|
26
|
+
};
|
|
27
|
+
assertWorkerAttemptIdentity(identity);
|
|
28
|
+
return identity;
|
|
29
|
+
}
|
|
30
|
+
export function identitiesEqual(a, b) {
|
|
31
|
+
return (a.featureId === b.featureId &&
|
|
32
|
+
a.taskId === b.taskId &&
|
|
33
|
+
a.workerRunId === b.workerRunId &&
|
|
34
|
+
a.batchRunId === b.batchRunId);
|
|
35
|
+
}
|
|
36
|
+
export function ownerMatchesIdentity(owner, identity) {
|
|
37
|
+
return (owner.featureId === identity.featureId &&
|
|
38
|
+
owner.taskId === identity.taskId &&
|
|
39
|
+
owner.workerRunId === identity.workerRunId &&
|
|
40
|
+
owner.batchRunId === identity.batchRunId);
|
|
41
|
+
}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, readdir, unlink } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
5
|
+
import { assertSafeTaskPoolId, getTaskPoolRoot, } from "./run-store.js";
|
|
6
|
+
import { TASK_POOL_STATE_ERROR_CODES, TaskPoolStateError } from "./types.js";
|
|
7
|
+
export function getAttemptLeasesRoot(repoRoot) {
|
|
8
|
+
return path.join(getTaskPoolRoot(repoRoot), "leases");
|
|
9
|
+
}
|
|
10
|
+
export function getAttemptLeasePath(repoRoot, featureId, taskId) {
|
|
11
|
+
assertSafeTaskPoolId(featureId, "featureId");
|
|
12
|
+
assertSafeTaskPoolId(taskId, "taskId");
|
|
13
|
+
const leasePath = path.join(getAttemptLeasesRoot(repoRoot), featureId, `${taskId}.json`);
|
|
14
|
+
const root = path.resolve(getAttemptLeasesRoot(repoRoot));
|
|
15
|
+
const resolved = path.resolve(leasePath);
|
|
16
|
+
const relative = path.relative(root, resolved);
|
|
17
|
+
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
18
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.PATH_MISMATCH, `attempt lease path escapes leases root: ${leasePath}`);
|
|
19
|
+
}
|
|
20
|
+
return leasePath;
|
|
21
|
+
}
|
|
22
|
+
function isLeaseRecord(value) {
|
|
23
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
24
|
+
return false;
|
|
25
|
+
const record = value;
|
|
26
|
+
return (record.schemaVersion === 1 &&
|
|
27
|
+
typeof record.featureId === "string" &&
|
|
28
|
+
typeof record.taskId === "string" &&
|
|
29
|
+
typeof record.workerRunId === "string" &&
|
|
30
|
+
typeof record.batchRunId === "string" &&
|
|
31
|
+
typeof record.token === "string" &&
|
|
32
|
+
typeof record.acquiredAt === "string" &&
|
|
33
|
+
typeof record.renewedAt === "string");
|
|
34
|
+
}
|
|
35
|
+
export async function readAttemptLease(repoRoot, featureId, taskId) {
|
|
36
|
+
const leasePath = getAttemptLeasePath(repoRoot, featureId, taskId);
|
|
37
|
+
try {
|
|
38
|
+
const raw = await readFile(leasePath, "utf-8");
|
|
39
|
+
const parsed = JSON.parse(raw);
|
|
40
|
+
return isLeaseRecord(parsed) ? parsed : undefined;
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
if (error.code === "ENOENT")
|
|
44
|
+
return undefined;
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Acquire Task-scoped attempt lease before beginWorkerAttempt.
|
|
50
|
+
* Same identity+token reuse is idempotent; different active identity → conflict.
|
|
51
|
+
*/
|
|
52
|
+
export async function acquireAttemptLease(input) {
|
|
53
|
+
const now = input.now ?? new Date();
|
|
54
|
+
const existing = await readAttemptLease(input.repoRoot, input.featureId, input.taskId);
|
|
55
|
+
if (existing) {
|
|
56
|
+
if (existing.workerRunId === input.workerRunId &&
|
|
57
|
+
existing.batchRunId === input.batchRunId &&
|
|
58
|
+
existing.featureId === input.featureId &&
|
|
59
|
+
existing.taskId === input.taskId) {
|
|
60
|
+
return { status: "idempotent-reuse", lease: existing };
|
|
61
|
+
}
|
|
62
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.CONFLICT, `attempt-lease-conflict for ${input.featureId}/${input.taskId}: held by ${existing.workerRunId}`);
|
|
63
|
+
}
|
|
64
|
+
const lease = {
|
|
65
|
+
schemaVersion: 1,
|
|
66
|
+
featureId: input.featureId,
|
|
67
|
+
taskId: input.taskId,
|
|
68
|
+
workerRunId: input.workerRunId,
|
|
69
|
+
batchRunId: input.batchRunId,
|
|
70
|
+
acquiredAt: now.toISOString(),
|
|
71
|
+
renewedAt: now.toISOString(),
|
|
72
|
+
token: `atk-${randomBytes(8).toString("hex")}`,
|
|
73
|
+
ownerProcess: { pid: process.pid },
|
|
74
|
+
...(input.revisionId ? { revisionId: input.revisionId } : {}),
|
|
75
|
+
...(input.controllerAnchor
|
|
76
|
+
? { controllerAnchor: input.controllerAnchor }
|
|
77
|
+
: {}),
|
|
78
|
+
};
|
|
79
|
+
await mkdir(path.dirname(getAttemptLeasePath(input.repoRoot, input.featureId, input.taskId)), {
|
|
80
|
+
recursive: true,
|
|
81
|
+
});
|
|
82
|
+
await writeJsonAtomic(getAttemptLeasePath(input.repoRoot, input.featureId, input.taskId), lease, { repoRoot: input.repoRoot });
|
|
83
|
+
const confirmed = await readAttemptLease(input.repoRoot, input.featureId, input.taskId);
|
|
84
|
+
if (!confirmed) {
|
|
85
|
+
throw new Error(`failed to persist attempt lease for ${input.featureId}/${input.taskId}`);
|
|
86
|
+
}
|
|
87
|
+
if (confirmed.workerRunId !== input.workerRunId) {
|
|
88
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.CONFLICT, `attempt-lease-conflict race for ${input.featureId}/${input.taskId}`);
|
|
89
|
+
}
|
|
90
|
+
return { status: "acquired", lease: confirmed };
|
|
91
|
+
}
|
|
92
|
+
export async function renewAttemptLease(input) {
|
|
93
|
+
const existing = await readAttemptLease(input.repoRoot, input.featureId, input.taskId);
|
|
94
|
+
if (!existing) {
|
|
95
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.CONFLICT, `cannot renew missing lease for ${input.featureId}/${input.taskId}`);
|
|
96
|
+
}
|
|
97
|
+
if (existing.token !== input.token || existing.workerRunId !== input.workerRunId) {
|
|
98
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.CONFLICT, `cannot renew lease: CAS token/identity mismatch for ${input.featureId}/${input.taskId}`);
|
|
99
|
+
}
|
|
100
|
+
const renewed = {
|
|
101
|
+
...existing,
|
|
102
|
+
renewedAt: (input.now ?? new Date()).toISOString(),
|
|
103
|
+
};
|
|
104
|
+
await writeJsonAtomic(getAttemptLeasePath(input.repoRoot, input.featureId, input.taskId), renewed, { repoRoot: input.repoRoot });
|
|
105
|
+
return renewed;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Release active lease after terminal/recovery evidence. Moves to lease-history then deletes active locator.
|
|
109
|
+
*/
|
|
110
|
+
export async function releaseAttemptLease(input) {
|
|
111
|
+
const existing = await readAttemptLease(input.repoRoot, input.featureId, input.taskId);
|
|
112
|
+
if (!existing)
|
|
113
|
+
return;
|
|
114
|
+
if (existing.token !== input.token || existing.workerRunId !== input.workerRunId) {
|
|
115
|
+
throw new TaskPoolStateError(TASK_POOL_STATE_ERROR_CODES.CONFLICT, `cannot release lease: CAS token/identity mismatch for ${input.featureId}/${input.taskId}`);
|
|
116
|
+
}
|
|
117
|
+
const historyRoot = path.join(getAttemptLeasesRoot(input.repoRoot), "_history");
|
|
118
|
+
await mkdir(historyRoot, { recursive: true });
|
|
119
|
+
const historyPath = path.join(historyRoot, `${input.featureId}__${input.taskId}__${existing.workerRunId}.json`);
|
|
120
|
+
const activePath = getAttemptLeasePath(input.repoRoot, input.featureId, input.taskId);
|
|
121
|
+
await writeJsonAtomic(historyPath, {
|
|
122
|
+
...existing,
|
|
123
|
+
releasedAt: (input.now ?? new Date()).toISOString(),
|
|
124
|
+
}, { repoRoot: input.repoRoot });
|
|
125
|
+
await unlink(activePath).catch(() => { });
|
|
126
|
+
}
|
|
127
|
+
export function decideAttemptLeaseReclaim(input) {
|
|
128
|
+
if (input.hasTerminalEvidence) {
|
|
129
|
+
return {
|
|
130
|
+
status: "reclaim-allowed",
|
|
131
|
+
reason: "terminal evidence present; active lease should have been released",
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
if (input.liveness === "healthy" || input.liveness === "recoverable") {
|
|
135
|
+
return {
|
|
136
|
+
status: "keep",
|
|
137
|
+
reason: `lease held by ${input.liveness} attempt; do not steal`,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
if (input.liveness === "conflict") {
|
|
141
|
+
return {
|
|
142
|
+
status: "keep",
|
|
143
|
+
reason: "identity conflict requires human-review before reclaim",
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
return {
|
|
147
|
+
status: "reclaim-allowed",
|
|
148
|
+
reason: `lease reclaim allowed after ${input.liveness} reconcile audit`,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
export async function listAttemptLeases(repoRoot) {
|
|
152
|
+
const root = getAttemptLeasesRoot(repoRoot);
|
|
153
|
+
let featureDirs;
|
|
154
|
+
try {
|
|
155
|
+
featureDirs = await readdir(root);
|
|
156
|
+
}
|
|
157
|
+
catch (error) {
|
|
158
|
+
if (error.code === "ENOENT")
|
|
159
|
+
return [];
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
const leases = [];
|
|
163
|
+
for (const featureId of featureDirs) {
|
|
164
|
+
if (featureId.startsWith("_"))
|
|
165
|
+
continue;
|
|
166
|
+
const featureRoot = path.join(root, featureId);
|
|
167
|
+
let entries;
|
|
168
|
+
try {
|
|
169
|
+
entries = await readdir(featureRoot);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
for (const entry of entries) {
|
|
175
|
+
if (!entry.endsWith(".json"))
|
|
176
|
+
continue;
|
|
177
|
+
const taskId = entry.slice(0, -".json".length);
|
|
178
|
+
const lease = await readAttemptLease(repoRoot, featureId, taskId);
|
|
179
|
+
if (lease)
|
|
180
|
+
leases.push(lease);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return leases;
|
|
184
|
+
}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { isSafeRetryDecision, resolveWorkerRecoveryDecision, } from "./recovery-decision.js";
|
|
2
|
+
/**
|
|
3
|
+
* Central Attempt transition reducer. Callers must not invent parallel state writers.
|
|
4
|
+
*/
|
|
5
|
+
export function reduceAttemptTransition(input) {
|
|
6
|
+
const { identity, from, event } = input;
|
|
7
|
+
const nowIso = new Date().toISOString();
|
|
8
|
+
switch (event.type) {
|
|
9
|
+
case "attempt-registered": {
|
|
10
|
+
return {
|
|
11
|
+
nextState: {
|
|
12
|
+
schemaVersion: 2,
|
|
13
|
+
featureId: identity.featureId,
|
|
14
|
+
taskId: identity.taskId,
|
|
15
|
+
status: "Running",
|
|
16
|
+
updatedAt: nowIso,
|
|
17
|
+
workerRunId: identity.workerRunId,
|
|
18
|
+
},
|
|
19
|
+
recovery: { recommendedAction: "none", reason: "attempt registered" },
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
case "heartbeat": {
|
|
23
|
+
if (!from) {
|
|
24
|
+
throw new Error("heartbeat requires existing attempt state");
|
|
25
|
+
}
|
|
26
|
+
return {
|
|
27
|
+
nextState: { ...from, updatedAt: nowIso },
|
|
28
|
+
recovery: resolveWorkerRecoveryDecision(input.facts ?? { state: from }),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
case "finalize-success": {
|
|
32
|
+
return {
|
|
33
|
+
nextState: {
|
|
34
|
+
schemaVersion: 2,
|
|
35
|
+
featureId: identity.featureId,
|
|
36
|
+
taskId: identity.taskId,
|
|
37
|
+
status: "Done",
|
|
38
|
+
updatedAt: event.terminalRun.recordedAt,
|
|
39
|
+
workerRunId: identity.workerRunId,
|
|
40
|
+
...(event.terminalRun.runRecordPath
|
|
41
|
+
? { lastRunRecordPath: event.terminalRun.runRecordPath }
|
|
42
|
+
: {}),
|
|
43
|
+
},
|
|
44
|
+
terminalRun: event.terminalRun,
|
|
45
|
+
recovery: { recommendedAction: "none", reason: "terminal success" },
|
|
46
|
+
auditEvent: {
|
|
47
|
+
schemaVersion: 1,
|
|
48
|
+
at: event.terminalRun.recordedAt,
|
|
49
|
+
type: "task-run-recorded",
|
|
50
|
+
batchRunId: event.terminalRun.batchRunId,
|
|
51
|
+
workerRunId: identity.workerRunId,
|
|
52
|
+
taskId: identity.taskId,
|
|
53
|
+
featureId: identity.featureId,
|
|
54
|
+
status: "succeeded",
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
case "finalize-failure": {
|
|
59
|
+
const failureV2 = event.failureV2 ??
|
|
60
|
+
event.terminalRun.failureV2 ??
|
|
61
|
+
undefined;
|
|
62
|
+
const recovery = resolveWorkerRecoveryDecision({
|
|
63
|
+
state: {
|
|
64
|
+
status: "Failed",
|
|
65
|
+
workerRunId: identity.workerRunId,
|
|
66
|
+
failure: event.terminalRun.failure,
|
|
67
|
+
failureV2,
|
|
68
|
+
},
|
|
69
|
+
terminalRun: event.terminalRun,
|
|
70
|
+
...(input.facts ?? {}),
|
|
71
|
+
});
|
|
72
|
+
const nextStatus = failureV2?.recommendedAction === "revise-contract" ||
|
|
73
|
+
failureV2?.retryability === "requires-contract-revision" ||
|
|
74
|
+
failureV2?.retryability === "requires-human"
|
|
75
|
+
? failureV2.retryability === "requires-contract-revision"
|
|
76
|
+
? "Blocked"
|
|
77
|
+
: "Failed"
|
|
78
|
+
: "Failed";
|
|
79
|
+
// Safe terminal run-error / failed → Failed so retry does not need mark-failed.
|
|
80
|
+
const status = event.terminalRun.status === "run-error" &&
|
|
81
|
+
failureV2?.retryability === "safe"
|
|
82
|
+
? "Failed"
|
|
83
|
+
: event.terminalRun.status === "run-error" &&
|
|
84
|
+
!failureV2 &&
|
|
85
|
+
event.terminalRun.failure?.category === "EnvFailure"
|
|
86
|
+
? "Failed"
|
|
87
|
+
: event.terminalRun.status === "run-error" && !failureV2
|
|
88
|
+
? "Blocked"
|
|
89
|
+
: nextStatus === "Blocked"
|
|
90
|
+
? "Blocked"
|
|
91
|
+
: "Failed";
|
|
92
|
+
return {
|
|
93
|
+
nextState: {
|
|
94
|
+
schemaVersion: 2,
|
|
95
|
+
featureId: identity.featureId,
|
|
96
|
+
taskId: identity.taskId,
|
|
97
|
+
status,
|
|
98
|
+
updatedAt: event.terminalRun.recordedAt,
|
|
99
|
+
workerRunId: identity.workerRunId,
|
|
100
|
+
...(event.terminalRun.runRecordPath
|
|
101
|
+
? { lastRunRecordPath: event.terminalRun.runRecordPath }
|
|
102
|
+
: {}),
|
|
103
|
+
...(event.terminalRun.failure
|
|
104
|
+
? { failure: event.terminalRun.failure }
|
|
105
|
+
: {}),
|
|
106
|
+
...(failureV2 ? { failureV2 } : {}),
|
|
107
|
+
},
|
|
108
|
+
terminalRun: event.terminalRun,
|
|
109
|
+
recovery,
|
|
110
|
+
auditEvent: {
|
|
111
|
+
schemaVersion: 1,
|
|
112
|
+
at: event.terminalRun.recordedAt,
|
|
113
|
+
type: "task-run-recorded",
|
|
114
|
+
batchRunId: event.terminalRun.batchRunId,
|
|
115
|
+
workerRunId: identity.workerRunId,
|
|
116
|
+
taskId: identity.taskId,
|
|
117
|
+
featureId: identity.featureId,
|
|
118
|
+
status: event.terminalRun.status,
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
case "retry-requested": {
|
|
123
|
+
if (!from) {
|
|
124
|
+
throw new Error("retry requires existing attempt state");
|
|
125
|
+
}
|
|
126
|
+
if (from.status === "Done") {
|
|
127
|
+
throw new Error(`cannot retry ${identity.taskId}: Done attempts are immutable`);
|
|
128
|
+
}
|
|
129
|
+
const decision = resolveWorkerRecoveryDecision({
|
|
130
|
+
state: from,
|
|
131
|
+
...(input.facts ?? {}),
|
|
132
|
+
});
|
|
133
|
+
if (!isSafeRetryDecision(decision) && from.status !== "Failed") {
|
|
134
|
+
throw new Error(`cannot retry ${identity.taskId}: recovery action is ${decision.recommendedAction} (${decision.reason})`);
|
|
135
|
+
}
|
|
136
|
+
if (from.status !== "Failed") {
|
|
137
|
+
throw new Error(`cannot retry ${identity.taskId}: expected Failed state, found ${from.status}`);
|
|
138
|
+
}
|
|
139
|
+
if (!from.workerRunId) {
|
|
140
|
+
throw new Error(`cannot retry ${identity.taskId}: failed state is missing workerRunId`);
|
|
141
|
+
}
|
|
142
|
+
const retryRequestedAt = event.now.toISOString();
|
|
143
|
+
return {
|
|
144
|
+
nextState: {
|
|
145
|
+
schemaVersion: 2,
|
|
146
|
+
featureId: identity.featureId,
|
|
147
|
+
taskId: identity.taskId,
|
|
148
|
+
status: "Ready",
|
|
149
|
+
updatedAt: retryRequestedAt,
|
|
150
|
+
workerRunId: from.workerRunId,
|
|
151
|
+
retryOfWorkerRunId: from.workerRunId,
|
|
152
|
+
retryRequestedAt,
|
|
153
|
+
...(event.reason ? { retryReason: event.reason } : {}),
|
|
154
|
+
...(from.lastRunRecordPath
|
|
155
|
+
? { lastRunRecordPath: from.lastRunRecordPath }
|
|
156
|
+
: {}),
|
|
157
|
+
...(from.failure ? { failure: from.failure } : {}),
|
|
158
|
+
...(from.failureV2 ? { failureV2: from.failureV2 } : {}),
|
|
159
|
+
},
|
|
160
|
+
recovery: {
|
|
161
|
+
recommendedAction: "retry",
|
|
162
|
+
reason: event.reason ?? "operator requested retry",
|
|
163
|
+
retryability: "safe",
|
|
164
|
+
},
|
|
165
|
+
auditEvent: {
|
|
166
|
+
schemaVersion: 1,
|
|
167
|
+
at: retryRequestedAt,
|
|
168
|
+
type: "task-retry-requested",
|
|
169
|
+
taskId: identity.taskId,
|
|
170
|
+
featureId: identity.featureId,
|
|
171
|
+
previousWorkerRunId: from.workerRunId,
|
|
172
|
+
...(event.reason ? { reason: event.reason } : {}),
|
|
173
|
+
},
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
case "mark-failed": {
|
|
177
|
+
if (!from) {
|
|
178
|
+
throw new Error("mark-failed requires existing attempt state");
|
|
179
|
+
}
|
|
180
|
+
if (from.status === "Done") {
|
|
181
|
+
throw new Error(`cannot mark-failed ${identity.taskId}: Done is immutable`);
|
|
182
|
+
}
|
|
183
|
+
const at = event.now.toISOString();
|
|
184
|
+
return {
|
|
185
|
+
nextState: {
|
|
186
|
+
...from,
|
|
187
|
+
schemaVersion: 2,
|
|
188
|
+
featureId: identity.featureId,
|
|
189
|
+
status: "Failed",
|
|
190
|
+
updatedAt: at,
|
|
191
|
+
operatorRecovery: {
|
|
192
|
+
action: "mark-failed",
|
|
193
|
+
reason: event.reason,
|
|
194
|
+
workerRunId: from.workerRunId,
|
|
195
|
+
at,
|
|
196
|
+
},
|
|
197
|
+
},
|
|
198
|
+
recovery: {
|
|
199
|
+
recommendedAction: "retry",
|
|
200
|
+
reason: "emergency mark-failed projection",
|
|
201
|
+
retryability: "safe",
|
|
202
|
+
},
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
default: {
|
|
206
|
+
const _exhaustive = event;
|
|
207
|
+
throw new Error(`unsupported attempt lifecycle event: ${JSON.stringify(_exhaustive)}`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { assertCompatibleBeforeWrite, evaluateControllerCompatibility, } from "../loop-agent/controller-protocol.js";
|
|
2
|
+
import { acquireAttemptLease, } from "./attempt-lease.js";
|
|
3
|
+
import { beginWorkerAttempt, } from "./begin-attempt.js";
|
|
4
|
+
/**
|
|
5
|
+
* Phase 4/5 begin path: optional protocol preflight → acquire Task lease →
|
|
6
|
+
* register owner → Running state.
|
|
7
|
+
*/
|
|
8
|
+
export async function beginWorkerAttemptWithLease(input) {
|
|
9
|
+
if (!input.skipProtocolPreflight && input.protocolEnvelope !== undefined) {
|
|
10
|
+
assertCompatibleBeforeWrite(evaluateControllerCompatibility({ envelope: input.protocolEnvelope }));
|
|
11
|
+
}
|
|
12
|
+
const leaseRegistration = await acquireAttemptLease({
|
|
13
|
+
repoRoot: input.repoRoot,
|
|
14
|
+
featureId: input.featureId,
|
|
15
|
+
taskId: input.taskId,
|
|
16
|
+
workerRunId: input.workerRunId,
|
|
17
|
+
batchRunId: input.batchRunId,
|
|
18
|
+
...(input.revisionId ? { revisionId: input.revisionId } : {}),
|
|
19
|
+
...(input.controllerAnchor
|
|
20
|
+
? { controllerAnchor: input.controllerAnchor }
|
|
21
|
+
: {}),
|
|
22
|
+
...(input.now ? { now: input.now } : {}),
|
|
23
|
+
});
|
|
24
|
+
const begun = await beginWorkerAttempt(input);
|
|
25
|
+
return { ...begun, leaseRegistration };
|
|
26
|
+
}
|