@felixdotgo/aidlc-kit 0.0.1 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEVELOPMENT.md +1 -1
- package/README.md +20 -9
- package/dist/assets/.agents/aidlc/conventions.md +8 -8
- package/dist/assets/.agents/aidlc/orchestrator.md +15 -10
- package/dist/assets/.agents/aidlc/phase-build.md +7 -7
- package/dist/assets/.agents/aidlc/phase-clarify.md +3 -3
- package/dist/assets/.agents/aidlc/phase-plan.md +6 -6
- package/dist/assets/.agents/aidlc/phase-wrap.md +1 -1
- package/dist/assets/.agents/aidlc/schemas/manifest.schema.json +1 -1
- package/dist/assets/.agents/aidlc/scripts/assignment.mjs +36 -0
- package/dist/assets/.agents/aidlc/scripts/context.mjs +2 -0
- package/dist/assets/.agents/aidlc/scripts/gate-check.mjs +2 -0
- package/dist/assets/.agents/aidlc/scripts/gate-view.mjs +2 -0
- package/dist/assets/.agents/aidlc/scripts/lib/assignment-store.mjs +181 -0
- package/dist/assets/.agents/aidlc/scripts/lib/authority.mjs +3 -0
- package/dist/assets/.agents/aidlc/scripts/lib/context-runtime.mjs +13 -1
- package/dist/assets/.agents/aidlc/scripts/lib/feature-store.mjs +70 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-drivers.mjs +220 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-fleet-cli.mjs +183 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-fleet-remote.mjs +100 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-fleet-supervisor.mjs +58 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-fleet.mjs +320 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-git.mjs +284 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-intake.mjs +51 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-lead-artifacts.mjs +63 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-lead-broker.mjs +53 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-lead-local-broker.mjs +44 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-lead-sandbox.mjs +88 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-lease.mjs +89 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-local-gate-receipt.mjs +32 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-local-workers.mjs +162 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-planner.mjs +13 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-runtime.mjs +260 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-supervisor-local.mjs +131 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-supervisor-run-local.mjs +55 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-supervisor.mjs +101 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-worker-docker.mjs +32 -0
- package/dist/assets/.agents/aidlc/scripts/lib/loop-worker-snapshot.mjs +66 -0
- package/dist/assets/.agents/aidlc/scripts/lib/remote-state-client.mjs +112 -0
- package/dist/assets/.agents/aidlc/scripts/lib/store.mjs +41 -6
- package/dist/assets/.agents/aidlc/scripts/loop-entry.mjs +90 -0
- package/dist/assets/.agents/aidlc/scripts/loop-gate.mjs +58 -0
- package/dist/assets/.agents/aidlc/scripts/loop.mjs +36 -0
- package/dist/assets/.agents/aidlc/scripts/remote-state.mjs +311 -0
- package/dist/assets/.agents/aidlc/scripts/render.mjs +2 -0
- package/dist/assets/.agents/aidlc/scripts/state.mjs +71 -11
- package/dist/assets/.agents/aidlc/scripts/task-next.mjs +2 -0
- package/dist/assets/.agents/aidlc/templates/model-contract.md +3 -3
- package/dist/assets/.agents/aidlc/templates/workplan.md +2 -2
- package/dist/assets/.agents/skills/aidlc-build/SKILL.md +2 -2
- package/dist/assets/.agents/skills/aidlc-clarify/SKILL.md +2 -2
- package/dist/assets/.agents/skills/aidlc-plan/SKILL.md +2 -2
- package/dist/assets/.agents/skills/aidlc-wrap/SKILL.md +1 -1
- package/dist/services/aidlc-coordination/.env.fleet.example +11 -0
- package/dist/services/aidlc-coordination/compose.yaml +23 -2
- package/dist/services/aidlc-coordination/package-lock.json +1 -0
- package/dist/services/aidlc-coordination/package.json +2 -1
- package/dist/services/aidlc-coordination/src/coordination.mjs +1 -1
- package/dist/services/aidlc-coordination/src/fleet-client.mjs +36 -0
- package/dist/services/aidlc-coordination/src/fleet-run.mjs +18 -0
- package/dist/services/aidlc-coordination/src/fleet.mjs +233 -0
- package/dist/services/aidlc-coordination/src/git-gateway.mjs +50 -0
- package/dist/services/aidlc-coordination/src/lifecycle-core.mjs +41 -6
- package/dist/services/aidlc-coordination/src/repository.mjs +167 -4
- package/dist/services/aidlc-coordination/src/server.mjs +53 -9
- package/dist/services/aidlc-coordination/test/fleet.test.mjs +351 -0
- package/dist/services/aidlc-coordination/test/git-gateway.test.mjs +75 -0
- package/dist/services/aidlc-coordination/test/supervisor-fencing.test.mjs +59 -0
- package/dist/src/adapters.js +32 -5
- package/dist/src/cli.js +146 -70
- package/dist/src/context.js +2 -0
- package/dist/src/gate.d.ts +1 -0
- package/dist/src/gate.js +51 -38
- package/dist/src/installer.d.ts +10 -1
- package/dist/src/installer.js +157 -37
- package/dist/src/managed-content.js +1 -1
- package/dist/src/model.d.ts +16 -1
- package/dist/src/model.js +1 -1
- package/dist/src/profiles.d.ts +9 -1
- package/dist/src/profiles.js +37 -3
- package/dist/src/project-migration.d.ts +1 -0
- package/dist/src/project-migration.js +7 -0
- package/dist/src/state/archive.d.ts +17 -0
- package/dist/src/state/archive.js +60 -0
- package/dist/src/state/items.d.ts +25 -0
- package/dist/src/state/items.js +146 -0
- package/dist/src/state/lessons.d.ts +7 -0
- package/dist/src/state/lessons.js +67 -0
- package/dist/src/state/lifecycle.d.ts +5 -0
- package/dist/src/state/lifecycle.js +78 -0
- package/dist/src/state/lock.d.ts +7 -0
- package/dist/src/state/lock.js +124 -0
- package/dist/src/state/memory.d.ts +8 -0
- package/dist/src/state/memory.js +88 -0
- package/dist/src/state/render.d.ts +26 -0
- package/dist/src/state/render.js +29 -0
- package/dist/src/state/shared.d.ts +29 -0
- package/dist/src/state/shared.js +71 -0
- package/dist/src/state/store.d.ts +4 -0
- package/dist/src/state/store.js +81 -0
- package/dist/src/state/transitions.d.ts +21 -0
- package/dist/src/state/transitions.js +93 -0
- package/dist/src/state/validate.d.ts +4 -0
- package/dist/src/state/validate.js +100 -0
- package/dist/src/state.d.ts +15 -110
- package/dist/src/state.js +15 -764
- package/dist/src/tracker.d.ts +7 -0
- package/dist/src/tracker.js +3 -0
- package/dist/src/upgrade-board.d.ts +5 -0
- package/dist/src/upgrade-board.js +89 -0
- package/dist/src/upgrade.d.ts +5 -6
- package/dist/src/upgrade.js +211 -193
- package/dist/src/workflow.d.ts +1 -0
- package/dist/src/workflow.js +3 -2
- package/docs/README.md +4 -4
- package/docs/architecture.md +10 -2
- package/docs/command-reference.md +72 -11
- package/docs/configuration.md +26 -4
- package/docs/coordination-service.md +59 -3
- package/docs/development.md +16 -2
- package/docs/operating-workflow.md +53 -14
- package/docs/testing-and-release.md +53 -10
- package/package.json +3 -3
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
3
|
+
import { runFleetCoordinator, validateFleetLimits } from "./loop-fleet.mjs";
|
|
4
|
+
|
|
5
|
+
const taskDigest = (task) => createHash("sha256").update(JSON.stringify(task)).digest("hex");
|
|
6
|
+
const targets = { G0_confirm: "plan", G1_review: "build", G2_codereview: "wrap" };
|
|
7
|
+
|
|
8
|
+
/** The service owns the checkpoint and atomically binds each run claim to this
|
|
9
|
+
* supervisor. An unavailable or ambiguous checkpoint never authorizes work. */
|
|
10
|
+
export async function runFleetSupervisor({ repository, taskId, limits: inputLimits, pollMs = 5000, signal, onEvent = () => {}, coordinator = runFleetCoordinator, ...options }) {
|
|
11
|
+
const limits = validateFleetLimits(inputLimits);
|
|
12
|
+
if (!repository || typeof taskId !== "string" || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(taskId) || !Number.isSafeInteger(pollMs) || pollMs < 1 || pollMs > 300000) throw new Error("Invalid fleet supervisor configuration");
|
|
13
|
+
for (const name of ["claimSupervisor", "renewSupervisor", "releaseSupervisor", "readGateCheckpoint", "readTask", "claimLead"]) if (typeof repository[name] !== "function") throw new Error(`Fleet supervisor requires ${name}`);
|
|
14
|
+
await repository.claimSupervisor({ taskId, leaseMs: limits.leaseMs, idempotencyKey: randomUUID() });
|
|
15
|
+
const controller = new AbortController();
|
|
16
|
+
const combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
|
|
17
|
+
let failure; let pending = Promise.resolve();
|
|
18
|
+
const timer = setInterval(() => { pending = pending.then(() => repository.renewSupervisor({ taskId, leaseMs: limits.leaseMs })).catch((error) => { failure = error; controller.abort(error); }); }, limits.heartbeatMs);
|
|
19
|
+
timer.unref?.();
|
|
20
|
+
try {
|
|
21
|
+
while (!combined.aborted) {
|
|
22
|
+
const snapshot = await repository.readTask({ taskId });
|
|
23
|
+
const checkpoint = await repository.readGateCheckpoint({ taskId });
|
|
24
|
+
if (!snapshot?.task || snapshot.task.id !== taskId || !Number.isSafeInteger(snapshot.revision)) throw new Error("Invalid fleet canonical snapshot");
|
|
25
|
+
const task = snapshot.task;
|
|
26
|
+
if (task.status === "paused" || task.handoff || ["blocked", "terminal"].includes(snapshot.nextAction?.classification)) return { status: "blocked", taskId, reason: "Task is paused, blocked, or terminal" };
|
|
27
|
+
if (snapshot.nextAction?.classification === "complete") return { status: "complete", taskId };
|
|
28
|
+
if (checkpoint?.status === "pending") {
|
|
29
|
+
if (task.status !== "blocked_on_user" || task.gate !== checkpoint.gate || taskDigest(task) !== checkpoint.taskDigest || snapshot.nextAction?.classification !== "await_user") return { status: "blocked", taskId, reason: "Pending checkpoint does not match canonical gate" };
|
|
30
|
+
await onEvent({ type: "waiting", taskId, gate: checkpoint.gate, snapshotId: checkpoint.snapshotId });
|
|
31
|
+
try { await sleep(pollMs, undefined, { signal: combined }); } catch (error) { if (error?.name !== "AbortError") throw error; }
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
if (checkpoint?.status === "consumed") return { status: "blocked", taskId, reason: "Consumed checkpoint requires reconciliation before restart" };
|
|
35
|
+
if (checkpoint && checkpoint.status !== "approved") return { status: "blocked", taskId, reason: "Approved checkpoint does not match canonical state" };
|
|
36
|
+
if (checkpoint) {
|
|
37
|
+
// A task snapshot older than the atomic approval is a read race between
|
|
38
|
+
// the two reads, not a mismatch: re-read the canonical task immediately.
|
|
39
|
+
if (Number.isSafeInteger(checkpoint.approvalRevision) && checkpoint.approvalRevision > snapshot.revision) { await onEvent({ type: "resync", taskId, gate: checkpoint.gate }); continue; }
|
|
40
|
+
if (task.phase !== targets[checkpoint.gate] || task.status !== "active" || !Number.isSafeInteger(checkpoint.approvalRevision)) return { status: "blocked", taskId, reason: "Approved checkpoint does not match canonical state" };
|
|
41
|
+
}
|
|
42
|
+
if (snapshot.nextAction?.classification !== "run_phase" || task.status !== "active") return { status: "blocked", taskId, reason: "Task is not actionable or gate checkpoint is missing" };
|
|
43
|
+
await repository.renewSupervisor({ taskId, leaseMs: limits.leaseMs });
|
|
44
|
+
if (combined.aborted) break;
|
|
45
|
+
const claim = await repository.claimLead({ taskId, leaseMs: limits.leaseMs, expectedStateRevision: snapshot.revision, expectedTaskDigest: taskDigest(task), checkpoint, idempotencyKey: randomUUID() });
|
|
46
|
+
const result = await coordinator({ ...options, repository, taskId, limits, claimedLead: claim, supervisorSignal: combined, onEvent });
|
|
47
|
+
if (failure) throw failure;
|
|
48
|
+
if (result?.status !== "gate") return result;
|
|
49
|
+
// gate_snapshot_put stores snapshot and checkpoint in one service transaction.
|
|
50
|
+
await onEvent({ type: "gate", taskId, gate: result.gate, snapshotId: result.snapshotId, gateView: result.gateView });
|
|
51
|
+
}
|
|
52
|
+
if (failure) throw new Error(`Supervisor lease renewal failed: ${failure.message}`);
|
|
53
|
+
return { status: "stopped", taskId };
|
|
54
|
+
} finally {
|
|
55
|
+
clearInterval(timer); await pending;
|
|
56
|
+
await repository.releaseSupervisor({ taskId });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fleet coordination: job reconciliation, coordinator loop, and worker loop
|
|
3
|
+
* for multi-host AI-DLC execution.
|
|
4
|
+
*/
|
|
5
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
6
|
+
import { readyTaskItems, formatGateView } from "./store.mjs";
|
|
7
|
+
import { isLoopProvider, LOOP_PROVIDER_LIST } from "./loop-drivers.mjs";
|
|
8
|
+
|
|
9
|
+
export const FLEET_LIMITS = Object.freeze({ maxTurns: 12, turnTimeoutMs: 600_000, maxStallTurns: 2, maxWorkers: 4, leaseMs: 300_000, heartbeatMs: 60_000 });
|
|
10
|
+
const ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
11
|
+
const digest = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
12
|
+
const integer = (value, name, max) => { if (!Number.isSafeInteger(value) || value < 1 || value > max) throw new Error(`${name} must be an integer from 1 to ${max}`); return value; };
|
|
13
|
+
const required = (object, method) => { if (typeof object?.[method] !== "function") throw new Error(`Fleet adapter requires ${method}()`); return object[method].bind(object); };
|
|
14
|
+
const assertId = (value, label) => { if (typeof value !== "string" || !ID.test(value)) throw new Error(`Invalid ${label}`); return value; };
|
|
15
|
+
const actionKind = (action) => { if (!["run_phase", "dispatch", "await_user", "reconcile", "blocked", "terminal", "complete"].includes(action?.classification)) throw new Error(`Invalid canonical nextAction: ${action?.classification}`); return action.classification; };
|
|
16
|
+
|
|
17
|
+
export function validateFleetLimits(input = {}) {
|
|
18
|
+
const value = { ...FLEET_LIMITS, ...input };
|
|
19
|
+
integer(value.maxTurns, "maxTurns", 1000);
|
|
20
|
+
integer(value.turnTimeoutMs, "turnTimeoutMs", 3_600_000);
|
|
21
|
+
integer(value.maxStallTurns, "maxStallTurns", 20);
|
|
22
|
+
integer(value.maxWorkers, "maxWorkers", 100);
|
|
23
|
+
integer(value.leaseMs, "leaseMs", 3_600_000);
|
|
24
|
+
integer(value.heartbeatMs, "heartbeatMs", 600_000);
|
|
25
|
+
if (value.heartbeatMs * 2 >= value.leaseMs) throw new Error("heartbeatMs must be less than half the leaseMs");
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function validateSnapshot(snapshot, taskId) {
|
|
30
|
+
if (!snapshot || snapshot.task?.id !== taskId || !Number.isSafeInteger(snapshot.revision) || snapshot.revision < 0) throw new Error(`Invalid remote task snapshot for ${taskId}`);
|
|
31
|
+
actionKind(snapshot.nextAction);
|
|
32
|
+
return snapshot;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function readyItems(task) {
|
|
36
|
+
const items = Array.isArray(task.tasks) ? task.tasks : [];
|
|
37
|
+
const ready = new Set(readyTaskItems(task).ready);
|
|
38
|
+
return items.filter((item) => item.status === "todo" && ready.has(item.id));
|
|
39
|
+
}
|
|
40
|
+
const overlaps = (left, right) => left === right || (left.endsWith("/") && right.startsWith(left)) || (right.endsWith("/") && left.startsWith(right));
|
|
41
|
+
const pathsOverlap = (left = [], right = []) => left.some((a) => right.some((b) => overlaps(a, b)));
|
|
42
|
+
const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
43
|
+
|
|
44
|
+
function key(...parts) { return digest(parts); }
|
|
45
|
+
|
|
46
|
+
export async function reconcileFleetJobs({ repository, taskId, leadEpoch, snapshot, handoff, now = Date.now(), allowCodeOnlyRetry = false, quiesceMs = 0 }) {
|
|
47
|
+
const jobs = await required(repository, "listJobs")({ taskId });
|
|
48
|
+
if (!Array.isArray(jobs)) throw new Error("Invalid job list");
|
|
49
|
+
const pending = [];
|
|
50
|
+
for (const job of jobs) {
|
|
51
|
+
if (job.taskId !== taskId || !job.assignmentId || !job.itemId) throw new Error("Invalid assignment in job list");
|
|
52
|
+
const item = snapshot.task.tasks?.find((candidate) => candidate.id === job.itemId);
|
|
53
|
+
if (item?.status === "done") {
|
|
54
|
+
if (["reported", "uncertain"].includes(job.state)) {
|
|
55
|
+
const result = job.state === "reported" ? await required(handoff, "verify")({ job, snapshot }) : await required(handoff, "recover")({ job, snapshot });
|
|
56
|
+
if (!result?.valid || !result.alreadyIntegrated) pending.push({ assignmentId: job.assignmentId, itemId: job.itemId, reason: result?.reason ?? "Canonical item done but reported commit is not integrated" });
|
|
57
|
+
else {
|
|
58
|
+
if (job.state === "uncertain") await required(repository, "recoverJob")({ assignmentId: job.assignmentId, leadEpoch, commitSha: result.report.commitSha, changedPaths: result.report.changedPaths, idempotencyKey: key("recover", job.assignmentId, leadEpoch, result.commitSha) });
|
|
59
|
+
await required(repository, "reconcileJob")({ assignmentId: job.assignmentId, leadEpoch, decision: "integrated", idempotencyKey: key("integrated", job.assignmentId, leadEpoch, result.commitSha) });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (job.state === "leased" && Date.parse(job.expiresAt) <= now) {
|
|
65
|
+
const reconcileJob = required(repository, "reconcileJob");
|
|
66
|
+
const updated = await reconcileJob({ assignmentId: job.assignmentId, leadEpoch, decision: "uncertain", reason: "worker lease expired", idempotencyKey: key("expire", job.assignmentId, leadEpoch, job.attempt, job.epoch) });
|
|
67
|
+
job.state = "uncertain";
|
|
68
|
+
job.uncertainAt = updated?.fleet?.jobs?.[job.assignmentId]?.uncertainAt ?? new Date(now).toISOString();
|
|
69
|
+
}
|
|
70
|
+
if (job.state === "uncertain") {
|
|
71
|
+
if (typeof handoff?.recover === "function") {
|
|
72
|
+
const recovered = await handoff.recover({ job, snapshot });
|
|
73
|
+
if (recovered?.recovered) {
|
|
74
|
+
await required(repository, "recoverJob")({ assignmentId: job.assignmentId, leadEpoch, commitSha: recovered.report.commitSha, changedPaths: recovered.report.changedPaths, idempotencyKey: key("recover", job.assignmentId, leadEpoch, recovered.report.commitSha) });
|
|
75
|
+
job.report = recovered.report;
|
|
76
|
+
job.state = "reported";
|
|
77
|
+
pending.push({ assignmentId: job.assignmentId, itemId: job.itemId, reason: "recovered pushed branch after lost report", artifact: { valid: true, ...recovered } });
|
|
78
|
+
} else if (allowCodeOnlyRetry && recovered?.retryable === true && Number.isFinite(Date.parse(job.uncertainAt))) {
|
|
79
|
+
const waitUntil = Date.parse(job.uncertainAt) + quiesceMs;
|
|
80
|
+
if (now < waitUntil) pending.push({ assignmentId: job.assignmentId, itemId: job.itemId, reason: "waiting for stale worker turn to quiesce", waitUntil });
|
|
81
|
+
else {
|
|
82
|
+
await required(repository, "reconcileJob")({ assignmentId: job.assignmentId, leadEpoch, decision: "ready", idempotencyKey: randomUUID() });
|
|
83
|
+
job.state = "ready";
|
|
84
|
+
}
|
|
85
|
+
} else pending.push({ assignmentId: job.assignmentId, itemId: job.itemId, reason: recovered?.reason ?? "uncertain assignment; inspect Git branch before retry" });
|
|
86
|
+
} else pending.push({ assignmentId: job.assignmentId, itemId: job.itemId, reason: "uncertain assignment; inspect Git branch before retry" });
|
|
87
|
+
} else if (job.state === "reported") {
|
|
88
|
+
if (typeof handoff?.verify !== "function") throw new Error("Reported job requires Git handoff verify adapter");
|
|
89
|
+
const result = await handoff.verify({ job, snapshot });
|
|
90
|
+
if (!result?.valid) pending.push({ assignmentId: job.assignmentId, itemId: job.itemId, reason: result?.reason ?? "Git artifact verification failed" });
|
|
91
|
+
else pending.push({ assignmentId: job.assignmentId, itemId: job.itemId, reason: "verified report awaits lead integration", artifact: result });
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return { jobs, pending };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async function withHeartbeat(renew, intervalMs, work) {
|
|
98
|
+
const abort = new AbortController();
|
|
99
|
+
let failure;
|
|
100
|
+
let pending = Promise.resolve();
|
|
101
|
+
const timer = setInterval(() => {
|
|
102
|
+
pending = pending.then(() => renew()).catch((error) => { failure = error; abort.abort(error); });
|
|
103
|
+
}, intervalMs);
|
|
104
|
+
timer.unref?.();
|
|
105
|
+
try {
|
|
106
|
+
const result = await work(abort.signal);
|
|
107
|
+
await pending;
|
|
108
|
+
if (failure) throw new Error(`Fleet lease renewal failed: ${failure.message}`);
|
|
109
|
+
return result;
|
|
110
|
+
} finally { clearInterval(timer); await pending; }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* One fenced coordinator invocation. The repository is the remote authority;
|
|
115
|
+
* no local lifecycle file or PID lease is consulted in fleet mode.
|
|
116
|
+
*/
|
|
117
|
+
export async function runFleetCoordinator({ repository, taskId, ownerId = randomUUID(), workerIds = [], workerProviders, reservedJobs = [], allowCodeOnlyRetry = false, limits: inputLimits, pollMs = 5000, pickupTimeoutMs: inputPickupTimeout, packet, driver, gateView, handoff, completeItem, claimedLead, supervisorSignal, onEvent = () => {} }) {
|
|
118
|
+
assertId(taskId, "task ID"); assertId(ownerId, "owner ID");
|
|
119
|
+
if (!Array.isArray(workerIds) || new Set(workerIds).size !== workerIds.length) throw new Error("workerIds must be distinct IDs");
|
|
120
|
+
for (const workerId of workerIds) assertId(workerId, "worker ID");
|
|
121
|
+
if (workerProviders !== undefined && (!workerProviders || typeof workerProviders !== "object" || Array.isArray(workerProviders) || Object.keys(workerProviders).length !== workerIds.length || workerIds.some((workerId) => !isLoopProvider(workerProviders[workerId])) || Object.keys(workerProviders).some((workerId) => !workerIds.includes(workerId)))) throw new Error(`workerProviders must map each configured worker ID to ${LOOP_PROVIDER_LIST}`);
|
|
122
|
+
const limits = validateFleetLimits(inputLimits);
|
|
123
|
+
// --- Validate limits and normalize inputs ---------------------------------
|
|
124
|
+
integer(pollMs, "pollMs", 300000);
|
|
125
|
+
const pickupTimeoutMs = integer(inputPickupTimeout ?? limits.leaseMs, "pickupTimeoutMs", 3_600_000);
|
|
126
|
+
const readTask = required(repository, "readTask");
|
|
127
|
+
const claimLead = required(repository, "claimLead");
|
|
128
|
+
const renewLead = required(repository, "renewLead");
|
|
129
|
+
const releaseLead = required(repository, "releaseLead");
|
|
130
|
+
const listJobs = required(repository, "listJobs");
|
|
131
|
+
const scheduleJob = required(repository, "scheduleJob");
|
|
132
|
+
const saveGateSnapshot = required(repository, "saveGateSnapshot");
|
|
133
|
+
let snapshot = validateSnapshot(await readTask({ taskId }), taskId);
|
|
134
|
+
const claim = claimedLead ?? await claimLead({ taskId, ownerId, leaseMs: limits.leaseMs, expectedRevision: snapshot.revision, idempotencyKey: randomUUID() });
|
|
135
|
+
if (!Number.isSafeInteger(claim?.epoch) || claim.epoch < 1) throw new Error("Lead claim lacks a fencing epoch");
|
|
136
|
+
const epoch = claim.epoch;
|
|
137
|
+
let turns = 0, stalls = 0, lastSignal = null;
|
|
138
|
+
const startedAt = Date.now();
|
|
139
|
+
const readySeen = new Map();
|
|
140
|
+
try {
|
|
141
|
+
return await withHeartbeat(() => renewLead({ taskId, ownerId, epoch, leaseMs: limits.leaseMs }), limits.heartbeatMs, async (leaseSignal) => {
|
|
142
|
+
const signal = supervisorSignal ? AbortSignal.any([leaseSignal, supervisorSignal]) : leaseSignal;
|
|
143
|
+
for (;;) {
|
|
144
|
+
if (signal.aborted) throw new Error("Lead lease renewal failed");
|
|
145
|
+
if (Date.now() - startedAt >= limits.maxTurns * limits.turnTimeoutMs) return { status: "paused", taskId, reason: "coordinator wall-time limit reached" };
|
|
146
|
+
snapshot = validateSnapshot(await readTask({ taskId }), taskId);
|
|
147
|
+
const kind = actionKind(snapshot.nextAction);
|
|
148
|
+
if (kind === "await_user") {
|
|
149
|
+
const reconciled = await reconcileFleetJobs({ repository, taskId, leadEpoch: epoch, snapshot, handoff, allowCodeOnlyRetry, quiesceMs: limits.turnTimeoutMs + limits.heartbeatMs + 1000 });
|
|
150
|
+
if (reconciled.pending.length && reconciled.pending.every((entry) => entry.waitUntil)) { await wait(pollMs); continue; }
|
|
151
|
+
if (reconciled.pending.length) return { status: "reconcile", taskId, revision: snapshot.revision, pending: reconciled.pending };
|
|
152
|
+
const gate = snapshot.nextAction.gate;
|
|
153
|
+
const baseReview = await (gateView ?? ((value) => formatGateView(value.task, [], "markdown")))(snapshot);
|
|
154
|
+
if (typeof baseReview !== "string" || !baseReview.trim()) throw new Error("Empty gate review snapshot");
|
|
155
|
+
const integrationCommit = gate === "G2_codereview" ? await required(handoff, "currentCommit")() : undefined;
|
|
156
|
+
const review = integrationCommit ? `${baseReview.trimEnd()}\n\nIntegration commit: ${integrationCommit}\n` : baseReview;
|
|
157
|
+
const saved = await saveGateSnapshot({ taskId, leadEpoch: epoch, expectedRevision: snapshot.revision, gate, gateView: review, integrationCommit, idempotencyKey: key("gate", taskId, snapshot.revision, gate, integrationCommit ?? "") });
|
|
158
|
+
onEvent({ type: "gate", taskId, gate, snapshotId: saved.snapshotId });
|
|
159
|
+
return { status: "gate", taskId, gate, snapshotId: saved.snapshotId, revision: snapshot.revision, gateView: review };
|
|
160
|
+
}
|
|
161
|
+
if (["blocked", "terminal", "complete"].includes(kind)) return { status: kind, taskId, revision: snapshot.revision, nextAction: snapshot.nextAction };
|
|
162
|
+
if (turns >= limits.maxTurns) return { status: "paused", taskId, reason: "turn limit reached", revision: snapshot.revision };
|
|
163
|
+
const before = digest({ revision: snapshot.revision, action: snapshot.nextAction });
|
|
164
|
+
if (kind === "dispatch" || (kind === "run_phase" && snapshot.task.phase === "build")) {
|
|
165
|
+
const reconciled = await reconcileFleetJobs({ repository, taskId, leadEpoch: epoch, snapshot, handoff, allowCodeOnlyRetry, quiesceMs: limits.turnTimeoutMs + limits.heartbeatMs + 1000 });
|
|
166
|
+
if (reconciled.pending.length && reconciled.pending.every((entry) => entry.waitUntil)) { await wait(pollMs); continue; }
|
|
167
|
+
if (reconciled.pending.length) {
|
|
168
|
+
const verified = reconciled.pending.filter((entry) => entry.artifact?.valid);
|
|
169
|
+
if (verified.length && typeof handoff?.integrate === "function" && typeof completeItem === "function") {
|
|
170
|
+
for (const entry of verified) {
|
|
171
|
+
const job = reconciled.jobs.find((candidate) => candidate.assignmentId === entry.assignmentId);
|
|
172
|
+
const integrated = await handoff.integrate({ job, artifact: entry.artifact });
|
|
173
|
+
const fresh = validateSnapshot(await readTask({ taskId }), taskId);
|
|
174
|
+
await completeItem({ taskId, itemId: job.itemId, job, integrated, snapshot: fresh, leadEpoch: epoch });
|
|
175
|
+
const committed = validateSnapshot(await readTask({ taskId }), taskId);
|
|
176
|
+
if (committed.task.tasks?.find((item) => item.id === job.itemId)?.status !== "done") throw new Error(`Canonical item ${job.itemId} was not completed after integration`);
|
|
177
|
+
await required(repository, "reconcileJob")({ assignmentId: job.assignmentId, leadEpoch: epoch, decision: "integrated", idempotencyKey: key("integrated", job.assignmentId, epoch, integrated.integrationCommit) });
|
|
178
|
+
onEvent({ type: "job_integrated", taskId, itemId: job.itemId, assignmentId: job.assignmentId, commitSha: integrated.integrationCommit });
|
|
179
|
+
}
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
return { status: "reconcile", taskId, revision: snapshot.revision, pending: reconciled.pending };
|
|
183
|
+
}
|
|
184
|
+
const jobs = reconciled.jobs;
|
|
185
|
+
const active = jobs.filter((job) => ["ready", "leased", "reported", "uncertain"].includes(job.state));
|
|
186
|
+
const pathClaims = [...active, ...reservedJobs].filter((job) => ["ready", "leased", "reported", "uncertain"].includes(job.state));
|
|
187
|
+
const reserved = new Set(active.map((job) => job.itemId));
|
|
188
|
+
const load = new Map(workerIds.map((id) => [id, active.filter((job) => job.workerId === id).length]));
|
|
189
|
+
for (const item of readyItems(snapshot.task)) {
|
|
190
|
+
if (active.length >= limits.maxWorkers) break;
|
|
191
|
+
if (reserved.has(item.id)) continue;
|
|
192
|
+
if (!workerIds.length) throw new Error("Fleet dispatch requires at least one configured worker ID");
|
|
193
|
+
if (typeof handoff?.prepare !== "function") throw new Error("Fleet dispatch requires Git handoff prepare adapter");
|
|
194
|
+
const artifact = await handoff.prepare({ taskId, item, snapshot });
|
|
195
|
+
if (pathClaims.some((job) => pathsOverlap(artifact.ownedPaths, job.ownedPaths))) continue;
|
|
196
|
+
const workerId = [...load].sort((a, b) => a[1] - b[1])[0][0];
|
|
197
|
+
const agent = workerProviders ? workerProviders[workerId] : artifact.agent;
|
|
198
|
+
if (!isLoopProvider(agent)) throw new Error(`Invalid provider for worker ${workerId}`);
|
|
199
|
+
const job = await scheduleJob({ taskId, itemId: item.id, workerId, leadEpoch: epoch, expectedRevision: snapshot.revision, baseCommit: artifact.baseCommit, branch: artifact.branch, ownedPaths: artifact.ownedPaths, agent, idempotencyKey: randomUUID() });
|
|
200
|
+
active.push(job); reserved.add(item.id);
|
|
201
|
+
pathClaims.push({ ...job, ownedPaths: artifact.ownedPaths, state: "ready" });
|
|
202
|
+
load.set(workerId, load.get(workerId) + 1);
|
|
203
|
+
onEvent({ type: "job_scheduled", taskId, itemId: item.id, assignmentId: job.assignmentId });
|
|
204
|
+
}
|
|
205
|
+
if (active.length) {
|
|
206
|
+
for (const job of active) {
|
|
207
|
+
if (job.state !== "ready") { readySeen.delete(job.assignmentId); continue; }
|
|
208
|
+
if (!readySeen.has(job.assignmentId)) readySeen.set(job.assignmentId, Date.now());
|
|
209
|
+
if (Date.now() - readySeen.get(job.assignmentId) >= pickupTimeoutMs) {
|
|
210
|
+
await required(repository, "reconcileJob")({ assignmentId: job.assignmentId, leadEpoch: epoch, decision: "uncertain", idempotencyKey: randomUUID() });
|
|
211
|
+
return { status: "blocked", taskId, reason: `Worker did not claim assignment ${job.assignmentId} within ${pickupTimeoutMs} ms` };
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
onEvent({ type: "jobs_active", taskId, assignments: active.map((job) => job.assignmentId) });
|
|
215
|
+
await wait(pollMs);
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (kind === "reconcile") return { status: "reconcile", taskId, revision: snapshot.revision, nextAction: snapshot.nextAction };
|
|
220
|
+
if (typeof packet !== "function" || typeof driver !== "function") throw new Error("Fleet lead requires packet and driver adapters");
|
|
221
|
+
const prompt = await packet(snapshot, epoch);
|
|
222
|
+
if (typeof prompt !== "string" || !prompt) throw new Error("Empty fleet lead packet");
|
|
223
|
+
await driver({ taskId, snapshot, leadEpoch: epoch, prompt, timeoutMs: limits.turnTimeoutMs, signal });
|
|
224
|
+
turns++;
|
|
225
|
+
const after = validateSnapshot(await readTask({ taskId }), taskId);
|
|
226
|
+
const nextSignal = digest({ revision: after.revision, action: after.nextAction });
|
|
227
|
+
stalls = nextSignal === before || nextSignal === lastSignal ? stalls + 1 : 0;
|
|
228
|
+
lastSignal = nextSignal;
|
|
229
|
+
if (stalls >= limits.maxStallTurns) return { status: "paused", taskId, reason: `${stalls} lead turns without canonical progress`, revision: after.revision };
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
} finally {
|
|
233
|
+
await releaseLead({ taskId, ownerId, epoch }).catch((error) => onEvent({ type: "release_failed", taskId, message: error.message }));
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** A worker claims one attempt, persists its exact CLI session, and reports a Git artifact. */
|
|
238
|
+
export async function runFleetWorker({ repository, workerId = randomUUID(), workerAgent, preflight, limits: inputLimits, driver, packet, handoff, onEvent = () => {} }) {
|
|
239
|
+
assertId(workerId, "worker ID");
|
|
240
|
+
if (workerAgent !== undefined && !isLoopProvider(workerAgent)) throw new Error(`workerAgent must be ${LOOP_PROVIDER_LIST}`);
|
|
241
|
+
const limits = validateFleetLimits(inputLimits);
|
|
242
|
+
// --- Validate limits and bind adapter methods ------------------------------
|
|
243
|
+
const claimJob = required(repository, "claimJob");
|
|
244
|
+
const renewJob = required(repository, "renewJob");
|
|
245
|
+
const checkpointJob = required(repository, "checkpointJob");
|
|
246
|
+
const reportJob = required(repository, "reportJob");
|
|
247
|
+
const readTask = required(repository, "readTask");
|
|
248
|
+
const job = await claimJob({ workerId, leaseMs: limits.leaseMs, idempotencyKey: key("worker-claim", workerId, randomUUID()) });
|
|
249
|
+
if (!job) return { status: "idle" };
|
|
250
|
+
if (!isLoopProvider(job.agent) || (workerAgent !== undefined && job.agent !== workerAgent)) throw new Error(`Assigned provider does not match worker ${workerId} capability`);
|
|
251
|
+
if (preflight !== undefined) {
|
|
252
|
+
if (typeof preflight !== "function") throw new Error("Worker preflight must be a function");
|
|
253
|
+
await preflight(job.agent);
|
|
254
|
+
}
|
|
255
|
+
assertId(job.taskId, "job task ID"); assertId(job.itemId, "job item ID");
|
|
256
|
+
if (!Number.isSafeInteger(job.epoch) || job.epoch < 1 || !Number.isSafeInteger(job.attempt) || job.attempt < 1 || !job.assignmentId) throw new Error("Job claim lacks a fenced attempt");
|
|
257
|
+
const identity = { assignmentId: job.assignmentId, attempt: job.attempt, epoch: job.epoch, workerId };
|
|
258
|
+
let sessionId = job.sessionId ?? null;
|
|
259
|
+
let turns = 0, stalls = 0;
|
|
260
|
+
const report = async (outcome) => reportJob({ ...identity, report: outcome, idempotencyKey: key("job-report", job.assignmentId, job.attempt, job.epoch, outcome.status, outcome.commitSha ?? "") });
|
|
261
|
+
return withHeartbeat(() => renewJob({ ...identity, leaseMs: limits.leaseMs }), limits.heartbeatMs, async (signal) => {
|
|
262
|
+
if (typeof driver !== "function" || typeof packet !== "function" || typeof handoff?.prepare !== "function" || typeof handoff?.observe !== "function" || typeof handoff?.finish !== "function") throw new Error("Worker requires driver, packet, and Git handoff adapters");
|
|
263
|
+
const workspace = await handoff.prepare(job);
|
|
264
|
+
if (!workspace?.root) throw new Error("Git handoff did not supply a worker root");
|
|
265
|
+
let lastObservation = null;
|
|
266
|
+
for (;;) {
|
|
267
|
+
if (signal.aborted) throw new Error("Job lease renewal failed");
|
|
268
|
+
const snapshot = validateSnapshot(await readTask({ taskId: job.taskId }), job.taskId);
|
|
269
|
+
const item = snapshot.task.tasks?.find((candidate) => candidate.id === job.itemId);
|
|
270
|
+
if (!item || item.status === "done") {
|
|
271
|
+
await report({ status: "superseded", reason: "Item absent or canonically done" });
|
|
272
|
+
return { status: "superseded", assignmentId: job.assignmentId };
|
|
273
|
+
}
|
|
274
|
+
const kind = actionKind(snapshot.nextAction);
|
|
275
|
+
if (kind === "await_user" || kind === "blocked" || kind === "terminal") {
|
|
276
|
+
await report({ status: "paused", reason: `Canonical action: ${kind}` });
|
|
277
|
+
return { status: "paused", assignmentId: job.assignmentId };
|
|
278
|
+
}
|
|
279
|
+
if (turns >= limits.maxTurns) {
|
|
280
|
+
await report({ status: "uncertain", reason: "turn limit reached", sessionId });
|
|
281
|
+
return { status: "uncertain", assignmentId: job.assignmentId };
|
|
282
|
+
}
|
|
283
|
+
const prompt = await packet({ job, snapshot, workspace, turns });
|
|
284
|
+
if (typeof prompt !== "string" || !prompt) throw new Error("Empty worker packet");
|
|
285
|
+
const result = await driver({ agent: job.agent, root: workspace.root, sessionId: sessionId ?? undefined, prompt: `${prompt}\n\nWorker scope: modify only assigned paths. Do not mutate AI-DLC lifecycle state or approve any gate. When the scoped implementation is ready for commit, end the final answer with AI-DLC-ITEM-COMPLETE on its own line.`, timeoutMs: limits.turnTimeoutMs, signal, onSessionId: async (id) => {
|
|
286
|
+
if (typeof id !== "string" || !id) throw new Error("Invalid worker session ID");
|
|
287
|
+
if (sessionId && sessionId !== id) throw new Error("Worker session changed during the same assignment");
|
|
288
|
+
sessionId = id;
|
|
289
|
+
}, onEvent });
|
|
290
|
+
if (result?.sessionId && sessionId && result.sessionId !== sessionId) throw new Error("Worker session changed during the same assignment");
|
|
291
|
+
if (result?.sessionId && !sessionId) {
|
|
292
|
+
sessionId = result.sessionId;
|
|
293
|
+
}
|
|
294
|
+
if (signal.aborted) throw new Error("Job lease renewal failed after worker turn");
|
|
295
|
+
await renewJob({ ...identity, leaseMs: limits.leaseMs });
|
|
296
|
+
if (signal.aborted) throw new Error("Job lease renewal failed after worker turn");
|
|
297
|
+
turns++;
|
|
298
|
+
if (sessionId) await checkpointJob({ ...identity, sessionId, checkpoint: { agent: job.agent, turns, completedAt: new Date().toISOString() } });
|
|
299
|
+
const observed = await handoff.observe({ job, workspace, result, turns });
|
|
300
|
+
if (observed?.done) {
|
|
301
|
+
await renewJob({ ...identity, leaseMs: limits.leaseMs });
|
|
302
|
+
if (signal.aborted) throw new Error("Job lease renewal failed before Git handoff");
|
|
303
|
+
const artifact = await handoff.finish({ job, workspace, result });
|
|
304
|
+
if (signal.aborted) throw new Error("Job lease renewal failed during Git handoff; reconcile remote state");
|
|
305
|
+
await renewJob({ ...identity, leaseMs: limits.leaseMs });
|
|
306
|
+
if (signal.aborted) throw new Error("Job lease renewal failed after Git handoff; reconcile remote state");
|
|
307
|
+
if (!artifact?.commitSha || !Array.isArray(artifact.changedPaths)) throw new Error("Invalid Git handoff artifact");
|
|
308
|
+
await report({ status: "reported", commitSha: artifact.commitSha, changedPaths: artifact.changedPaths, sessionId });
|
|
309
|
+
return { status: "reported", assignmentId: job.assignmentId, commitSha: artifact.commitSha };
|
|
310
|
+
}
|
|
311
|
+
const currentObservation = digest({ revision: snapshot.revision, artifact: observed?.fingerprint ?? null });
|
|
312
|
+
stalls = currentObservation === lastObservation ? stalls + 1 : 0;
|
|
313
|
+
lastObservation = currentObservation;
|
|
314
|
+
if (stalls >= limits.maxStallTurns) {
|
|
315
|
+
await report({ status: "uncertain", reason: `${stalls} worker turns without observable progress`, sessionId });
|
|
316
|
+
return { status: "uncertain", assignmentId: job.assignmentId };
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
});
|
|
320
|
+
}
|