gsd-pi 2.78.1-dev.d8826a445 → 2.78.1-dev.eccf86e27

package/dist/resources/extensions/gsd/db/unit-dispatches.js

@@ -0,0 +1,322 @@
+// gsd-2 + Unit dispatch ledger (DB-backed coordination, Phase B)
+//
+// Records every auto-mode unit dispatch (plan-slice, run-task, summarize, …)
+// with worker_id, fencing token, status lifecycle, and retry metadata. The
+// ledger is the substrate Phase C will consume to migrate stuck-state.json
+// and paused-session.json out of the runtime/ directory.
+//
+// Codex review MEDIUM B2: partial unique index
+//   idx_unit_dispatches_active_per_unit ON unit_dispatches(unit_id)
+//   WHERE status IN ('claimed','running')
+// enforces that two workers cannot simultaneously claim the same unit.
+// recordDispatchClaim relies on the index to fail fast at INSERT time
+// rather than racing in application code.
+import { randomUUID } from "node:crypto";
+import { _getAdapter, isDbAvailable, transaction, insertAuditEvent, } from "../gsd-db.js";
+function isAlreadyActiveConstraintError(err) {
+    const code = err && typeof err === "object" && "code" in err
+        ? String(err.code ?? "")
+        : "";
+    const msg = err instanceof Error ? err.message : String(err);
+    if (/\bFOREIGN KEY\b/i.test(msg)) {
+        return false;
+    }
+    if (code === "SQLITE_CONSTRAINT" || code === "SQLITE_CONSTRAINT_UNIQUE") {
+        return true;
+    }
+    return /\bUNIQUE\b|\bconstraint failed\b/i.test(msg);
+}
+/**
+ * Insert a new dispatch row in `claimed` state. Atomic guard against
+ * double-claim (B2): the partial unique index
+ * idx_unit_dispatches_active_per_unit refuses the INSERT if any row for
+ * the same unit_id already has status IN ('claimed','running').
+ */
+export function recordDispatchClaim(input) {
+    if (!isDbAvailable()) {
+        throw new Error("recordDispatchClaim: DB unavailable");
+    }
+    const now = new Date().toISOString();
+    return transaction(() => {
+        const db = _getAdapter();
+        const lease = db.prepare(`SELECT fencing_token
+       FROM milestone_leases
+       WHERE milestone_id = :milestone_id
+         AND worker_id = :worker_id
+         AND fencing_token = :token
+         AND status = 'held'`).get({
+            ":milestone_id": input.milestoneId,
+            ":worker_id": input.workerId,
+            ":token": input.milestoneLeaseToken,
+        });
+        if (!lease) {
+            return {
+                ok: false,
+                error: "stale_lease",
+                milestoneId: input.milestoneId,
+                workerId: input.workerId,
+                milestoneLeaseToken: input.milestoneLeaseToken,
+            };
+        }
+        try {
+            const result = db.prepare(`INSERT INTO unit_dispatches (
+          trace_id, turn_id, worker_id, milestone_lease_token,
+          milestone_id, slice_id, task_id,
+          unit_type, unit_id, status, attempt_n,
+          started_at, max_attempts
+        ) VALUES (
+          :trace_id, :turn_id, :worker_id, :milestone_lease_token,
+          :milestone_id, :slice_id, :task_id,
+          :unit_type, :unit_id, 'claimed', :attempt_n,
+          :started_at, :max_attempts
+        )`).run({
+                ":trace_id": input.traceId,
+                ":turn_id": input.turnId ?? null,
+                ":worker_id": input.workerId,
+                ":milestone_lease_token": input.milestoneLeaseToken,
+                ":milestone_id": input.milestoneId,
+                ":slice_id": input.sliceId ?? null,
+                ":task_id": input.taskId ?? null,
+                ":unit_type": input.unitType,
+                ":unit_id": input.unitId,
+                ":attempt_n": input.attemptN ?? 1,
+                ":started_at": now,
+                ":max_attempts": input.maxAttempts ?? 3,
+            });
+            const id = Number(result.lastInsertRowid ?? 0);
+            insertAuditEvent({
+                eventId: randomUUID(),
+                traceId: input.traceId,
+                turnId: input.turnId ?? undefined,
+                category: "orchestration",
+                type: "dispatch-claimed",
+                ts: now,
+                payload: {
+                    dispatchId: id,
+                    unitId: input.unitId,
+                    unitType: input.unitType,
+                    workerId: input.workerId,
+                    attemptN: input.attemptN ?? 1,
+                },
+            });
+            return { ok: true, dispatchId: id };
+        }
+        catch (err) {
+            if (!isAlreadyActiveConstraintError(err))
+                throw err;
+            // Partial unique index rejected the INSERT — surface the existing
+            // active dispatch so callers can decide what to do.
+            const existing = db.prepare(`SELECT id, status, worker_id FROM unit_dispatches
+         WHERE unit_id = :unit_id AND status IN ('claimed','running')
+         ORDER BY id DESC LIMIT 1`).get({ ":unit_id": input.unitId });
+            return {
+                ok: false,
+                error: "already_active",
+                existingId: existing?.id ?? 0,
+                existingStatus: existing?.status ?? "claimed",
+                existingWorker: existing?.worker_id ?? "unknown",
+            };
+        }
+    });
+}
+/** Transition a `claimed` dispatch into `running`. */
+export function markRunning(dispatchId) {
+    if (!isDbAvailable())
+        return;
+    const db = _getAdapter();
+    db.prepare(`UPDATE unit_dispatches SET status = 'running'
+     WHERE id = :id AND status = 'claimed'`).run({ ":id": dispatchId });
+}
+/** Transition a dispatch into `completed`. */
+export function markCompleted(dispatchId, opts) {
+    if (!isDbAvailable())
+        return;
+    const now = new Date().toISOString();
+    const db = _getAdapter();
+    let changes = 0;
+    transaction(() => {
+        const result = db.prepare(`UPDATE unit_dispatches
+       SET status = 'completed', ended_at = :ended_at,
+           exit_reason = :exit_reason,
+           verification_evidence_id = :evidence_id
+       WHERE id = :id
+         AND status IN ('claimed','running')`).run({
+            ":id": dispatchId,
+            ":ended_at": now,
+            ":exit_reason": opts?.exitReason ?? null,
+            ":evidence_id": opts?.verificationEvidenceId ?? null,
+        });
+        changes =
+            typeof result.changes === "number"
+                ? result.changes
+                : 0;
+    });
+    if (changes < 1)
+        return;
+    insertAuditEvent({
+        eventId: randomUUID(),
+        traceId: dispatchId.toString(),
+        category: "orchestration",
+        type: "dispatch-completed",
+        ts: now,
+        payload: { dispatchId },
+    });
+}
+/** Transition a dispatch into `failed`, optionally scheduling a retry. */
+export function markFailed(dispatchId, opts) {
+    if (!isDbAvailable())
+        return;
+    const now = new Date();
+    const nowIso = now.toISOString();
+    const nextRunIso = opts.retryAfterMs
+        ? new Date(now.getTime() + opts.retryAfterMs).toISOString()
+        : null;
+    const db = _getAdapter();
+    let changes = 0;
+    transaction(() => {
+        const result = db.prepare(`UPDATE unit_dispatches
+       SET status = 'failed', ended_at = :ended_at,
+           error_summary = :error_summary,
+           last_error_code = :last_error_code,
+           last_error_at = :last_error_at,
+           retry_after_ms = :retry_after_ms,
+           next_run_at = :next_run_at
+       WHERE id = :id
+         AND status IN ('claimed','running')`).run({
+            ":id": dispatchId,
+            ":ended_at": nowIso,
+            ":error_summary": opts.errorSummary,
+            ":last_error_code": opts.errorCode ?? null,
+            ":last_error_at": nowIso,
+            ":retry_after_ms": opts.retryAfterMs ?? null,
+            ":next_run_at": nextRunIso,
+        });
+        changes =
+            typeof result.changes === "number"
+                ? result.changes
+                : 0;
+    });
+    if (changes < 1)
+        return;
+    insertAuditEvent({
+        eventId: randomUUID(),
+        traceId: dispatchId.toString(),
+        category: "orchestration",
+        type: "dispatch-failed",
+        ts: nowIso,
+        payload: { dispatchId, errorSummary: opts.errorSummary, retryAfterMs: opts.retryAfterMs ?? null },
+    });
+}
+/** Transition a dispatch into `stuck`. */
+export function markStuck(dispatchId, reason) {
+    if (!isDbAvailable())
+        return;
+    const now = new Date().toISOString();
+    const db = _getAdapter();
+    const result = transaction(() => {
+        return db.prepare(`UPDATE unit_dispatches
+       SET status = 'stuck', ended_at = :ended_at, exit_reason = :reason
+       WHERE id = :id
+         AND status IN ('claimed','running')`).run({ ":id": dispatchId, ":ended_at": now, ":reason": reason });
+    });
+    const changes = typeof result.changes === "number"
+        ? result.changes
+        : 0;
+    if (changes <= 0)
+        return;
+    insertAuditEvent({
+        eventId: randomUUID(),
+        traceId: dispatchId.toString(),
+        category: "orchestration",
+        type: "dispatch-stuck",
+        ts: now,
+        payload: { dispatchId, reason },
+    });
+}
+/** Transition a dispatch into `paused`. */
+export function markPaused(dispatchId) {
+    if (!isDbAvailable())
+        return;
+    const now = new Date().toISOString();
+    const db = _getAdapter();
+    db.prepare(`UPDATE unit_dispatches
+     SET status = 'paused', ended_at = :ended_at
+     WHERE id = :id AND status IN ('claimed','running')`).run({ ":id": dispatchId, ":ended_at": now });
+}
+/** Transition a dispatch into `canceled`. */
+export function markCanceled(dispatchId, reason) {
+    if (!isDbAvailable())
+        return;
+    const now = new Date().toISOString();
+    const db = _getAdapter();
+    db.prepare(`UPDATE unit_dispatches
+     SET status = 'canceled', ended_at = :ended_at, exit_reason = :reason
+     WHERE id = :id AND status IN ('pending','claimed','running')`).run({ ":id": dispatchId, ":ended_at": now, ":reason": reason });
+}
+/**
+ * Fetch the most recent N dispatches for a unit. Used by recordDispatchClaim
+ * callers to compute attempt_n and by detect-stuck.ts (B3) to consult
+ * retry budget before tripping the stuck verdict.
+ */
+export function getRecentForUnit(unitId, limit = 10) {
+    if (!isDbAvailable())
+        return [];
+    const db = _getAdapter();
+    return db.prepare(`SELECT * FROM unit_dispatches WHERE unit_id = :unit_id ORDER BY id DESC LIMIT :limit`).all({ ":unit_id": unitId, ":limit": limit });
+}
+/**
+ * Fetch the latest dispatch for a unit, regardless of status. Returns null
+ * if the unit has never been dispatched.
+ */
+export function getLatestForUnit(unitId) {
+    if (!isDbAvailable())
+        return null;
+    const db = _getAdapter();
+    const row = db.prepare(`SELECT * FROM unit_dispatches WHERE unit_id = :unit_id ORDER BY id DESC LIMIT 1`).get({ ":unit_id": unitId });
+    return row ?? null;
+}
+/**
+ * Phase C — return the most recent unit_id values for a worker, oldest-first.
+ *
+ * Drop-in replacement for the persistence side of stuck-state.json's
+ * `recentUnits` field. The auto-loop uses this to seed loopState.recentUnits
+ * on session start so the stuck-detector window survives a session restart
+ * (#3704). Returned in oldest-first order to match the in-memory window
+ * shape that detect-stuck.ts expects.
+ */
+export function getRecentUnitKeysForWorker(workerId, limit = 20) {
+    if (!isDbAvailable())
+        return [];
+    const db = _getAdapter();
+    const rows = db.prepare(`SELECT unit_id FROM unit_dispatches
+     WHERE worker_id = :worker_id
+     ORDER BY started_at DESC, id DESC
+     LIMIT :limit`).all({ ":worker_id": workerId, ":limit": limit });
+    // Reverse so callers consume oldest-first (sliding-window semantics).
+    return rows.reverse().map((r) => ({ key: r.unit_id }));
+}
+export function getRecentUnitKeysForProjectRoot(projectRootRealpath, limit = 20) {
+    if (!isDbAvailable())
+        return [];
+    const db = _getAdapter();
+    const rows = db.prepare(`SELECT ud.unit_id
+     FROM unit_dispatches ud
+     INNER JOIN workers w ON w.worker_id = ud.worker_id
+     WHERE w.project_root_realpath = :project_root_realpath
+     ORDER BY ud.started_at DESC, ud.id DESC
+     LIMIT :limit`).all({
+        ":project_root_realpath": projectRootRealpath,
+        ":limit": limit,
+    });
+    return rows.reverse().map((r) => ({ key: r.unit_id }));
+}
+/**
+ * Fetch dispatches for a milestone filtered by status. Useful for janitors
+ * + dashboards.
+ */
+export function getDispatchesByStatus(milestoneId, status) {
+    if (!isDbAvailable())
+        return [];
+    const db = _getAdapter();
+    return db.prepare(`SELECT * FROM unit_dispatches WHERE milestone_id = :mid AND status = :status ORDER BY id`).all({ ":mid": milestoneId, ":status": status });
+}

package/dist/resources/extensions/gsd/docs/COORDINATION.md

@@ -0,0 +1,42 @@
+# Auto-mode coordination is single-host
+
+The DB-backed coordination tables introduced by Phase B (`workers`,
+`milestone_leases`, `unit_dispatches`, `cancellation_requests`,
+`command_queue`) and the supporting `runtime_kv` table from Phase C all
+rely on **shared SQLite WAL on local disk**. They do not work across
+machines.
+
+## Why single-host only
+
+- SQLite WAL coordination — the locking primitives that make
+  `claimMilestoneLease`, `recordDispatchClaim`, and `claimNextCommand`
+  atomic — is local-disk only. Network filesystems (NFS, SMB, S3FS) and
+  fuse mounts break the lock semantics that the WAL relies on.
+- Heartbeat TTL (`workers.last_heartbeat_at`) compares timestamps written
+  with SQLite wall-clock time (`datetime('now')`). Across machines without
+  wall-clock synchronization (for example NTP/chrony), TTL filtering can
+  produce phantom-active or premature-crashed verdicts. Monotonic clocks
+  are not used for these comparisons.
+- Fencing tokens (`milestone_leases.fencing_token`) are monotonically
+  ordered by SQL within a single transaction. Cross-host races could
+  produce duplicate tokens if two SQLite processes opened the same DB
+  on a network mount.
+
+## What does work
+
+- Multiple `gsd auto` worker processes on the **same machine**, sharing
+  the project's SQLite DB via WAL. The lease check refuses concurrent
+  claims on the same milestone; the dispatch ledger's partial unique
+  index refuses double-claims of the same unit.
+- A single `gsd auto` worker plus arbitrary read-only consumers
+  (dashboards, doctors) on the same machine.
+- Worktree-based parallelism on the same machine, where each worker
+  holds a different milestone lease.
+
+## Multi-host alternatives
+
+If you need to coordinate `gsd auto` workers across machines, you need
+a real coordinator: Postgres for the ledger + a leader-election service
+(etcd, Consul) for the leases. That's out of scope for these phases.
+The schema and module shapes here would need a non-trivial backend
+swap before they could ride on top of either.

package/dist/resources/extensions/gsd/doctor-proactive.js

@@ -24,6 +24,7 @@ import { resolveMilestoneIntegrationBranch } from "./git-service.js";
 import { nativeIsRepo, nativeHasChanges, nativeLastCommitEpoch, nativeGetCurrentBranch, nativeAddTracked, nativeCommit } from "./native-git-bridge.js";
 import { loadEffectiveGSDPreferences } from "./preferences.js";
 import { runEnvironmentChecks } from "./doctor-environment.js";
+import { ensureDbOpen } from "./bootstrap/dynamic-tools.js";
 /** In-memory health history for the current auto-mode session. */
 let healthHistory = [];
 /** Count of consecutive units with unresolved errors. */
@@ -159,6 +160,9 @@ export async function preDispatchHealthGate(basePath) {
     // If a stale lock exists, the crash recovery path should handle it,
     // not a new dispatch. This prevents double-dispatch after crashes.
     try {
+        if (existsSync(join(gsdRoot(basePath), "gsd.db"))) {
+            await ensureDbOpen(basePath);
+        }
         const lock = readCrashLock(basePath);
         if (lock && !isLockProcessAlive(lock)) {
             // Auto-clear it since we're about to dispatch anyway

package/dist/resources/extensions/gsd/doctor-runtime-checks.js

@@ -6,6 +6,8 @@ import { deriveState, isGhostMilestone, isReusableGhostMilestone } from "./state
 import { saveFile } from "./files.js";
 import { nativeIsRepo, nativeForEachRef, nativeUpdateRef } from "./native-git-bridge.js";
 import { readCrashLock, isLockProcessAlive, clearLock } from "./crash-recovery.js";
+import { getActiveAutoWorkers } from "./db/auto-workers.js";
+import { normalizeRealPath } from "./paths.js";
 import { ensureGitignore, isGsdGitignored } from "./gitignore.js";
 import { readAllSessionStatuses, isSessionStale, removeSessionStatus } from "./session-status-io.js";
 import { recoverFailedMigration } from "./migrate-external.js";
@@ -26,6 +28,9 @@ function hasAssessmentVerdict(basePath, mid, sid) {
 export async function checkRuntimeHealth(basePath, issues, fixesApplied, shouldFix) {
     const root = gsdRoot(basePath);
     // ── Stale crash lock ──────────────────────────────────────────────────
+    // Phase C pt 2: the lock state lives in the workers + unit_dispatches
+    // tables now, not auto.lock. readCrashLock synthesizes a LockData from
+    // the DB; isLockProcessAlive is a pure OS PID check.
     try {
         const lock = readCrashLock(basePath);
         if (lock) {
@@ -36,13 +41,13 @@ export async function checkRuntimeHealth(basePath, issues, fixesApplied, shouldF
                     code: "stale_crash_lock",
                     scope: "project",
                     unitId: "project",
-                    message: `Stale auto.lock from PID ${lock.pid} (started ${lock.startedAt}, was executing ${lock.unitType} ${lock.unitId}) — process is no longer running`,
-                    file: ".gsd/auto.lock",
+                    message: `Stale auto-mode worker (PID ${lock.pid}, started ${lock.startedAt}, was executing ${lock.unitType} ${lock.unitId}) — process is no longer running`,
+                    file: "<workers table>",
                     fixable: true,
                 });
                 if (shouldFix("stale_crash_lock")) {
                     clearLock(basePath);
-                    fixesApplied.push("cleared stale auto.lock");
+                    fixesApplied.push("cleared stale auto-mode worker state");
                 }
             }
         }
@@ -60,9 +65,20 @@ export async function checkRuntimeHealth(basePath, issues, fixesApplied, shouldF
         if (existsSync(lockDir)) {
             const statRes = statSync(lockDir);
             if (statRes.isDirectory()) {
-                // Check if any live process actually holds this lock
-                const lock = readCrashLock(basePath);
-                const lockHolderAlive = lock ? isLockProcessAlive(lock) : false;
+                // Phase C pt 2: "any live process holds the lock?" check now means
+                // "is any worker registered with status='active' AND a fresh
+                // heartbeat for this project?" — readCrashLock returns null for
+                // healthy live workers (it surfaces stale ones only), so we must
+                // consult getActiveAutoWorkers directly.
+                const projectRoot = normalizeRealPath(basePath);
+                const activeWorkers = getActiveAutoWorkers().filter((w) => w.project_root_realpath === projectRoot && isLockProcessAlive({
+                    pid: w.pid,
+                    startedAt: w.started_at,
+                    unitType: "starting",
+                    unitId: "bootstrap",
+                    unitStartedAt: w.started_at,
+                }));
+                const lockHolderAlive = activeWorkers.length > 0;
                 if (!lockHolderAlive) {
                     issues.push({
                         severity: "error",

package/dist/resources/extensions/gsd/doctor.js

@@ -2,8 +2,8 @@ import { existsSync, mkdirSync, lstatSync, readdirSync, readFileSync } from "nod
 import { join } from "node:path";
 import { loadFile, parseSummary, saveFile, parseTaskPlanMustHaves, countMustHavesMentionedInSummary } from "./files.js";
 import { parseRoadmap as parseLegacyRoadmap, parsePlan as parseLegacyPlan } from "./parsers-legacy.js";
-import { isDbAvailable, getMilestoneSlices, getSliceTasks } from "./gsd-db.js";
-import { resolveMilestoneFile, resolveMilestonePath, resolveSliceFile, resolveSlicePath, resolveTaskFile, resolveTasksDir, milestonesDir, gsdRoot, relMilestoneFile, relSliceFile, relTaskFile, relSlicePath, relGsdRootFile, resolveGsdRootFile, relMilestonePath } from "./paths.js";
+import { isDbAvailable, openDatabase, getMilestoneSlices, getSliceTasks } from "./gsd-db.js";
+import { resolveMilestoneFile, resolveMilestonePath, resolveSliceFile, resolveSlicePath, resolveTaskFile, resolveTasksDir, milestonesDir, gsdRoot, relMilestoneFile, relSliceFile, relTaskFile, relSlicePath, relGsdRootFile, resolveGsdRootFile, relMilestonePath, resolveGsdPathContract } from "./paths.js";
 import { deriveState, isMilestoneComplete } from "./state.js";
 import { invalidateAllCaches } from "./cache.js";
 import { loadEffectiveGSDPreferences } from "./preferences.js";
@@ -309,6 +309,16 @@ export async function runGSDDoctor(basePath, options) {
     const fix = options?.fix === true;
     const dryRun = options?.dryRun === true;
     const fixLevel = options?.fixLevel ?? "all";
+    // CLI doctor can run before any tool handler has opened the DB. Runtime
+    // health checks need the existing project DB to surface DB-backed crash
+    // locks, paused sessions, and coordination rows.
+    const dbPath = resolveGsdPathContract(basePath).projectDb;
+    if (existsSync(dbPath)) {
+        try {
+            openDatabase(dbPath);
+        }
+        catch { /* surfaced later as db_unavailable */ }
+    }
     // Issue codes that represent completion state transitions — creating summary
     // stubs, marking slices/milestones done in the roadmap. These belong to the
     // dispatch lifecycle (complete-slice, complete-milestone units), not to