@stravica-ai/digital-operator-pa 0.1.21 → 0.1.23

package/dist/engineering-pa-cli/src/sync/run.mjs

@@ -85,6 +85,7 @@ import {
   push,
   rebaseAbort,
   rebaseContinue,
+  revParseSha,
   statusPorcelain,
 } from './git.mjs';
 import { acquire } from './lock.mjs';
@@ -94,6 +95,11 @@ import {
 } from './regenerated-artifacts.mjs';
 import { computeCommitScope, filterInScopePaths, renderScopeSection } from './registry-scope.mjs';
 import { composeBody } from './runs-digest.mjs';
+import {
+  attemptSelfHeal,
+  drainPending,
+  finaliseDrainOnPushSuccess,
+} from './self-heal.mjs';
 
 function log(line, { stderr = process.stderr } = {}) {
   stderr.write(`[estate-sync] ${line}\n`);
@@ -126,6 +132,57 @@ async function writeAudit(runsLog, record, stderr) {
   }
 }
 
+// Alert-level ordering: needs-human > error > warn. `needs-human` outranks
+// `error` because a divergence detected at drain time can never be fixed
+// by more attempts — it needs an operator to reconcile the two upstream
+// resolutions, so its status must NOT get de-escalated behind a
+// mechanical attempt-count.
+const ALERT_RANK = { warn: 0, error: 1, 'needs-human': 2 };
+function worstAlertLevel(queued) {
+  let best = 'warn';
+  for (const q of queued) {
+    if ((ALERT_RANK[q.alertLevel] ?? -1) > (ALERT_RANK[best] ?? -1)) best = q.alertLevel;
+  }
+  return best;
+}
+
+// Compact per-queued summary safe to serialise to the audit record.
+function summariseQueued(q) {
+  return {
+    id: q.id,
+    originalPath: q.originalPath,
+    attemptCount: q.attemptCount,
+    alertLevel: q.alertLevel,
+    upstreamBlobShaAtQueue: q.upstreamBlobShaAtQueue,
+    conflictingUpstreamCommitSha: q.conflictingUpstreamCommitSha,
+    pendingPath: q.pendingPath,
+  };
+}
+
+function summariseDrained(drained) {
+  return drained.map((d) => ({
+    id: d.id,
+    originalPath: d.originalPath,
+    action: d.action,
+    attemptCount: d.attemptCount,
+    ...(d.alertLevel ? { alertLevel: d.alertLevel } : {}),
+    ...(d.upstreamBlobShaAtQueue !== undefined ? { upstreamBlobShaAtQueue: d.upstreamBlobShaAtQueue } : {}),
+    ...(d.upstreamBlobShaNow !== undefined ? { upstreamBlobShaNow: d.upstreamBlobShaNow } : {}),
+  }));
+}
+
+// Human-readable alert reason for the audit `reason` field. Names every
+// queued path with attempt count + alert level + upstream blob at queue
+// time, so a librarian-alerts email or grep of runs.jsonl shows the full
+// alert payload without needing to open the sidecar.
+function renderSelfHealReason(queued) {
+  const parts = queued.map((q) => {
+    const blob = q.upstreamBlobShaAtQueue ? q.upstreamBlobShaAtQueue.slice(0, 8) : 'absent';
+    return `${q.originalPath} attempt=${q.attemptCount} alert=${q.alertLevel} upstreamBlob=${blob}`;
+  });
+  return `estate-sync self-heal on rebase conflict; queued: ${parts.join('; ')}`;
+}
+
 export async function runSync({
   repoRoot,
   logDir,
@@ -190,6 +247,46 @@ export async function runSync({
       }
     }
 
+    // Step 2.5 — DRAIN the self-heal pending queue (R2, w-2026-08-12-004).
+    // Runs AFTER fetch so the upstream-divergence check inside drainPending
+    // compares against fresh origin refs (a stale check would silently
+    // let us clobber a resolution another writer pushed since we queued).
+    // Skipped in dry-run for the same reason git commits are.
+    //
+    // Pending entries whose upstream state is unchanged since queue time
+    // are copied back into the working tree, so step 3's status sees them
+    // as dirty and step 6 stages them normally. Divergence cases stay on
+    // disk with alertLevel='needs-human' and DO NOT touch the workspace
+    // (the write would clobber the other writer's resolution — this is
+    // the same down-path clobber class the whole train exists to kill).
+    // Already-delivered cases are cleaned up in place.
+    //
+    // Branch resolution: currentBranch is called ahead of drain so
+    // upstream refs (`origin/<branch>`) are addressable inside the drain.
+    // On a detached HEAD this returns 'HEAD' — drain still runs but the
+    // upstream check will fail (origin/HEAD is not a real ref), which
+    // routes every pending entry to divergence-held. That's the safe
+    // outcome — detached HEAD is not the state estate-sync should be
+    // draining into.
+    const branchForDrain = await currentBranch(runner, { cwd: repoRoot });
+    const pendingDir = join(repoRoot, '.cache/estate-sync/pending');
+    let drainedEntries = [];
+    let drainWarnings = [];
+    if (!dryRun && upfrontFetch.code === 0) {
+      const drainResult = await drainPending({
+        runner,
+        repoRoot,
+        pendingDir,
+        branch: branchForDrain,
+        stderr,
+      });
+      drainedEntries = drainResult.drained;
+      drainWarnings = drainResult.warnings || [];
+      for (const w of drainWarnings) {
+        log(`drain warning: ${w.sidecar}: ${w.reason}`, { stderr });
+      }
+    }
+
     // Step 3 — examine working tree and upstream relation.
     const status = await statusPorcelain(runner, { cwd: repoRoot });
     if (status.code !== 0) {
@@ -211,7 +308,8 @@ export async function runSync({
     const behindCount = Number.isFinite(behind) ? behind : 0;
     log(`tree dirty=${dirty}, ahead=${aheadCount}, behind=${behindCount}`, { stderr });
 
-    const branch = await currentBranch(runner, { cwd: repoRoot });
+    // Branch was resolved earlier for the drain step — reuse it here.
+    const branch = branchForDrain;
 
     // Step 4 — noop short-circuit.
     if (!dirty && aheadCount === 0 && behindCount === 0) {
@@ -226,6 +324,11 @@ export async function runSync({
           ahead: aheadCount,
           behind: behindCount,
           branch,
+          // Drain may have run and reported already-delivered / divergence-
+          // held entries even on an otherwise-noop tick; surface them
+          // even though staged-for-retry cases wouldn't reach here (they
+          // would have made the tree dirty).
+          ...(drainedEntries.length > 0 ? { drained: summariseDrained(drainedEntries) } : {}),
         }, stderr);
       }
       return 0;
@@ -269,7 +372,15 @@ export async function runSync({
     // replays a clean local commit on top of origin's history. This is
     // the same shape a human follows when integrating: stage, commit,
     // then rebase onto upstream.
+    //
+    // ourCommitSha (R2, w-2026-08-12-004): the sha of estate-sync's
+    // commit is recorded RIGHT AFTER the commit lands. It's the load-
+    // bearing guard for the self-heal reset — the reset only ever runs
+    // when HEAD is still exactly this sha at reset time. If anything
+    // else committed between (a human commit, a hook) HEAD^..HEAD is no
+    // longer just our commit and the guard refuses to reset.
     let didCommit = false;
+    let ourCommitSha = null;
     if (dirty) {
       let addRes;
       let stagedCount = 0;
@@ -329,6 +440,20 @@ export async function runSync({
           return 3;
         }
         didCommit = true;
+        // Record the sha immediately — this is the guard the self-heal
+        // reset checks against before dropping. Any subsequent operation
+        // that moves HEAD (a hook, a concurrent write) will fail the
+        // guard and self-heal will refuse to reset, preserving the
+        // moved-on state for a human to inspect.
+        const shaRes = await revParseSha(runner, { cwd: repoRoot, ref: 'HEAD' });
+        if (shaRes.code === 0 && shaRes.sha) {
+          ourCommitSha = shaRes.sha;
+        } else {
+          log(
+            `warning: could not resolve HEAD sha after commit (${shaRes.stderr.trim()}) — self-heal guard cannot be checked; a subsequent rebase conflict will be handled as fail-rebase (no reset)`,
+            { stderr },
+          );
+        }
         log('committed', { stderr });
       } else {
         log('nothing staged after add — skipping commit', { stderr });
@@ -347,6 +472,7 @@ export async function runSync({
     // either no-ops (if local refs already match) or fails spuriously.
     let autoResolvedPaths = [];
     let autostashConflict = false;
+    let selfHealedQueued = [];
     if (upfrontFetch.code === 0) {
       const prRes = await pullRebase(runner, { cwd: repoRoot, branch });
       autostashConflict = detectAutostashConflict(prRes, stderr) || autostashConflict;
@@ -358,8 +484,12 @@ export async function runSync({
         // conflict just means two hosts pushed regenerations in the same
         // window. Auto-resolve by preferring origin's version (see the
         // rebase-inversion note on `checkoutOursDuringRebase`) and
-        // continue. If ANY conflict is outside the allowlist, we still
-        // abort and surface fail-rebase — real conflicts need a human.
+        // continue. If ANY conflict is outside the allowlist, hand off
+        // to self-heal (R2, w-2026-08-12-004): queue our version, drop
+        // our commit under guard, restore fast-forwardability, alert.
+        // If self-heal itself declines (queue-write failure, guard
+        // failure) we fall through to the classic fail-rebase branch
+        // and stay wedged loudly.
         const resolved = await tryAutoResolveRebase({
           runner,
           repoRoot,
@@ -371,6 +501,67 @@ export async function runSync({
             `auto-resolved rebase conflict on ${resolved.paths.length} regenerated artifact(s): ${resolved.paths.join(', ')}`,
             { stderr },
           );
+        } else if (resolved.disallowed && didCommit && ourCommitSha) {
+          const heal = await attemptSelfHeal({
+            runner,
+            repoRoot,
+            pendingDir,
+            branch,
+            disallowedPaths: resolved.disallowed,
+            ourCommitSha,
+            firstSeenTs: tsIso,
+            stderr,
+          });
+          if (heal.ok) {
+            selfHealedQueued = heal.queued;
+            log(
+              `self-heal: queued ${heal.queued.length} path(s) for retry, dropped local commit ${ourCommitSha.slice(0, 8)}, workspace fast-forwardable again`,
+              { stderr },
+            );
+            // Emit a fail-* audit record so librarian-alerts (if wired
+            // to estate-sync's runs.jsonl) picks this up as a failure
+            // requiring human attention — but exit code stays 0 because
+            // the workspace itself IS healthy (fast-forwardable, not
+            // wedged). This is the whole point: cron process healthy,
+            // human-visible alert loud, next tick's drain will retry.
+            const maxAlert = worstAlertLevel(heal.queued);
+            const status =
+              maxAlert === 'error' ? 'fail-rebase-stuck'
+              : maxAlert === 'needs-human' ? 'fail-rebase-needs-human'
+              : 'fail-rebase-queued';
+            await writeAudit(runsLog, {
+              ts: tsIso,
+              status,
+              exitCode: 0,
+              durationSec: Math.round((Date.now() - startedAtMs) / 1000),
+              reason: renderSelfHealReason(heal.queued),
+              committed: false,
+              selfHealed: true,
+              droppedCommit: ourCommitSha,
+              queued: heal.queued.map(summariseQueued),
+              conflictedPaths: resolved.disallowed,
+              branch,
+              ...(drainedEntries.length > 0 ? { drained: summariseDrained(drainedEntries) } : {}),
+            }, stderr);
+            return 0;
+          }
+          log(
+            `self-heal DECLINED (stage=${heal.stage}): ${heal.reason} — falling back to fail-rebase, workspace remains wedged (loud)`,
+            { stderr },
+          );
+          // Fall through to fail-rebase.
+          await rebaseAbort(runner, { cwd: repoRoot });
+          await writeAudit(runsLog, {
+            ts: tsIso,
+            status: 'fail-rebase',
+            exitCode: 2,
+            durationSec: Math.round((Date.now() - startedAtMs) / 1000),
+            reason: `git pull --rebase failed — self-heal declined (${heal.stage}): ${heal.reason}; ${resolved.reason}`,
+            committed: didCommit,
+            selfHealDeclined: { stage: heal.stage, reason: heal.reason },
+            ...(resolved.disallowed ? { conflictedPaths: resolved.disallowed } : {}),
+          }, stderr);
+          return 2;
         } else {
           log(`aborting rebase: ${resolved.reason}`, { stderr });
           // Defensive abort. If no rebase is in progress (auto-resolve
@@ -407,6 +598,14 @@ export async function runSync({
     const needsPush = Number.isFinite(aheadAfter) && aheadAfter > 0;
     if (!needsPush) {
       log('nothing to push (up-to-date with origin)', { stderr });
+      // If drain staged pending content into the workspace but nothing
+      // needed pushing (e.g. drain wrote content matching HEAD after a
+      // race), still finalise so we don't hold pending files that are
+      // already delivered. finaliseDrainOnPushSuccess compares HEAD's
+      // bytes to the pending bytes before removing.
+      const finaliseIdle = await finaliseDrainOnPushSuccess({
+        runner, repoRoot, pendingDir, drained: drainedEntries, stderr,
+      });
       await writeAudit(runsLog, {
         ts: tsIso,
         status: 'ok',
@@ -416,6 +615,8 @@ export async function runSync({
         pushed: false,
         branch,
         ...autoResolveFields,
+        ...(drainedEntries.length > 0 ? { drained: summariseDrained(drainedEntries) } : {}),
+        ...(finaliseIdle.finalised.length > 0 ? { drainFinalised: finaliseIdle.finalised } : {}),
       }, stderr);
       return 0;
     }
@@ -423,6 +624,9 @@ export async function runSync({
     const p1 = await push(runner, { cwd: repoRoot });
     if (p1.code === 0) {
       log('pushed', { stderr });
+      const finalise1 = await finaliseDrainOnPushSuccess({
+        runner, repoRoot, pendingDir, drained: drainedEntries, stderr,
+      });
       await writeAudit(runsLog, {
         ts: tsIso,
         status: 'ok',
@@ -432,6 +636,8 @@ export async function runSync({
         pushed: true,
         branch,
         ...autoResolveFields,
+        ...(drainedEntries.length > 0 ? { drained: summariseDrained(drainedEntries) } : {}),
+        ...(finalise1.finalised.length > 0 ? { drainFinalised: finalise1.finalised } : {}),
       }, stderr);
       return 0;
     }
@@ -454,8 +660,10 @@ export async function runSync({
     if (pr2.code !== 0) {
       log(`git pull --rebase (for push-retry) failed: ${pr2.stderr.trim()}`, { stderr });
       // Symmetric with step 7: if the push-retry rebase conflicts only
-      // on regenerated artifacts, auto-resolve and continue. Otherwise
-      // abort as before.
+      // on regenerated artifacts, auto-resolve and continue. If any path
+      // is outside the allowlist, hand off to self-heal (R2, w-2026-08-
+      // 12-004) exactly as the primary path does. Otherwise fall through
+      // to the classic fail-rebase branch.
       const resolved2 = await tryAutoResolveRebase({
         runner,
         repoRoot,
@@ -467,6 +675,60 @@ export async function runSync({
           `auto-resolved push-retry rebase on ${resolved2.paths.length} regenerated artifact(s): ${resolved2.paths.join(', ')}`,
           { stderr },
         );
+      } else if (resolved2.disallowed && didCommit && ourCommitSha) {
+        const heal2 = await attemptSelfHeal({
+          runner,
+          repoRoot,
+          pendingDir,
+          branch,
+          disallowedPaths: resolved2.disallowed,
+          ourCommitSha,
+          firstSeenTs: tsIso,
+          stderr,
+        });
+        if (heal2.ok) {
+          selfHealedQueued = [...selfHealedQueued, ...heal2.queued];
+          log(
+            `self-heal (push-retry): queued ${heal2.queued.length} path(s) for retry, dropped local commit ${ourCommitSha.slice(0, 8)}, workspace fast-forwardable again`,
+            { stderr },
+          );
+          const maxAlert = worstAlertLevel(heal2.queued);
+          const status =
+            maxAlert === 'error' ? 'fail-rebase-stuck'
+            : maxAlert === 'needs-human' ? 'fail-rebase-needs-human'
+            : 'fail-rebase-queued';
+          await writeAudit(runsLog, {
+            ts: tsIso,
+            status,
+            exitCode: 0,
+            durationSec: Math.round((Date.now() - startedAtMs) / 1000),
+            reason: renderSelfHealReason(heal2.queued),
+            committed: false,
+            selfHealed: true,
+            pushRetryPath: true,
+            droppedCommit: ourCommitSha,
+            queued: heal2.queued.map(summariseQueued),
+            conflictedPaths: resolved2.disallowed,
+            branch,
+            ...(drainedEntries.length > 0 ? { drained: summariseDrained(drainedEntries) } : {}),
+          }, stderr);
+          return 0;
+        }
+        log(
+          `self-heal (push-retry) DECLINED (stage=${heal2.stage}): ${heal2.reason} — falling back to fail-rebase`,
+          { stderr },
+        );
+        await rebaseAbort(runner, { cwd: repoRoot });
+        await writeAudit(runsLog, {
+          ts: tsIso,
+          status: 'fail-rebase',
+          exitCode: 2,
+          durationSec: Math.round((Date.now() - startedAtMs) / 1000),
+          reason: `push rejected; follow-up rebase self-heal declined (${heal2.stage}): ${heal2.reason}; ${resolved2.reason}`,
+          selfHealDeclined: { stage: heal2.stage, reason: heal2.reason },
+          ...(resolved2.disallowed ? { conflictedPaths: resolved2.disallowed } : {}),
+        }, stderr);
+        return 2;
       } else {
         log(`aborting push-retry rebase: ${resolved2.reason}`, { stderr });
         await rebaseAbort(runner, { cwd: repoRoot });
@@ -501,6 +763,9 @@ export async function runSync({
         : {}),
       ...(autostashConflict ? { autostashConflict: true } : {}),
     };
+    const finalise2 = await finaliseDrainOnPushSuccess({
+      runner, repoRoot, pendingDir, drained: drainedEntries, stderr,
+    });
     await writeAudit(runsLog, {
       ts: tsIso,
       status: 'ok',
@@ -511,6 +776,8 @@ export async function runSync({
       retriedAfterRebase: true,
       branch,
       ...autoResolveFieldsFinal,
+      ...(drainedEntries.length > 0 ? { drained: summariseDrained(drainedEntries) } : {}),
+      ...(finalise2.finalised.length > 0 ? { drainFinalised: finalise2.finalised } : {}),
     }, stderr);
     return 0;
   } finally {