dahrk-node 0.1.13 → 0.1.14

package/dist/main.js

@@ -2,10 +2,10 @@
 
 // src/main.ts
 import { execFileSync as execFileSync9 } from "child_process";
-import { existsSync as existsSync13, realpathSync as realpathSync5 } from "fs";
+import { existsSync as existsSync14, realpathSync as realpathSync5 } from "fs";
 import { randomUUID as randomUUID3 } from "crypto";
 import { homedir as homedir7, platform as osPlatform4 } from "os";
-import { basename as basename2, join as join16 } from "path";
+import { basename as basename2, join as join17 } from "path";
 import { pathToFileURL } from "url";
 
 // ../../packages/edge/src/ws-client.ts
@@ -1590,6 +1590,7 @@ function createGitService(opts = {}) {
     gitUrl: spec.gitUrl,
     repo: spec.repo ?? spec.repoId,
     baseBranch: spec.baseBranch,
+    branch: sanitizeBranchName(spec.branch ?? `dahrk/${spec.runId}`),
     worktreePath,
     scratchPath: join3(worktreePath, ".skakel", "scratch")
   });
@@ -1730,6 +1731,31 @@ function createGitService(opts = {}) {
       }
       return { headSha, pushed: true, nothingToCommit: !dirty, wipRef };
     },
+    async reconcileInterrupted(ref, opts2) {
+      const { worktreePath } = ref;
+      if (!existsSync(worktreePath) || !gitOk2(worktreePath, ["rev-parse", "--git-dir"])) {
+        throw new Error(`worktree missing for reconcile: ${worktreePath}`);
+      }
+      const headSha = git(worktreePath, ["rev-parse", "HEAD"]).trim();
+      const wipRef = sanitizeBranchName(opts2.branch);
+      const { headSha: tailSha, dirty } = commitPending(worktreePath, opts2.message);
+      if (!dirty) return { dirty: false, headSha, pushed: false };
+      git(worktreePath, ["branch", "--force", wipRef, tailSha]);
+      let pushed = false;
+      const auth = opts2.credentialToken ? setupAuth(opts2.credentialToken) : void 0;
+      const remote = opts2.credentialToken ? withTokenUser(ref.gitUrl) : ref.gitUrl;
+      try {
+        git(worktreePath, ["push", "--force", remote, `${tailSha}:refs/heads/${wipRef}`], netEnv(auth?.env));
+        pushed = true;
+      } catch (e) {
+        log.warn(`could not push the preserved tail to ${wipRef}: ${e.message}`);
+      } finally {
+        auth?.cleanup();
+      }
+      git(worktreePath, ["reset", "--hard", headSha]);
+      git(worktreePath, ["clean", "-fd", "--exclude", SCRATCH_DIR]);
+      return { dirty: true, headSha, tailSha, wipRef, pushed };
+    },
     async openPrAmbient(ref, opts2) {
       const ownerRepo = parseOwnerRepo(ref.gitUrl);
       if (!ownerRepo) return { prError: `cannot derive owner/repo from ${ref.gitUrl}` };
@@ -2110,6 +2136,75 @@ function collectHealth(inputs) {
   };
 }
 
+// ../../packages/edge/src/job-ledger.ts
+import { chmodSync, existsSync as existsSync5, mkdirSync as mkdirSync5, readFileSync as readFileSync4, renameSync, unlinkSync, writeFileSync as writeFileSync5 } from "fs";
+import { dirname as dirname4, join as join8 } from "path";
+var FILE_MODE = 384;
+var DIR_MODE = 448;
+function nullJobLedger() {
+  return { all: () => [], stale: () => [], upsert: () => {
+  }, remove: () => {
+  }, clear: () => {
+  } };
+}
+var isEntry = (v) => {
+  if (typeof v !== "object" || v === null) return false;
+  const e = v;
+  return typeof e["jobId"] === "string" && !!e["jobId"] && typeof e["runId"] === "string" && (e["kind"] === "stage" || e["kind"] === "push") && typeof e["startedAt"] === "number" && typeof e["nodePid"] === "number";
+};
+function fileJobLedger(file, warn = console.warn) {
+  const read = () => {
+    if (!existsSync5(file)) return [];
+    try {
+      const parsed = JSON.parse(readFileSync4(file, "utf8"));
+      if (!Array.isArray(parsed)) return [];
+      return parsed.filter(isEntry);
+    } catch {
+      return [];
+    }
+  };
+  const write = (entries) => {
+    const tmp = `${file}.${process.pid}.tmp`;
+    try {
+      mkdirSync5(dirname4(file), { recursive: true, mode: DIR_MODE });
+      writeFileSync5(tmp, `${JSON.stringify(entries, null, 2)}
+`, { mode: FILE_MODE });
+      chmodSync(tmp, FILE_MODE);
+      renameSync(tmp, file);
+    } catch (e) {
+      warn(`could not persist the job ledger to ${file}: ${e.message}`);
+      try {
+        if (existsSync5(tmp)) unlinkSync(tmp);
+      } catch {
+      }
+    }
+  };
+  return {
+    all: read,
+    stale: (currentPid) => read().filter((e) => e.nodePid !== currentPid),
+    upsert(entry) {
+      write([...read().filter((e) => e.jobId !== entry.jobId), entry]);
+    },
+    remove(jobId) {
+      const next = read().filter((e) => e.jobId !== jobId);
+      write(next);
+    },
+    clear() {
+      write([]);
+    }
+  };
+}
+function announceableJobs(entries) {
+  const out = [];
+  for (const e of entries) {
+    if (e.payloadVersion) out.push({ jobId: e.jobId, payloadVersion: e.payloadVersion });
+  }
+  return out;
+}
+function jobLedgerFile(stateDir2) {
+  return join8(stateDir2, "jobs.json");
+}
+
 // ../../packages/edge/src/log-shipper.ts
 import { shouldShip } from "@dahrk/contracts";
 var SHIP_BUFFER_MAX = 500;
@@ -2210,8 +2305,8 @@ function ceilingFromEnv(env) {
 }
 
 // ../../packages/edge/src/logger.ts
-import { closeSync, existsSync as existsSync5, mkdirSync as mkdirSync5, openSync, renameSync, rmSync as rmSync4, statSync as statSync2, writeSync } from "fs";
-import { join as join8 } from "path";
+import { closeSync, existsSync as existsSync6, mkdirSync as mkdirSync6, openSync, renameSync as renameSync2, rmSync as rmSync4, statSync as statSync2, writeSync } from "fs";
+import { join as join9 } from "path";
 import pino from "pino";
 
 // ../../packages/edge/src/redact.ts
@@ -2307,7 +2402,7 @@ var RotatingFile = class {
   open() {
     try {
       this.fd = openSync(this.path, "a");
-      this.size = existsSync5(this.path) ? statSync2(this.path).size : 0;
+      this.size = existsSync6(this.path) ? statSync2(this.path).size : 0;
     } catch (e) {
       this.disable(e);
     }
@@ -2335,12 +2430,12 @@ var RotatingFile = class {
       closeSync(this.fd);
       this.fd = void 0;
       const oldest = `${this.path}.${this.maxFiles}`;
-      if (existsSync5(oldest)) rmSync4(oldest, { force: true });
+      if (existsSync6(oldest)) rmSync4(oldest, { force: true });
       for (let i = this.maxFiles - 1; i >= 1; i--) {
         const from = `${this.path}.${i}`;
-        if (existsSync5(from)) renameSync(from, `${this.path}.${i + 1}`);
+        if (existsSync6(from)) renameSync2(from, `${this.path}.${i + 1}`);
       }
-      if (existsSync5(this.path)) renameSync(this.path, `${this.path}.1`);
+      if (existsSync6(this.path)) renameSync2(this.path, `${this.path}.1`);
       this.open();
     } catch (e) {
       this.disable(e);
@@ -2400,8 +2495,8 @@ function createNodeLogger(opts = {}) {
   if (level !== "silent") streams.push({ level, stream: humanSink });
   if (opts.dir && fileLevel !== "silent") {
     try {
-      mkdirSync5(opts.dir, { recursive: true, mode: 448 });
-      const file = new RotatingFile(join8(opts.dir, "node.jsonl"), opts.maxBytes ?? 10 * 1024 * 1024, opts.maxFiles ?? 5);
+      mkdirSync6(opts.dir, { recursive: true, mode: 448 });
+      const file = new RotatingFile(join9(opts.dir, "node.jsonl"), opts.maxBytes ?? 10 * 1024 * 1024, opts.maxFiles ?? 5);
       streams.push({ level: fileLevel, stream: file });
     } catch (e) {
       process.stderr.write(`dahrk: file logging disabled (${e.message})
@@ -2459,9 +2554,9 @@ function denyToolRule(tool3) {
 // ../../packages/edge/src/stage-runner.ts
 import { execFileSync as execFileSync5 } from "child_process";
 import { createHash as createHash3 } from "crypto";
-import { mkdirSync as mkdirSync6, readdirSync as readdirSync3, readFileSync as readFileSync4, rmSync as rmSync5, writeFileSync as writeFileSync5 } from "fs";
+import { mkdirSync as mkdirSync7, readdirSync as readdirSync3, readFileSync as readFileSync5, rmSync as rmSync5, writeFileSync as writeFileSync6 } from "fs";
 import { tmpdir as tmpdir4 } from "os";
-import { isAbsolute as isAbsolute3, join as join10, relative as relative3, resolve as resolve2, sep as sep3 } from "path";
+import { isAbsolute as isAbsolute3, join as join11, relative as relative3, resolve as resolve2, sep as sep3 } from "path";
 import { attachedDocBasename as attachedDocBasename2 } from "@dahrk/contracts";
 
 // ../../packages/edge/src/builtins.ts
@@ -2469,9 +2564,9 @@ import { execFileSync as execFileSync4 } from "child_process";
 
 // ../../packages/edge/src/fs-roots.ts
 import { execFileSync as execFileSync3 } from "child_process";
-import { existsSync as existsSync6, realpathSync as realpathSync2 } from "fs";
+import { existsSync as existsSync7, realpathSync as realpathSync2 } from "fs";
 import { homedir as homedir3, tmpdir as tmpdir3 } from "os";
-import { dirname as dirname4, isAbsolute as isAbsolute2, join as join9, relative as relative2, resolve, sep as sep2 } from "path";
+import { dirname as dirname5, isAbsolute as isAbsolute2, join as join10, relative as relative2, resolve, sep as sep2 } from "path";
 function isUnder(root, target) {
   const rel = relative2(resolve(root), resolve(target));
   return rel === "" || !rel.startsWith(`..${sep2}`) && rel !== ".." && !isAbsolute2(rel);
@@ -2484,16 +2579,16 @@ function expandPath(raw, cwd) {
 function realish(p) {
   let head = p;
   const tail = [];
-  while (head !== dirname4(head)) {
-    if (existsSync6(head)) {
+  while (head !== dirname5(head)) {
+    if (existsSync7(head)) {
       try {
-        return join9(realpathSync2.native(head), ...tail.reverse());
+        return join10(realpathSync2.native(head), ...tail.reverse());
       } catch {
         return p;
       }
     }
-    tail.push(head.slice(dirname4(head).length + 1));
-    head = dirname4(head);
+    tail.push(head.slice(dirname5(head).length + 1));
+    head = dirname5(head);
   }
   return p;
 }
@@ -2556,26 +2651,26 @@ function computeFsRoots(opts) {
     "/System",
     "/proc",
     "/sys",
-    join9(home, ".gitconfig"),
-    join9(home, ".config"),
-    join9(home, ".npmrc"),
-    join9(home, ".cache"),
-    join9(home, "Library", "Caches"),
-    join9(home, "Library", "pnpm"),
-    join9(home, ".local", "share"),
-    join9(home, ".nvm"),
-    join9(home, ".volta"),
-    join9(home, ".asdf"),
-    join9(home, ".cargo"),
-    join9(home, ".rustup"),
+    join10(home, ".gitconfig"),
+    join10(home, ".config"),
+    join10(home, ".npmrc"),
+    join10(home, ".cache"),
+    join10(home, "Library", "Caches"),
+    join10(home, "Library", "pnpm"),
+    join10(home, ".local", "share"),
+    join10(home, ".nvm"),
+    join10(home, ".volta"),
+    join10(home, ".asdf"),
+    join10(home, ".cargo"),
+    join10(home, ".rustup"),
     ...splitRoots(process.env.DAHRK_FS_EXTRA_READ_ROOTS).map((p) => realish(resolve(p)))
   ].map(realish);
   const deny = [
-    join9(home, ".ssh"),
-    join9(home, ".aws"),
-    join9(home, ".gnupg"),
-    join9(home, ".config", "gcloud"),
-    join9(home, "Library", "Keychains"),
+    join10(home, ".ssh"),
+    join10(home, ".aws"),
+    join10(home, ".gnupg"),
+    join10(home, ".config", "gcloud"),
+    join10(home, "Library", "Keychains"),
     "/Volumes",
     "/etc/shadow",
     "/etc/sudoers"
@@ -3097,32 +3192,32 @@ var attemptOf = (jobId) => {
 };
 var digest = (value) => `sha256:${createHash3("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16)}`;
 function writeScratchState(ref, job, attempt, status) {
-  const statePath = join10(ref.scratchPath, "state.json");
+  const statePath = join11(ref.scratchPath, "state.json");
   let state;
   try {
-    state = JSON.parse(readFileSync4(statePath, "utf8"));
+    state = JSON.parse(readFileSync5(statePath, "utf8"));
   } catch {
     state = { runId: job.runId, tenantId: job.tenantId, stages: {} };
   }
   state.stages[job.stageId] = { currentAttempt: attempt, status };
-  mkdirSync6(ref.scratchPath, { recursive: true });
-  writeFileSync5(statePath, JSON.stringify(state, null, 2));
+  mkdirSync7(ref.scratchPath, { recursive: true });
+  writeFileSync6(statePath, JSON.stringify(state, null, 2));
 }
 function writeIssueContext(ref, issueContext) {
   if (issueContext === void 0) return;
   try {
-    mkdirSync6(ref.scratchPath, { recursive: true });
-    writeFileSync5(join10(ref.scratchPath, "issue.md"), issueContext);
+    mkdirSync7(ref.scratchPath, { recursive: true });
+    writeFileSync6(join11(ref.scratchPath, "issue.md"), issueContext);
   } catch {
   }
 }
 function writeAttachedDocuments(ref, docs) {
   if (!docs || docs.length === 0) return;
   try {
-    const dir = join10(ref.scratchPath, "docs");
-    mkdirSync6(dir, { recursive: true });
+    const dir = join11(ref.scratchPath, "docs");
+    mkdirSync7(dir, { recursive: true });
     for (const doc of docs) {
-      writeFileSync5(join10(dir, `${attachedDocBasename2(doc)}.md`), doc.content);
+      writeFileSync6(join11(dir, `${attachedDocBasename2(doc)}.md`), doc.content);
     }
   } catch {
   }
@@ -3140,8 +3235,8 @@ ${lines.join("\n")}
 function writeGuidance(ref, guidance) {
   if (!guidance || guidance.length === 0) return;
   try {
-    mkdirSync6(ref.scratchPath, { recursive: true });
-    writeFileSync5(join10(ref.scratchPath, "guidance.md"), renderGuidanceMarkdown(guidance));
+    mkdirSync7(ref.scratchPath, { recursive: true });
+    writeFileSync6(join11(ref.scratchPath, "guidance.md"), renderGuidanceMarkdown(guidance));
   } catch {
   }
 }
@@ -3167,7 +3262,7 @@ function readEmittedArtifact(ref, relPath) {
   const path = resolveWorktreeRelativePath(ref, relPath);
   if (!path) return void 0;
   try {
-    const raw = readFileSync4(path, "utf8");
+    const raw = readFileSync5(path, "utf8");
     return { path: relPath, content: capContent(raw) };
   } catch {
     return void 0;
@@ -3175,13 +3270,13 @@ function readEmittedArtifact(ref, relPath) {
 }
 function scanScratchOutput(ref, preferRel) {
   try {
-    const names = readdirSync3(join10(ref.worktreePath, SCRATCH_OUTPUT_DIR)).filter(
+    const names = readdirSync3(join11(ref.worktreePath, SCRATCH_OUTPUT_DIR)).filter(
       (n) => n.toLowerCase().endsWith(".md")
     );
     if (names.length === 0) return void 0;
     const preferBase = preferRel?.split("/").pop();
     const pick = preferBase && names.includes(preferBase) ? preferBase : names[0];
-    const raw = readFileSync4(join10(ref.worktreePath, SCRATCH_OUTPUT_DIR, pick), "utf8");
+    const raw = readFileSync5(join11(ref.worktreePath, SCRATCH_OUTPUT_DIR, pick), "utf8");
     if (raw.trim().length === 0) return void 0;
     return { path: `${SCRATCH_OUTPUT_DIR}/${pick}`, content: capContent(raw) };
   } catch {
@@ -3201,7 +3296,7 @@ function scanChangedMarkdown(ref) {
   );
   for (const rel of rels) {
     try {
-      const raw = readFileSync4(join10(ref.worktreePath, rel), "utf8");
+      const raw = readFileSync5(join11(ref.worktreePath, rel), "utf8");
       if (raw.trim().length > 0) return { path: rel, content: capContent(raw) };
     } catch {
     }
@@ -3324,10 +3419,10 @@ function createStageRunner(deps) {
               ...job.workspaceRef.credentialToken ? { credentialToken: job.workspaceRef.credentialToken } : {}
             });
           } else {
-            const base = deps.scratchRoot ?? join10(tmpdir4(), "dahrk", "scratch");
-            const worktreePath = join10(base, runId);
-            const scratchPath = join10(worktreePath, ".skakel", "scratch");
-            mkdirSync6(scratchPath, { recursive: true });
+            const base = deps.scratchRoot ?? join11(tmpdir4(), "dahrk", "scratch");
+            const worktreePath = join11(base, runId);
+            const scratchPath = join11(worktreePath, ".skakel", "scratch");
+            mkdirSync7(scratchPath, { recursive: true });
             ref = { repoId: "", gitUrl: "", repo: "", baseBranch: "", worktreePath, scratchPath };
             scratchOnly.add(runId);
           }
@@ -3342,7 +3437,7 @@ function createStageRunner(deps) {
           const slash = job.agentConfig.emitArtifact.lastIndexOf("/");
           if (artifactDir && slash > 0) {
             try {
-              mkdirSync6(join10(ref.worktreePath, job.agentConfig.emitArtifact.slice(0, slash)), { recursive: true });
+              mkdirSync7(join11(ref.worktreePath, job.agentConfig.emitArtifact.slice(0, slash)), { recursive: true });
             } catch {
             }
           }
@@ -3370,8 +3465,8 @@ function createStageRunner(deps) {
           if (!sink) return;
           const base = { tenantId: job.tenantId, runId, stageId, attempt };
           try {
-            for (const name of readdirSync3(join10(writer.dir, "blobs"))) {
-              const bytes = readFileSync4(join10(writer.dir, "blobs", name));
+            for (const name of readdirSync3(join11(writer.dir, "blobs"))) {
+              const bytes = readFileSync5(join11(writer.dir, "blobs", name));
               const { url } = await sink.requestBlobUrl({
                 ...base,
                 sha256: name,
@@ -3385,7 +3480,7 @@ function createStageRunner(deps) {
           }
           let archiveKey;
           try {
-            const bytes = readFileSync4(join10(writer.dir, "trace.jsonl"));
+            const bytes = readFileSync5(join11(writer.dir, "trace.jsonl"));
             const sha = createHash3("sha256").update(bytes).digest("hex");
             const { key, url } = await sink.requestBlobUrl({
               ...base,
@@ -3771,6 +3866,7 @@ async function startEdgeNode(opts) {
   const log = opts.logger ?? createNodeLogger({ level: levelFromEnv(process.env) });
   const rules = opts.denyTool ? [denyToolRule(opts.denyTool)] : [];
   const counters = new HealthCounters();
+  const ledger = opts.jobLedger ?? nullJobLedger();
   const shipper = opts.shipper;
   const telemetryCeiling = ceilingFromEnv(process.env);
   let currentRuntimes = opts.runtimes;
@@ -3869,6 +3965,39 @@ async function startEdgeNode(opts) {
   };
   const stageRunner = createStageRunner(stageDeps);
   stageRunnerRef = stageRunner;
+  const reconcileInterruptedJobs = async () => {
+    const stale = ledger.stale(process.pid);
+    if (!stale.length) return;
+    log.warn({ count: stale.length }, `EDGE_INTERRUPTED:${stale.length} jobs died with the previous process`);
+    for (const entry of stale) {
+      const entryLog = log.child({ runId: entry.runId, jobId: entry.jobId, ...entry.stageId ? { stageId: entry.stageId } : {} });
+      if (!entry.worktreePath || !entry.gitUrl) {
+        entryLog.warn({}, `EDGE_INTERRUPTED_ABANDONED:${entry.jobId} no worktree recorded`);
+        continue;
+      }
+      try {
+        const r = await gitService.reconcileInterrupted(
+          { worktreePath: entry.worktreePath, gitUrl: entry.gitUrl },
+          {
+            message: `wip: work in progress when the node was interrupted (run ${entry.runId})`,
+            branch: `dahrk/wip/${entry.runId}`
+          }
+        );
+        if (r.dirty) {
+          entryLog.warn(
+            { wipRef: r.wipRef, tailSha: r.tailSha, headSha: r.headSha, pushed: r.pushed },
+            `EDGE_INTERRUPTED_RESET:${entry.jobId} preserved the uncommitted tail on ${r.wipRef}${r.pushed ? "" : " (locally only)"} and reset to ${r.headSha.slice(0, 8)}`
+          );
+        } else {
+          entryLog.info({ headSha: r.headSha }, `EDGE_INTERRUPTED_CLEAN:${entry.jobId} worktree was already clean`);
+        }
+      } catch (e) {
+        entryLog.warn({ err: e }, `EDGE_INTERRUPTED_ERROR:${entry.jobId} ${e.message}`);
+      }
+    }
+    ledger.clear();
+  };
+  await reconcileInterruptedJobs();
   void stageRunner.reapWorktrees().then((r) => {
     counters.worktreeCount = Math.max(r.scanned - r.reaped.length, 0);
     if (r.reaped.length) {
@@ -3895,7 +4024,22 @@ async function startEdgeNode(opts) {
       // Advertise the resolved worktree base so the hub records each run's real worktree location in
       // the projection instead of an advisory placeholder. Single-sourced from the git service so it
       // always matches where worktrees actually land.
-      worktreesDir: gitService.worktreesDir
+      worktreesDir: gitService.worktreesDir,
+      // What we are running RIGHT NOW (DHK-416), which is what lets a new hub build ADOPT an in-flight
+      // stage rather than duplicate it (DHK-415). Without this the hub cannot tell "this node is midway
+      // through the stage you are about to re-dispatch" from "this node is idle", so a hub roll or a
+      // reconnect re-ran the stage from scratch.
+      //
+      // Always sent, even empty. Per the wire contract an ABSENT list means "unknown" (a node too old to
+      // answer, for which the hub must keep its old re-dispatch behaviour), while an EMPTY list means "I
+      // have nothing in flight" - a positive statement the hub can act on. We always know, so we always
+      // say, and the two must not be conflated.
+      //
+      // Only genuinely-running jobs are in `running`: boot reconciliation has already dropped the entries
+      // whose runner died with the previous process, so we never claim to be running something we are not.
+      // `announceableJobs` drops what we cannot version-stamp - announcing such a job would make the hub
+      // KILL it, not adopt it. See there.
+      inFlightJobs: announceableJobs(running.values())
     });
   };
   let reprobeTimer;
@@ -3915,7 +4059,15 @@ async function startEdgeNode(opts) {
     }, opts.runtimeRecheckMs ?? 6e4);
     reprobeTimer.unref?.();
   }
-  const running = /* @__PURE__ */ new Set();
+  const running = /* @__PURE__ */ new Map();
+  const trackJob = (entry) => {
+    running.set(entry.jobId, entry);
+    ledger.upsert(entry);
+  };
+  const untrackJob = (jobId) => {
+    running.delete(jobId);
+    ledger.remove(jobId);
+  };
   const onMessage = async (raw) => {
     const msg = decode(raw);
     if (msg.type === "welcome") {
@@ -3984,8 +4136,20 @@ async function startEdgeNode(opts) {
         pushLog.info({}, `PUSH_REPLAY:${job2.runId} ${job2.jobId}`);
         return;
       }
-      running.add(job2.jobId);
       const pushStartedAt = Date.now();
+      trackJob({
+        jobId: job2.jobId,
+        runId: job2.runId,
+        kind: "push",
+        // No `payloadVersion`: `PushJob` does not carry one (DHK-415 added the field to `JobRequest`
+        // only). It is ledgered anyway - boot reconciliation still has to clean its worktree - but it is
+        // deliberately never announced. See `sendHello`.
+        ...job2.workspaceRef?.worktreePath ? { worktreePath: job2.workspaceRef.worktreePath } : {},
+        ...job2.branch ? { branch: job2.branch } : {},
+        ...job2.workspaceRef?.gitUrl ? { gitUrl: job2.workspaceRef.gitUrl } : {},
+        startedAt: pushStartedAt,
+        nodePid: process.pid
+      });
       pushLog.info({}, `PUSH_STARTED:${job2.runId} ${job2.branch}`);
       try {
         const result = await stageRunner.runPush(job2);
@@ -4006,7 +4170,7 @@ async function startEdgeNode(opts) {
         send(frame);
         pushLog.error({ err: e, durationMs: Date.now() - pushStartedAt }, `PUSH_ERROR:${job2.runId} ${e.message}`);
       } finally {
-        running.delete(job2.jobId);
+        untrackJob(job2.jobId);
       }
       return;
     }
@@ -4030,9 +4194,23 @@ async function startEdgeNode(opts) {
       jobLog.info({}, `JOB_REPLAY:${job.stageId} ${job.jobId}`);
       return;
     }
-    running.add(job.jobId);
-    counters.activeJobs = running.size;
     const startedAt = Date.now();
+    trackJob({
+      jobId: job.jobId,
+      runId: job.runId,
+      kind: "stage",
+      stageId: job.stageId,
+      // The hub stamped this on dispatch. Persisting it is what lets a reconnecting node prove, after a
+      // restart, that the job it is announcing was dispatched under a contract the adopting build can
+      // still read. Announced without it, the hub's gate version-rejects the job rather than adopting it.
+      ...job.payloadVersion ? { payloadVersion: job.payloadVersion } : {},
+      ...job.workspaceRef?.worktreePath ? { worktreePath: job.workspaceRef.worktreePath } : {},
+      ...job.workspaceRef?.branch ? { branch: job.workspaceRef.branch } : {},
+      ...job.workspaceRef?.gitUrl ? { gitUrl: job.workspaceRef.gitUrl } : {},
+      startedAt,
+      nodePid: process.pid
+    });
+    counters.activeJobs = running.size;
     jobLog.info({}, `JOB_STARTED:${job.stageId} ${job.jobId}`);
     try {
       const result = await stageRunner.runJob(job);
@@ -4054,7 +4232,7 @@ async function startEdgeNode(opts) {
       counters.recordError(classifyError(e));
       jobLog.error({ err: e, durationMs: Date.now() - startedAt }, `JOB_ERROR:${job.stageId} ${e.message}`);
     } finally {
-      running.delete(job.jobId);
+      untrackJob(job.jobId);
       counters.activeJobs = running.size;
     }
   };
@@ -4660,8 +4838,8 @@ function usage(bin, command) {
 }
 
 // src/diagnose.ts
-import { existsSync as existsSync7, mkdirSync as mkdirSync7, readdirSync as readdirSync4, readFileSync as readFileSync5, statSync as statSync3, writeFileSync as writeFileSync6 } from "fs";
-import { join as join11 } from "path";
+import { existsSync as existsSync8, mkdirSync as mkdirSync8, readdirSync as readdirSync4, readFileSync as readFileSync6, statSync as statSync3, writeFileSync as writeFileSync7 } from "fs";
+import { join as join12 } from "path";
 var BUNDLE_LOG_LINES = 2e3;
 function tailJsonl(raw, n) {
   const records = [];
@@ -4702,7 +4880,7 @@ async function buildBundle(deps) {
   if (deps.exists(deps.crashDir)) {
     for (const name of deps.listDir(deps.crashDir).filter((f) => f.endsWith(".json")).sort()) {
       try {
-        crashes.push(JSON.parse(deps.readFile(join11(deps.crashDir, name))));
+        crashes.push(JSON.parse(deps.readFile(join12(deps.crashDir, name))));
       } catch (e) {
         warnings.push(`could not read crash record ${name} (${e.message})`);
       }
@@ -4759,19 +4937,19 @@ var defaultDiagnoseDeps = (paths, clientVersion, doctor) => ({
   ...paths,
   clientVersion,
   ...doctor ? { doctor } : {},
-  readFile: (p) => readFileSync5(p, "utf8"),
+  readFile: (p) => readFileSync6(p, "utf8"),
   listDir: (p) => readdirSync4(p),
-  exists: (p) => existsSync7(p),
+  exists: (p) => existsSync8(p),
   writeFile: (p, content) => {
-    const dir = join11(p, "..");
-    if (!existsSync7(dir)) mkdirSync7(dir, { recursive: true });
-    writeFileSync6(p, content, { mode: 384 });
+    const dir = join12(p, "..");
+    if (!existsSync8(dir)) mkdirSync8(dir, { recursive: true });
+    writeFileSync7(p, content, { mode: 384 });
   },
   out: (line) => void process.stdout.write(`${line}
 `)
 });
 function defaultBundlePath(cwd, now) {
-  return join11(cwd, `dahrk-diagnose-${now.toISOString().replace(/[:.]/g, "-")}.json`);
+  return join12(cwd, `dahrk-diagnose-${now.toISOString().replace(/[:.]/g, "-")}.json`);
 }
 
 // src/doctor.ts
@@ -4906,8 +5084,8 @@ async function runDoctor(inputs, deps = {}) {
 }
 
 // src/lock.ts
-import { existsSync as existsSync8, mkdirSync as mkdirSync8, readFileSync as readFileSync6, rmSync as rmSync6, writeFileSync as writeFileSync7 } from "fs";
-import { dirname as dirname5 } from "path";
+import { existsSync as existsSync9, mkdirSync as mkdirSync9, readFileSync as readFileSync7, rmSync as rmSync6, writeFileSync as writeFileSync8 } from "fs";
+import { dirname as dirname6 } from "path";
 function parseLock(content) {
   if (!content) return void 0;
   const pid = Number(content.trim());
@@ -4939,14 +5117,14 @@ var defaultLockDeps = (file) => ({
   pid: process.pid,
   readFile: (path) => {
     try {
-      return readFileSync6(path, "utf8");
+      return readFileSync7(path, "utf8");
     } catch {
       return void 0;
     }
   },
   writeFile: (path, content) => {
-    if (!existsSync8(dirname5(path))) mkdirSync8(dirname5(path), { recursive: true, mode: 448 });
-    writeFileSync7(path, content);
+    if (!existsSync9(dirname6(path))) mkdirSync9(dirname6(path), { recursive: true, mode: 448 });
+    writeFileSync8(path, content);
   },
   removeFile: (path) => rmSync6(path, { force: true }),
   isAlive
@@ -4954,7 +5132,7 @@ var defaultLockDeps = (file) => ({
 
 // src/logs.ts
 import { spawn } from "child_process";
-import { copyFileSync, existsSync as existsSync9, readFileSync as readFileSync7, statSync as statSync4, truncateSync } from "fs";
+import { copyFileSync, existsSync as existsSync10, readFileSync as readFileSync8, statSync as statSync4, truncateSync } from "fs";
 var MAX_LOG_BYTES = 10 * 1024 * 1024;
 function logsCommand(inputs) {
   return [
@@ -5035,7 +5213,7 @@ async function runStructuredLogs(inputs, deps) {
 }
 function rotateIfLarge(file, maxBytes = MAX_LOG_BYTES) {
   try {
-    if (!existsSync9(file) || statSync4(file).size <= maxBytes) return;
+    if (!existsSync10(file) || statSync4(file).size <= maxBytes) return;
     copyFileSync(file, `${file}.1`);
     truncateSync(file, 0);
   } catch {
@@ -5044,8 +5222,8 @@ function rotateIfLarge(file, maxBytes = MAX_LOG_BYTES) {
 var defaultLogsDeps = (files, jsonlFile) => ({
   files,
   jsonlFile,
-  fileExists: (path) => existsSync9(path),
-  readFile: (path) => readFileSync7(path, "utf8"),
+  fileExists: (path) => existsSync10(path),
+  readFile: (path) => readFileSync8(path, "utf8"),
   run: (argv) => new Promise((resolve3) => {
     const [cmd, ...args] = argv;
     const child = spawn(cmd, args, { stdio: "inherit" });
@@ -5057,13 +5235,13 @@ var defaultLogsDeps = (files, jsonlFile) => ({
 });
 
 // src/process-safety.ts
-import { mkdirSync as mkdirSync9, writeFileSync as writeFileSync8 } from "fs";
-import { join as join12 } from "path";
+import { mkdirSync as mkdirSync10, writeFileSync as writeFileSync9 } from "fs";
+import { join as join13 } from "path";
 function writeCrashRecord(dir, record) {
   try {
-    mkdirSync9(dir, { recursive: true, mode: 448 });
-    const file = join12(dir, `${record.at.replace(/[:.]/g, "-")}.json`);
-    writeFileSync8(file, `${JSON.stringify(record, null, 2)}
+    mkdirSync10(dir, { recursive: true, mode: 448 });
+    const file = join13(dir, `${record.at.replace(/[:.]/g, "-")}.json`);
+    writeFileSync9(file, `${JSON.stringify(record, null, 2)}
 `, { mode: 384 });
     return file;
   } catch {
@@ -5118,9 +5296,9 @@ function installProcessSafetyNet(opts) {
 // src/preflight.ts
 import { execFileSync as execFileSync6 } from "child_process";
 import { randomUUID as randomUUID2 } from "crypto";
-import { accessSync, constants as fsConstants, existsSync as existsSync10, readdirSync as readdirSync5, statfsSync as statfsSync2 } from "fs";
+import { accessSync, constants as fsConstants, existsSync as existsSync11, readdirSync as readdirSync5, statfsSync as statfsSync2 } from "fs";
 import { homedir as homedir4 } from "os";
-import { join as join13 } from "path";
+import { join as join14 } from "path";
 var REPORT_BASE_URL = "https://app.dahrk.ai/r";
 var LOW_DISK_BYTES = 512 * 1024 * 1024;
 var PREFLIGHT_STAGES = [
@@ -5257,8 +5435,8 @@ function commandPresent(cmd) {
 }
 function sshKeyPresent() {
   try {
-    const dir = join13(homedir4(), ".ssh");
-    if (existsSync10(dir) && readdirSync5(dir).some((f) => f.endsWith(".pub"))) return true;
+    const dir = join14(homedir4(), ".ssh");
+    if (existsSync11(dir) && readdirSync5(dir).some((f) => f.endsWith(".pub"))) return true;
   } catch {
   }
   try {
@@ -5277,12 +5455,12 @@ function writable(dir) {
   }
 }
 function worktreeRoot(env) {
-  return env.DAHRK_WORKTREES_DIR ?? join13(env.DAHRK_STATE_DIR ?? join13(homedir4(), ".dahrk"), "worktrees");
+  return env.DAHRK_WORKTREES_DIR ?? join14(env.DAHRK_STATE_DIR ?? join14(homedir4(), ".dahrk"), "worktrees");
 }
 function nearestExisting(dir) {
   let cur = dir;
-  while (!existsSync10(cur)) {
-    const parent = join13(cur, "..");
+  while (!existsSync11(cur)) {
+    const parent = join14(cur, "..");
     if (parent === cur) break;
     cur = parent;
   }
@@ -5333,52 +5511,52 @@ function gatherHostFacts(repoPath) {
 
 // src/service.ts
 import { execFileSync as execFileSync7 } from "child_process";
-import { chmodSync as chmodSync2, existsSync as existsSync12, mkdirSync as mkdirSync11, readFileSync as readFileSync9, realpathSync as realpathSync3, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "fs";
+import { chmodSync as chmodSync3, existsSync as existsSync13, mkdirSync as mkdirSync12, readFileSync as readFileSync10, realpathSync as realpathSync3, rmSync as rmSync7, writeFileSync as writeFileSync11 } from "fs";
 import { homedir as homedir6, platform as osPlatform3, userInfo } from "os";
-import { join as join15 } from "path";
+import { join as join16 } from "path";
 
 // src/state.ts
-import { chmodSync, existsSync as existsSync11, mkdirSync as mkdirSync10, readFileSync as readFileSync8, writeFileSync as writeFileSync9 } from "fs";
+import { chmodSync as chmodSync2, existsSync as existsSync12, mkdirSync as mkdirSync11, readFileSync as readFileSync9, writeFileSync as writeFileSync10 } from "fs";
 import { homedir as homedir5 } from "os";
-import { join as join14 } from "path";
+import { join as join15 } from "path";
 var RUNTIMES = ["claude-code", "codex", "pi"];
 var isRuntime = (v) => RUNTIMES.includes(v);
 var STRING_FIELDS = ["nodeId", "enrolToken", "name", "tenantId", "updateCheckedAt", "updateLatest"];
 var isDesired = (v) => v === "running" || v === "stopped";
-var FILE_MODE = 384;
-var DIR_MODE = 448;
+var FILE_MODE2 = 384;
+var DIR_MODE2 = 448;
 function stateDir(env) {
-  return env.DAHRK_STATE_DIR ?? join14(homedir5(), ".dahrk");
+  return env.DAHRK_STATE_DIR ?? join15(homedir5(), ".dahrk");
 }
 function legacyStateDir(env) {
-  return env.DAHRK_STATE_DIR ? void 0 : join14(homedir5(), ".skakel");
+  return env.DAHRK_STATE_DIR ? void 0 : join15(homedir5(), ".skakel");
 }
 function stateFile(env) {
-  return join14(stateDir(env), "node.json");
+  return join15(stateDir(env), "node.json");
 }
 function logDir(env) {
-  return join14(stateDir(env), "logs");
+  return join15(stateDir(env), "logs");
 }
 function logFiles(env) {
   const dir = logDir(env);
-  return { out: join14(dir, "node.out.log"), err: join14(dir, "node.err.log") };
+  return { out: join15(dir, "node.out.log"), err: join15(dir, "node.err.log") };
 }
 function jsonlLogFile(env) {
-  return join14(logDir(env), "node.jsonl");
+  return join15(logDir(env), "node.jsonl");
 }
 function crashDir(env) {
-  return join14(logDir(env), "crashes");
+  return join15(logDir(env), "crashes");
 }
 function lockFile(env) {
-  return join14(stateDir(env), "node.pid");
+  return join15(stateDir(env), "node.pid");
 }
 function setDesired(env, desired) {
   writeState(env, { desired });
 }
 function readState(file) {
-  if (!existsSync11(file)) return {};
+  if (!existsSync12(file)) return {};
   try {
-    const parsed = JSON.parse(readFileSync8(file, "utf8"));
+    const parsed = JSON.parse(readFileSync9(file, "utf8"));
     const state = {};
     for (const key of STRING_FIELDS) {
       const value = parsed[key];
@@ -5398,11 +5576,11 @@ function writeState(env, patch) {
   const dir = stateDir(env);
   const file = stateFile(env);
   try {
-    mkdirSync10(dir, { recursive: true, mode: DIR_MODE });
+    mkdirSync11(dir, { recursive: true, mode: DIR_MODE2 });
     const next = { ...readState(file), ...patch };
-    writeFileSync9(file, `${JSON.stringify(next, null, 2)}
-`, { mode: FILE_MODE });
-    chmodSync(file, FILE_MODE);
+    writeFileSync10(file, `${JSON.stringify(next, null, 2)}
+`, { mode: FILE_MODE2 });
+    chmodSync2(file, FILE_MODE2);
   } catch (e) {
     console.warn(`could not persist node state to ${file}: ${e.message}`);
   }
@@ -5481,9 +5659,9 @@ ${envEntries}
   <key>ThrottleInterval</key>
   <integer>10</integer>
   <key>StandardOutPath</key>
-  <string>${xmlEscape(join15(inputs.logDir, "node.out.log"))}</string>
+  <string>${xmlEscape(join16(inputs.logDir, "node.out.log"))}</string>
   <key>StandardErrorPath</key>
-  <string>${xmlEscape(join15(inputs.logDir, "node.err.log"))}</string>
+  <string>${xmlEscape(join16(inputs.logDir, "node.err.log"))}</string>
 </dict>
 </plist>
 `;
@@ -5506,8 +5684,8 @@ Type=simple
 ExecStart=${exec}
 ${envLines}
 WorkingDirectory=${inputs.homeDir}
-StandardOutput=append:${join15(inputs.logDir, "node.out.log")}
-StandardError=append:${join15(inputs.logDir, "node.err.log")}
+StandardOutput=append:${join16(inputs.logDir, "node.out.log")}
+StandardError=append:${join16(inputs.logDir, "node.err.log")}
 Restart=on-failure
 RestartSec=3
 RestartPreventExitStatus=78
@@ -5518,7 +5696,7 @@ WantedBy=default.target
 }
 function buildPlan(inputs) {
   if (inputs.manager === "launchd") {
-    const filePath2 = join15(inputs.homeDir, "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`);
+    const filePath2 = join16(inputs.homeDir, "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`);
     return {
       manager: "launchd",
       label: LAUNCHD_LABEL,
@@ -5535,7 +5713,7 @@ function buildPlan(inputs) {
       logHint: "dahrk logs -f"
     };
   }
-  const filePath = join15(inputs.homeDir, ".config", "systemd", "user", SYSTEMD_UNIT);
+  const filePath = join16(inputs.homeDir, ".config", "systemd", "user", SYSTEMD_UNIT);
   const user = userInfo().username;
   return {
     manager: "systemd",
@@ -5565,7 +5743,7 @@ function unitIsCurrent(plan, onDisk) {
   return onDisk === plan.content;
 }
 function unitPath(manager, homeDir) {
-  return manager === "launchd" ? join15(homeDir, "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`) : join15(homeDir, ".config", "systemd", "user", SYSTEMD_UNIT);
+  return manager === "launchd" ? join16(homeDir, "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`) : join16(homeDir, ".config", "systemd", "user", SYSTEMD_UNIT);
 }
 function statusCommand(manager) {
   return manager === "launchd" ? ["launchctl", "list", LAUNCHD_LABEL] : ["systemctl", "--user", "show", SYSTEMD_UNIT, "-p", "ActiveState", "-p", "MainPID"];
@@ -5806,23 +5984,23 @@ var defaultDeps3 = () => ({
   // Snapshot the operator's PATH at install time so the daemon finds git + the runtime CLIs (Homebrew /
   // npm-global bins) that a supervisor's minimal PATH would otherwise hide.
   pathEnv: process.env.PATH,
-  mkdirp: (dir) => void mkdirSync11(dir, { recursive: true }),
+  mkdirp: (dir) => void mkdirSync12(dir, { recursive: true }),
   // The unit's environment block carries the enrolment token, so the file is a secret: write it
   // owner-only. `writeFileSync`'s `mode` applies only when it CREATES the file, so chmod explicitly
   // too - re-installing over a unit an older client left at 0644 must tighten it, not keep it.
   writeFile: (path, content) => {
-    writeFileSync10(path, content, { mode: UNIT_FILE_MODE });
-    chmodSync2(path, UNIT_FILE_MODE);
+    writeFileSync11(path, content, { mode: UNIT_FILE_MODE });
+    chmodSync3(path, UNIT_FILE_MODE);
   },
   readFile: (path) => {
     try {
-      return readFileSync9(path, "utf8");
+      return readFileSync10(path, "utf8");
     } catch {
       return void 0;
     }
   },
   removeFile: (path) => rmSync7(path, { force: true }),
-  fileExists: (path) => existsSync12(path),
+  fileExists: (path) => existsSync13(path),
   run: (argv) => {
     const [cmd, ...args] = argv;
     try {
@@ -6103,7 +6281,7 @@ async function runStatus(inputs, deps) {
 }
 
 // src/main.ts
-var CLIENT_VERSION = "0.1.13";
+var CLIENT_VERSION = "0.1.14";
 var DEFAULT_HUB_URL = "wss://api.dahrk.ai";
 var list = (v) => (v ?? "").split(",").map((s) => s.trim()).filter(Boolean);
 var RUNTIMES2 = ["claude-code", "codex", "pi"];
@@ -6115,7 +6293,7 @@ function resolveNodeId(env, opts = {}) {
   if (existing) return existing;
   const legacy = legacyStateDir(env);
   if (legacy) {
-    const legacyId = readState(join16(legacy, "node.json")).nodeId;
+    const legacyId = readState(join17(legacy, "node.json")).nodeId;
     if (legacyId) return legacyId;
   }
   const nodeId = randomUUID3();
@@ -6258,6 +6436,10 @@ async function startForeground(env, flags) {
     // (a pinned override never re-probes to something else) and the mock runner path stays stable.
     reprobeRuntimes: () => resolveRuntimes(env),
     ...env.DAHRK_RUNTIME_RECHECK_MS ? { runtimeRecheckMs: Number(env.DAHRK_RUNTIME_RECHECK_MS) } : {},
+    // What this node is running, on disk, so a crash mid-stage does not silently re-run the stage from
+    // scratch (DHK-416). Skipped for an ephemeral node for the same reason it never caches a token: a
+    // one-shot CI node touches no state dir, and has no next boot to recover into.
+    ...flags.ephemeral ? {} : { jobLedger: fileJobLedger(jobLedgerFile(stateDir(env))) },
     ...persist ? {
       onEnrolled: (welcome) => persistEnrolment(env, { token, name: welcome.name, tenantId: welcome.tenantId })
     } : {}
@@ -6317,7 +6499,7 @@ function statusDeps(env) {
     env,
     binPath: process.argv[1],
     detectRuntimes: () => resolveRuntimes(env),
-    fileExists: (path) => existsSync13(path),
+    fileExists: (path) => existsSync14(path),
     capture: (argv) => {
       const [cmd, ...args] = argv;
       try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dahrk-node",
-  "version": "0.1.13",
+  "version": "0.1.14",
   "private": false,
   "license": "Apache-2.0",
   "type": "module",
@@ -34,7 +34,7 @@
   "exports": "./dist/main.js",
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "0.3.183",
-    "@dahrk/contracts": "^0.2.0",
+    "@dahrk/contracts": "^0.3.0",
     "@openai/codex-sdk": "0.141.0",
     "pino": "^10.3.1",
     "ws": "^8.18.0",