harnery 0.42.0 → 0.44.0

package/src/core/journal/index.ts

@@ -20,6 +20,14 @@ import {
   readLiveCoordinationRow,
   readLiveCoordinationRows,
 } from "../agents/state/live-coordination-view.ts";
+import {
+  readCoordinationViewV3,
+  requireAuthoritySafeCoordinationViewV3,
+} from "../events/v3/coordination-view.ts";
+import {
+  liveInstanceIdV3,
+  observeLiveEventLedgerRouteV3,
+} from "../events/v3/live-route-observer.ts";
 
 /**
  * Agent journal: per-agent markdown journal at `.harnery/journal/<instance_id>.md`.
@@ -421,11 +429,20 @@ export function sweepOrphanJournals(): string[] {
   const dir = journalDir();
   const root = monorepoRoot();
   const archived: string[] = [];
+  if (!root) return archived;
+  let view: ReturnType<typeof readCoordinationViewV3>;
+  try {
+    if (observeLiveEventLedgerRouteV3(root).state === "blocked") return archived;
+    view = requireAuthoritySafeCoordinationViewV3(readCoordinationViewV3(root));
+    if (!view.source_complete) return archived;
+  } catch {
+    // A missing or unsafe read does not prove that an owner has ended.
+    return archived;
+  }
   for (const f of readdirSync(dir)) {
     if (!f.endsWith(".md")) continue;
     const instanceId = f.replace(/\.md$/, "");
-    const hb = root ? readLiveCoordinationRow(root, instanceId) : null;
-    if (!hb) {
+    if (!view.instances[liveInstanceIdV3(instanceId)]) {
       const dest = archiveJournal(instanceId);
       if (dest) archived.push(dest);
     }

package/src/core/resources/contract.ts

@@ -1,3 +1,5 @@
+import type { ProcessGenerationVerdict } from "../agents/state/proc-start.ts";
+
 export const RESOURCE_SNAPSHOT_SCHEMA_VERSION = 2;
 export const RESOURCE_SERVICE_STATUS_SCHEMA_VERSION = 1;
 
@@ -154,6 +156,8 @@ export interface ResourceServiceStatusRecord {
 export interface ResourceServiceStatus {
   running: boolean;
   stale: boolean;
+  generation: ProcessGenerationVerdict;
+  liveness: "running" | "stopped" | "unknown";
   record?: ResourceServiceStatusRecord;
   status_path: string;
   snapshot_path: string;

package/src/core/resources/service-status.ts

@@ -1,5 +1,9 @@
+import { existsSync } from "node:fs";
 import { hostname } from "node:os";
-import { checkPidToken } from "../agents/state/proc-start.ts";
+import {
+  type ProcessGenerationVerdict,
+  verifyProcessGeneration,
+} from "../agents/state/proc-start.ts";
 import {
   RESOURCE_SERVICE_STATUS_SCHEMA_VERSION,
   type ResourceServiceStatus,
@@ -9,9 +13,15 @@ import { readResourceServiceRecord, resourcePaths } from "./storage.ts";
 
 const STALE_HEARTBEAT_MS = 15_000;
 
+export interface ResourceProcessDependencies {
+  readToken?: (pid: number) => string | null;
+  probePid?: (pid: number) => unknown;
+}
+
 export function readResourceServiceStatus(
   coordRoot: string,
   nowMs = Date.now(),
+  dependencies: ResourceProcessDependencies = {},
 ): ResourceServiceStatus {
   const paths = resourcePaths(coordRoot);
   const record = readResourceServiceRecord(coordRoot);
@@ -19,21 +29,22 @@ export function readResourceServiceStatus(
     return {
       running: false,
       stale: false,
+      generation: "unknown",
+      liveness: existsSync(paths.service) ? "unknown" : "stopped",
       status_path: paths.service,
       snapshot_path: paths.snapshot,
     };
   }
   const heartbeatAt = Date.parse(record.heartbeat_at);
   const stale = !Number.isFinite(heartbeatAt) || nowMs - heartbeatAt > STALE_HEARTBEAT_MS;
-  const local = record.host === hostname();
-  const running =
-    local &&
-    !stale &&
-    (record.state === "starting" || record.state === "running" || record.state === "stopping") &&
-    pidStillMatches(record.pid, record.start_token);
+  const owner = resourceProcessLiveness(record, dependencies);
+  const liveness =
+    owner.generation === "same" && record.state === "stopped" ? "stopped" : owner.liveness;
   return {
-    running,
+    running: liveness === "running" && !stale,
     stale,
+    generation: owner.generation,
+    liveness,
     record,
     status_path: paths.service,
     snapshot_path: paths.snapshot,
@@ -52,11 +63,23 @@ function validRecord(
   );
 }
 
-function pidStillMatches(pid: number, startToken: string | undefined): boolean {
+export function resourceProcessLiveness(
+  owner: { pid: number; host: string; start_token?: string },
+  dependencies: ResourceProcessDependencies = {},
+): { generation: ProcessGenerationVerdict; liveness: ResourceServiceStatus["liveness"] } {
+  if (owner.host !== hostname()) return { generation: "unknown", liveness: "unknown" };
   try {
-    process.kill(pid, 0);
+    (dependencies.probePid ?? ((pid) => process.kill(pid, 0)))(owner.pid);
   } catch (error) {
-    if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
+    return {
+      generation: "unknown",
+      liveness: (error as NodeJS.ErrnoException).code === "ESRCH" ? "stopped" : "unknown",
+    };
   }
-  return checkPidToken(pid, startToken) !== "mismatch";
+  const generation = verifyProcessGeneration(owner.pid, owner.start_token, dependencies.readToken);
+  return {
+    generation,
+    liveness:
+      generation === "same" ? "running" : generation === "different" ? "stopped" : "unknown",
+  };
 }

package/src/core/resources/service.ts

@@ -12,7 +12,12 @@ import {
 } from "node:fs";
 import { hostname, platform } from "node:os";
 import { resolve } from "node:path";
-import { checkPidToken, processStartToken } from "../agents/state/proc-start.ts";
+import {
+  type ProcessGenerationVerdict,
+  processStartToken,
+  signalIfSameGeneration,
+} from "../agents/state/proc-start.ts";
+import { resolveBinName } from "../config.ts";
 import { closeProcessLoggers, legacyLogFields, processLogger } from "../storage/logger.ts";
 import { stateDirMode, stateFileMode } from "../storage/modes.ts";
 import {
@@ -22,7 +27,11 @@ import {
   type ResourceServiceStatusRecord,
 } from "./contract.ts";
 import { sampleResources } from "./sampler.ts";
-import { readResourceServiceStatus } from "./service-status.ts";
+import {
+  type ResourceProcessDependencies,
+  readResourceServiceStatus,
+  resourceProcessLiveness,
+} from "./service-status.ts";
 import { resourcePaths, writePrivateJsonAtomic } from "./storage.ts";
 
 export const RESOURCE_SERVICE_DEFAULT_INTERVAL_MS = 2_000;
@@ -56,13 +65,14 @@ export async function spawnResourceService(
 ): Promise<ResourceServiceStatus> {
   const coordRoot = resolve(coordRootRaw);
   const current = readResourceServiceStatus(coordRoot);
-  if (current.running) {
+  if (current.liveness === "unknown") throw unknownResourceOwnerError();
+  if (current.liveness === "running") {
     throw new Error(`resource observer is already running under pid ${current.record?.pid}`);
   }
   const intervalMs = normalizeInterval(options.intervalMs);
   const paths = resourcePaths(coordRoot);
+  assertResourceLeaseAvailable(paths.lease);
   mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
-  rmSync(paths.stop, { force: true });
   const harnBin = new URL("../../../bin/harn", import.meta.url).pathname;
   if (!existsSync(harnBin)) throw new Error(`cannot find harn executable at ${harnBin}`);
   let spawnError: Error | undefined;
@@ -119,29 +129,49 @@ export async function ensureResourceServiceRunning(
   }
 }
 
-export function requestResourceServiceStop(coordRootRaw: string): ResourceServiceStatus {
+export interface ResourceServiceStopResult extends ResourceServiceStatus {
+  signal_sent: boolean;
+  signal_verdict: ProcessGenerationVerdict;
+}
+
+export function requestResourceServiceStop(
+  coordRootRaw: string,
+  dependencies: ResourceProcessDependencies & {
+    kill?: (pid: number, signal: NodeJS.Signals | number) => unknown;
+  } = {},
+): ResourceServiceStopResult {
   const coordRoot = resolve(coordRootRaw);
   const paths = resourcePaths(coordRoot);
-  const status = readResourceServiceStatus(coordRoot);
+  const status = readResourceServiceStatus(coordRoot, Date.now(), dependencies);
   mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
   writePrivateJsonAtomic(paths.stop, {
     requested_at: new Date().toISOString(),
     requested_by_pid: process.pid,
   });
-  if (status.running && status.record?.host === hostname()) {
-    try {
-      process.kill(status.record.pid, "SIGTERM");
-    } catch {
-      // The durable stop file handles a racing exit.
-    }
-  }
-  return readResourceServiceStatus(coordRoot);
+  const signal =
+    status.liveness === "running" && status.record?.host === hostname()
+      ? signalIfSameGeneration(
+          status.record.pid,
+          status.record.start_token,
+          "SIGTERM",
+          dependencies,
+        )
+      : { sent: false, verdict: status.generation };
+  return {
+    ...readResourceServiceStatus(coordRoot, Date.now(), dependencies),
+    signal_sent: signal.sent,
+    signal_verdict: signal.verdict,
+  };
 }
 
 export async function runResourceService(
   input: RunResourceServiceInput,
 ): Promise<ResourceServiceStatusRecord> {
   const coordRoot = resolve(input.coordRoot);
+  const current = readResourceServiceStatus(coordRoot);
+  if (current.liveness === "unknown") throw unknownResourceOwnerError();
+  if (current.liveness === "running")
+    throw new Error(`resource observer is already running under pid ${current.record?.pid}`);
   const paths = resourcePaths(coordRoot);
   const intervalMs = normalizeInterval(input.intervalMs);
   const now = input.now ?? (() => new Date());
@@ -253,7 +283,9 @@ function acquireLease(coordRoot: string, now: Date): ResourceServiceLease {
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
       const existing = readLease(paths.lease);
-      if (existing && leaseIsLive(existing)) {
+      const liveness = existing ? leaseIsLive(existing) : "unknown";
+      if (liveness === "unknown") throw unknownResourceOwnerError();
+      if (existing && liveness === "running") {
         throw new Error(`resource observer lease is held by pid ${existing.pid}`);
       }
       try {
@@ -279,7 +311,13 @@ function releaseLease(path: string, nonce: string): void {
 function readLease(path: string): ResourceServiceLease | undefined {
   try {
     const value = JSON.parse(readFileSync(path, "utf8")) as ResourceServiceLease;
-    return value && Number.isInteger(value.pid) && typeof value.nonce === "string"
+    return value &&
+      Number.isSafeInteger(value.pid) &&
+      value.pid > 0 &&
+      typeof value.host === "string" &&
+      value.host.length > 0 &&
+      typeof value.nonce === "string" &&
+      value.nonce.length > 0
       ? value
       : undefined;
   } catch {
@@ -287,14 +325,23 @@ function readLease(path: string): ResourceServiceLease | undefined {
   }
 }
 
-function leaseIsLive(lease: ResourceServiceLease): boolean {
-  if (lease.host !== hostname()) return true;
-  try {
-    process.kill(lease.pid, 0);
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
-  }
-  return checkPidToken(lease.pid, lease.start_token) !== "mismatch";
+function leaseIsLive(lease: ResourceServiceLease): ResourceServiceStatus["liveness"] {
+  return resourceProcessLiveness(lease).liveness;
+}
+
+function assertResourceLeaseAvailable(path: string): void {
+  if (!existsSync(path)) return;
+  const lease = readLease(path);
+  const liveness = lease ? leaseIsLive(lease) : "unknown";
+  if (liveness === "unknown") throw unknownResourceOwnerError();
+  if (liveness === "running")
+    throw new Error(`resource observer lease is held by pid ${lease?.pid}`);
+}
+
+function unknownResourceOwnerError(): Error {
+  return new Error(
+    `resource observer process identity is unknown; run ${resolveBinName()} supervisor stop to write the durable stop request before starting again`,
+  );
 }
 
 function log(

package/src/core/resources/status.ts

@@ -10,8 +10,9 @@ import {
   SUPERVISOR_PRESSURE_SCHEMA_VERSION,
   type SupervisorCapability,
   type SupervisorPressureRecord,
+  type SupervisorStatus,
 } from "../supervisor/contract.ts";
-import { readSupervisorStatus } from "../supervisor/status.ts";
+import { installedSupervisorBuild, readSupervisorStatus } from "../supervisor/status.ts";
 import { supervisorPaths } from "../supervisor/storage.ts";
 import {
   RESOURCE_SNAPSHOT_SCHEMA_VERSION,
@@ -43,7 +44,17 @@ export interface ResourceStatus {
   platform: NodeJS.Platform | null;
   namespace: ResourceSnapshot["namespace"];
   support: ResourceSnapshot["support"] | null;
-  writer: { running: boolean; stale: boolean };
+  writer: Pick<
+    SupervisorStatus,
+    | "running"
+    | "stale"
+    | "generation"
+    | "liveness"
+    | "running_build"
+    | "installed_build"
+    | "restart_needed"
+    | "build_status"
+  >;
   /**
    * The one published pressure assessment, read from the observer's record so
    * this surface, the prompt notice, the dashboard, and a diagnostic bundle all
@@ -222,12 +233,16 @@ export function formatResourceSummary(status: ResourceStatus): string {
       ? "unknown"
       : `${formatBytes(m.memory_available_bytes)} available`;
   const diskText = disk ? `${formatBytes(disk.available_bytes as number)} available` : "unknown";
-  return `${lead} ${status.namespace}/${status.platform}: CPU ${cpu}, RAM ${ram}, disk ${diskText}; ${Math.round((status.sample_age_ms ?? 0) / 1_000)}s old${status.support?.state === "partial" ? "; partial" : ""}${!status.writer.running ? "; writer stopped" : ""}.`;
+  return `${lead} ${status.namespace}/${status.platform}: CPU ${cpu}, RAM ${ram}, disk ${diskText}; ${Math.round((status.sample_age_ms ?? 0) / 1_000)}s old${status.support?.state === "partial" ? "; partial" : ""}${!status.writer.running ? `; writer ${status.writer.liveness === "unknown" ? "unknown" : status.writer.stale ? "stale" : "stopped"}` : ""}.`;
 }
 
 export function formatResourceStatus(status: ResourceStatus, bin: string): string {
   const a = status.assessment;
   const lines = [formatResourceSummary(status)];
+  const writer = status.writer;
+  lines.push(
+    `Writer: ${writer.liveness}; generation ${writer.generation}; running build ${writer.running_build ? `${writer.running_build.version} (${writer.running_build.revision})` : "unknown"}; installed build ${writer.installed_build.version} (${writer.installed_build.revision}); ${writer.build_status}.`,
+  );
   lines.push(
     `Assessment: ${a.state} at ${a.scope} scope; limiting resource ${a.limiting_resource}; trend ${a.trend}; evidence ${a.evidence_state}; recommended action ${a.recommended_action}.`,
   );
@@ -247,7 +262,11 @@ export function formatResourceStatus(status: ResourceStatus, bin: string): strin
     lines.push(
       `Unattributed memory: ${Math.round(a.unattributed_memory_percent)}% of machine memory has no validated owner.`,
     );
-  if (status.state !== "fresh" || !status.writer.running)
+  if (status.writer.liveness === "unknown")
+    lines.push(
+      `Inspect the writer with ${bin} supervisor status. Run ${bin} supervisor stop to write its durable stop request before starting again.`,
+    );
+  else if (status.state !== "fresh" || !status.writer.running)
     lines.push(
       `Start or inspect the writer with ${bin} supervisor start --keep-alive and ${bin} supervisor status.`,
     );
@@ -345,7 +364,15 @@ export function readPublishedPressure(
   });
   // A record can only be trusted while its writer is alive. Every surface makes
   // the same check here so none of them reports a state the others do not.
-  if (!(writer ?? readWriterState(coordRoot, nowMs)).running) {
+  const observer = writer ?? readWriterState(coordRoot, nowMs);
+  if (observer.liveness === "unknown") {
+    return gap(
+      "error",
+      "pressure_observer_unknown",
+      "Local resource pressure cannot be determined because the resource observer's process identity is unknown.",
+    );
+  }
+  if (!observer.running) {
     return gap(
       "expired",
       "pressure_observer_not_running",
@@ -407,7 +434,7 @@ function readWriterState(coordRoot: string, nowMs: number): ResourceStatus["writ
     // Keep malformed service metadata from making the read expensive. The shared
     // status reader retains ownership of PID/start-token liveness checks.
     if (statSync(supervisorPaths(coordRoot).service).size > 65_536) {
-      return { running: false, stale: false };
+      return unknownWriterState();
     }
     const writer = readSupervisorStatus(coordRoot, nowMs);
     const futureHeartbeat =
@@ -415,13 +442,35 @@ function readWriterState(coordRoot: string, nowMs: number): ResourceStatus["writ
     return {
       running: writer.running && !futureHeartbeat,
       stale: writer.stale || !!futureHeartbeat,
+      generation: writer.generation,
+      liveness: writer.liveness,
+      running_build: writer.running_build,
+      installed_build: writer.installed_build,
+      restart_needed: writer.restart_needed,
+      build_status: writer.build_status,
     };
-  } catch {
+  } catch (error) {
     // A missing writer must not break normal coordination.
-    return { running: false, stale: false };
+    return {
+      ...unknownWriterState(),
+      liveness: (error as NodeJS.ErrnoException).code === "ENOENT" ? "stopped" : "unknown",
+    };
   }
 }
 
+function unknownWriterState(): ResourceStatus["writer"] {
+  return {
+    running: false,
+    stale: false,
+    generation: "unknown",
+    liveness: "unknown",
+    running_build: null,
+    installed_build: installedSupervisorBuild(),
+    restart_needed: true,
+    build_status: "running build unknown; restart needed",
+  };
+}
+
 function validPressureRecord(v: unknown): v is SupervisorPressureRecord {
   if (
     !object(v) ||