harnery 0.43.0 → 0.45.0

package/src/core/resources/service.ts

@@ -1,4 +1,3 @@
-import { spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import {
   closeSync,
@@ -10,9 +9,15 @@ import {
   unlinkSync,
   writeFileSync,
 } from "node:fs";
-import { hostname, platform } from "node:os";
+import { hostname } from "node:os";
 import { resolve } from "node:path";
-import { checkPidToken, processStartToken } from "../agents/state/proc-start.ts";
+import {
+  type ProcessGenerationSignalDependencies,
+  type ProcessGenerationVerdict,
+  processStartToken,
+  signalIfSameGeneration,
+} from "../agents/state/proc-start.ts";
+import { resolveBinName } from "../config.ts";
 import { closeProcessLoggers, legacyLogFields, processLogger } from "../storage/logger.ts";
 import { stateDirMode, stateFileMode } from "../storage/modes.ts";
 import {
@@ -22,7 +27,11 @@ import {
   type ResourceServiceStatusRecord,
 } from "./contract.ts";
 import { sampleResources } from "./sampler.ts";
-import { readResourceServiceStatus } from "./service-status.ts";
+import {
+  type ResourceProcessDependencies,
+  readResourceServiceStatus,
+  resourceProcessLiveness,
+} from "./service-status.ts";
 import { resourcePaths, writePrivateJsonAtomic } from "./storage.ts";
 
 export const RESOURCE_SERVICE_DEFAULT_INTERVAL_MS = 2_000;
@@ -46,102 +55,47 @@ export interface RunResourceServiceInput {
   wait?: (milliseconds: number) => Promise<void>;
 }
 
-export type EnsureResourceServiceResult =
-  | { state: "running" | "started"; status: ResourceServiceStatus }
-  | { state: "unsupported" | "unavailable"; status: ResourceServiceStatus; error: string };
-
-export async function spawnResourceService(
-  coordRootRaw: string,
-  options: { intervalMs?: number } = {},
-): Promise<ResourceServiceStatus> {
-  const coordRoot = resolve(coordRootRaw);
-  const current = readResourceServiceStatus(coordRoot);
-  if (current.running) {
-    throw new Error(`resource observer is already running under pid ${current.record?.pid}`);
-  }
-  const intervalMs = normalizeInterval(options.intervalMs);
-  const paths = resourcePaths(coordRoot);
-  mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
-  rmSync(paths.stop, { force: true });
-  const harnBin = new URL("../../../bin/harn", import.meta.url).pathname;
-  if (!existsSync(harnBin)) throw new Error(`cannot find harn executable at ${harnBin}`);
-  let spawnError: Error | undefined;
-  const child = spawn(
-    harnBin,
-    ["resources", "service", "daemon", "--root", coordRoot, "--interval-ms", String(intervalMs)],
-    {
-      cwd: coordRoot,
-      detached: true,
-      stdio: ["ignore", "ignore", "ignore"],
-      env: {
-        ...process.env,
-        HARNERY_COORD_ROOT_OVERRIDE: coordRoot,
-        HARNERY_OUTPUT_SESSION_TEE: "0",
-      },
-    },
-  );
-  child.once("error", (error) => {
-    spawnError = error;
-  });
-  child.unref();
-  const deadline = Date.now() + 5_000;
-  while (Date.now() < deadline) {
-    await delay(50);
-    if (spawnError) throw spawnError;
-    const status = readResourceServiceStatus(coordRoot);
-    if (status.running && status.record?.pid === child.pid) return status;
-    if (child.exitCode !== null) break;
-  }
-  throw new Error(`resource observer failed to start; inspect ${paths.service}`);
-}
-
-export async function ensureResourceServiceRunning(
-  coordRoot: string,
-): Promise<EnsureResourceServiceResult> {
-  const status = readResourceServiceStatus(coordRoot);
-  if (status.running) return { state: "running", status };
-  if (platform() !== "linux") {
-    return {
-      state: "unsupported",
-      status,
-      error: `resource observer does not yet support ${platform()}`,
-    };
-  }
-  try {
-    const started = await spawnResourceService(coordRoot);
-    return { state: "started", status: started };
-  } catch (error) {
-    return {
-      state: "unavailable",
-      status: readResourceServiceStatus(coordRoot),
-      error: error instanceof Error ? error.message : String(error),
-    };
-  }
+export interface ResourceServiceStopResult extends ResourceServiceStatus {
+  signal_sent: boolean;
+  signal_verdict: ProcessGenerationVerdict;
 }
 
-export function requestResourceServiceStop(coordRootRaw: string): ResourceServiceStatus {
+export function requestResourceServiceStop(
+  coordRootRaw: string,
+  dependencies: ResourceProcessDependencies & ProcessGenerationSignalDependencies = {},
+): ResourceServiceStopResult {
   const coordRoot = resolve(coordRootRaw);
   const paths = resourcePaths(coordRoot);
-  const status = readResourceServiceStatus(coordRoot);
+  const status = readResourceServiceStatus(coordRoot, Date.now(), dependencies);
   mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
   writePrivateJsonAtomic(paths.stop, {
     requested_at: new Date().toISOString(),
     requested_by_pid: process.pid,
   });
-  if (status.running && status.record?.host === hostname()) {
-    try {
-      process.kill(status.record.pid, "SIGTERM");
-    } catch {
-      // The durable stop file handles a racing exit.
-    }
-  }
-  return readResourceServiceStatus(coordRoot);
+  const signal =
+    status.liveness === "running" && status.record?.host === hostname()
+      ? signalIfSameGeneration(
+          status.record.pid,
+          status.record.start_token,
+          "SIGTERM",
+          dependencies,
+        )
+      : { sent: false, verdict: status.generation };
+  return {
+    ...readResourceServiceStatus(coordRoot, Date.now(), dependencies),
+    signal_sent: signal.sent,
+    signal_verdict: signal.verdict,
+  };
 }
 
 export async function runResourceService(
   input: RunResourceServiceInput,
 ): Promise<ResourceServiceStatusRecord> {
   const coordRoot = resolve(input.coordRoot);
+  const current = readResourceServiceStatus(coordRoot);
+  if (current.liveness === "unknown") throw unknownResourceOwnerError();
+  if (current.liveness === "running")
+    throw new Error(`resource observer is already running under pid ${current.record?.pid}`);
   const paths = resourcePaths(coordRoot);
   const intervalMs = normalizeInterval(input.intervalMs);
   const now = input.now ?? (() => new Date());
@@ -253,7 +207,9 @@ function acquireLease(coordRoot: string, now: Date): ResourceServiceLease {
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
       const existing = readLease(paths.lease);
-      if (existing && leaseIsLive(existing)) {
+      const liveness = existing ? leaseIsLive(existing) : "unknown";
+      if (liveness === "unknown") throw unknownResourceOwnerError();
+      if (existing && liveness === "running") {
         throw new Error(`resource observer lease is held by pid ${existing.pid}`);
       }
       try {
@@ -279,7 +235,13 @@ function releaseLease(path: string, nonce: string): void {
 function readLease(path: string): ResourceServiceLease | undefined {
   try {
     const value = JSON.parse(readFileSync(path, "utf8")) as ResourceServiceLease;
-    return value && Number.isInteger(value.pid) && typeof value.nonce === "string"
+    return value &&
+      Number.isSafeInteger(value.pid) &&
+      value.pid > 0 &&
+      typeof value.host === "string" &&
+      value.host.length > 0 &&
+      typeof value.nonce === "string" &&
+      value.nonce.length > 0
       ? value
       : undefined;
   } catch {
@@ -287,14 +249,14 @@ function readLease(path: string): ResourceServiceLease | undefined {
   }
 }
 
-function leaseIsLive(lease: ResourceServiceLease): boolean {
-  if (lease.host !== hostname()) return true;
-  try {
-    process.kill(lease.pid, 0);
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
-  }
-  return checkPidToken(lease.pid, lease.start_token) !== "mismatch";
+function leaseIsLive(lease: ResourceServiceLease): ResourceServiceStatus["liveness"] {
+  return resourceProcessLiveness(lease).liveness;
+}
+
+function unknownResourceOwnerError(): Error {
+  return new Error(
+    `resource observer process identity is unknown; run ${resolveBinName()} supervisor stop to write the durable stop request before starting again`,
+  );
 }
 
 function log(

package/src/core/resources/status.ts

@@ -10,8 +10,9 @@ import {
   SUPERVISOR_PRESSURE_SCHEMA_VERSION,
   type SupervisorCapability,
   type SupervisorPressureRecord,
+  type SupervisorStatus,
 } from "../supervisor/contract.ts";
-import { readSupervisorStatus } from "../supervisor/status.ts";
+import { installedSupervisorBuild, readSupervisorStatus } from "../supervisor/status.ts";
 import { supervisorPaths } from "../supervisor/storage.ts";
 import {
   RESOURCE_SNAPSHOT_SCHEMA_VERSION,
@@ -43,7 +44,17 @@ export interface ResourceStatus {
   platform: NodeJS.Platform | null;
   namespace: ResourceSnapshot["namespace"];
   support: ResourceSnapshot["support"] | null;
-  writer: { running: boolean; stale: boolean };
+  writer: Pick<
+    SupervisorStatus,
+    | "running"
+    | "stale"
+    | "generation"
+    | "liveness"
+    | "running_build"
+    | "installed_build"
+    | "restart_needed"
+    | "build_status"
+  >;
   /**
    * The one published pressure assessment, read from the observer's record so
    * this surface, the prompt notice, the dashboard, and a diagnostic bundle all
@@ -222,12 +233,16 @@ export function formatResourceSummary(status: ResourceStatus): string {
       ? "unknown"
       : `${formatBytes(m.memory_available_bytes)} available`;
   const diskText = disk ? `${formatBytes(disk.available_bytes as number)} available` : "unknown";
-  return `${lead} ${status.namespace}/${status.platform}: CPU ${cpu}, RAM ${ram}, disk ${diskText}; ${Math.round((status.sample_age_ms ?? 0) / 1_000)}s old${status.support?.state === "partial" ? "; partial" : ""}${!status.writer.running ? "; writer stopped" : ""}.`;
+  return `${lead} ${status.namespace}/${status.platform}: CPU ${cpu}, RAM ${ram}, disk ${diskText}; ${Math.round((status.sample_age_ms ?? 0) / 1_000)}s old${status.support?.state === "partial" ? "; partial" : ""}${!status.writer.running ? `; writer ${status.writer.liveness === "unknown" ? "unknown" : status.writer.stale ? "stale" : "stopped"}` : ""}.`;
 }
 
 export function formatResourceStatus(status: ResourceStatus, bin: string): string {
   const a = status.assessment;
   const lines = [formatResourceSummary(status)];
+  const writer = status.writer;
+  lines.push(
+    `Writer: ${writer.liveness}; generation ${writer.generation}; running build ${writer.running_build ? `${writer.running_build.version} (${writer.running_build.revision})` : "unknown"}; installed build ${writer.installed_build.version} (${writer.installed_build.revision}); ${writer.build_status}.`,
+  );
   lines.push(
     `Assessment: ${a.state} at ${a.scope} scope; limiting resource ${a.limiting_resource}; trend ${a.trend}; evidence ${a.evidence_state}; recommended action ${a.recommended_action}.`,
   );
@@ -247,7 +262,11 @@ export function formatResourceStatus(status: ResourceStatus, bin: string): strin
     lines.push(
       `Unattributed memory: ${Math.round(a.unattributed_memory_percent)}% of machine memory has no validated owner.`,
     );
-  if (status.state !== "fresh" || !status.writer.running)
+  if (status.writer.liveness === "unknown")
+    lines.push(
+      `Inspect the writer with ${bin} supervisor status. Run ${bin} supervisor stop to write its durable stop request before starting again.`,
+    );
+  else if (status.state !== "fresh" || !status.writer.running)
     lines.push(
       `Start or inspect the writer with ${bin} supervisor start --keep-alive and ${bin} supervisor status.`,
     );
@@ -345,7 +364,15 @@ export function readPublishedPressure(
   });
   // A record can only be trusted while its writer is alive. Every surface makes
   // the same check here so none of them reports a state the others do not.
-  if (!(writer ?? readWriterState(coordRoot, nowMs)).running) {
+  const observer = writer ?? readWriterState(coordRoot, nowMs);
+  if (observer.liveness === "unknown") {
+    return gap(
+      "error",
+      "pressure_observer_unknown",
+      "Local resource pressure cannot be determined because the resource observer's process identity is unknown.",
+    );
+  }
+  if (!observer.running) {
     return gap(
       "expired",
       "pressure_observer_not_running",
@@ -407,7 +434,7 @@ function readWriterState(coordRoot: string, nowMs: number): ResourceStatus["writ
     // Keep malformed service metadata from making the read expensive. The shared
     // status reader retains ownership of PID/start-token liveness checks.
     if (statSync(supervisorPaths(coordRoot).service).size > 65_536) {
-      return { running: false, stale: false };
+      return unknownWriterState();
     }
     const writer = readSupervisorStatus(coordRoot, nowMs);
     const futureHeartbeat =
@@ -415,13 +442,35 @@ function readWriterState(coordRoot: string, nowMs: number): ResourceStatus["writ
     return {
       running: writer.running && !futureHeartbeat,
       stale: writer.stale || !!futureHeartbeat,
+      generation: writer.generation,
+      liveness: writer.liveness,
+      running_build: writer.running_build,
+      installed_build: writer.installed_build,
+      restart_needed: writer.restart_needed,
+      build_status: writer.build_status,
     };
-  } catch {
+  } catch (error) {
     // A missing writer must not break normal coordination.
-    return { running: false, stale: false };
+    return {
+      ...unknownWriterState(),
+      liveness: (error as NodeJS.ErrnoException).code === "ENOENT" ? "stopped" : "unknown",
+    };
   }
 }
 
+function unknownWriterState(): ResourceStatus["writer"] {
+  return {
+    running: false,
+    stale: false,
+    generation: "unknown",
+    liveness: "unknown",
+    running_build: null,
+    installed_build: installedSupervisorBuild(),
+    restart_needed: true,
+    build_status: "running build unknown; restart needed",
+  };
+}
+
 function validPressureRecord(v: unknown): v is SupervisorPressureRecord {
   if (
     !object(v) ||

package/src/core/semantic/service.ts

@@ -225,19 +225,13 @@ export async function ensureSemanticServiceRunning(
 
 export function requestSemanticServiceStop(coordRootRaw: string): SemanticServiceStatus {
   const coordRoot = resolve(coordRootRaw);
-  const status = readSemanticServiceStatus(coordRoot);
   mkdirSync(semanticPaths(coordRoot).root, { recursive: true, mode: stateDirMode() });
   writePrivateJsonAtomic(semanticPaths(coordRoot).stop, {
     requested_at: new Date().toISOString(),
     requested_by_pid: process.pid,
   });
-  if (status.running && status.record?.host === hostname()) {
-    try {
-      process.kill(status.record.pid, "SIGTERM");
-    } catch {
-      // The durable stop request remains for a racing or restarted daemon.
-    }
-  }
+  // This record has no generation witness. The durable request is the stop
+  // authority; a numeric PID cannot authorize an additional signal.
   return readSemanticServiceStatus(coordRoot);
 }