harnery 0.36.0 → 0.38.0

package/src/core/resources/service.ts

@@ -0,0 +1,337 @@
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  rmSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { hostname, platform } from "node:os";
+import { resolve } from "node:path";
+import { checkPidToken, processStartToken } from "../agents/state/proc-start.ts";
+import { closeProcessLoggers, legacyLogFields, processLogger } from "../storage/logger.ts";
+import {
+  RESOURCE_SERVICE_STATUS_SCHEMA_VERSION,
+  type ResourceSamplerState,
+  type ResourceServiceStatus,
+  type ResourceServiceStatusRecord,
+} from "./contract.ts";
+import { sampleResources } from "./sampler.ts";
+import { readResourceServiceStatus } from "./service-status.ts";
+import { resourcePaths, writePrivateJsonAtomic } from "./storage.ts";
+
+export const RESOURCE_SERVICE_DEFAULT_INTERVAL_MS = 2_000;
+const RESOURCE_SERVICE_MIN_INTERVAL_MS = 500;
+const RESOURCE_SERVICE_MAX_INTERVAL_MS = 60_000;
+
+interface ResourceServiceLease {
+  pid: number;
+  start_token?: string;
+  host: string;
+  nonce: string;
+  created_at: string;
+}
+
+export interface RunResourceServiceInput {
+  coordRoot: string;
+  intervalMs?: number;
+  maxSamples?: number;
+  now?: () => Date;
+  sample?: typeof sampleResources;
+  wait?: (milliseconds: number) => Promise<void>;
+}
+
+export type EnsureResourceServiceResult =
+  | { state: "running" | "started"; status: ResourceServiceStatus }
+  | { state: "unsupported" | "unavailable"; status: ResourceServiceStatus; error: string };
+
+export async function spawnResourceService(
+  coordRootRaw: string,
+  options: { intervalMs?: number } = {},
+): Promise<ResourceServiceStatus> {
+  const coordRoot = resolve(coordRootRaw);
+  const current = readResourceServiceStatus(coordRoot);
+  if (current.running) {
+    throw new Error(`resource observer is already running under pid ${current.record?.pid}`);
+  }
+  const intervalMs = normalizeInterval(options.intervalMs);
+  const paths = resourcePaths(coordRoot);
+  mkdirSync(paths.root, { recursive: true, mode: 0o700 });
+  rmSync(paths.stop, { force: true });
+  const harnBin = new URL("../../../bin/harn", import.meta.url).pathname;
+  if (!existsSync(harnBin)) throw new Error(`cannot find harn executable at ${harnBin}`);
+  let spawnError: Error | undefined;
+  const child = spawn(
+    harnBin,
+    ["resources", "service", "daemon", "--root", coordRoot, "--interval-ms", String(intervalMs)],
+    {
+      cwd: coordRoot,
+      detached: true,
+      stdio: ["ignore", "ignore", "ignore"],
+      env: {
+        ...process.env,
+        HARNERY_COORD_ROOT_OVERRIDE: coordRoot,
+        HARNERY_OUTPUT_SESSION_TEE: "0",
+      },
+    },
+  );
+  child.once("error", (error) => {
+    spawnError = error;
+  });
+  child.unref();
+  const deadline = Date.now() + 5_000;
+  while (Date.now() < deadline) {
+    await delay(50);
+    if (spawnError) throw spawnError;
+    const status = readResourceServiceStatus(coordRoot);
+    if (status.running && status.record?.pid === child.pid) return status;
+    if (child.exitCode !== null) break;
+  }
+  throw new Error(`resource observer failed to start; inspect ${paths.service}`);
+}
+
+export async function ensureResourceServiceRunning(
+  coordRoot: string,
+): Promise<EnsureResourceServiceResult> {
+  const status = readResourceServiceStatus(coordRoot);
+  if (status.running) return { state: "running", status };
+  if (platform() !== "linux") {
+    return {
+      state: "unsupported",
+      status,
+      error: `resource observer does not yet support ${platform()}`,
+    };
+  }
+  try {
+    const started = await spawnResourceService(coordRoot);
+    return { state: "started", status: started };
+  } catch (error) {
+    return {
+      state: "unavailable",
+      status: readResourceServiceStatus(coordRoot),
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+export function requestResourceServiceStop(coordRootRaw: string): ResourceServiceStatus {
+  const coordRoot = resolve(coordRootRaw);
+  const paths = resourcePaths(coordRoot);
+  const status = readResourceServiceStatus(coordRoot);
+  mkdirSync(paths.root, { recursive: true, mode: 0o700 });
+  writePrivateJsonAtomic(paths.stop, {
+    requested_at: new Date().toISOString(),
+    requested_by_pid: process.pid,
+  });
+  if (status.running && status.record?.host === hostname()) {
+    try {
+      process.kill(status.record.pid, "SIGTERM");
+    } catch {
+      // The durable stop file handles a racing exit.
+    }
+  }
+  return readResourceServiceStatus(coordRoot);
+}
+
+export async function runResourceService(
+  input: RunResourceServiceInput,
+): Promise<ResourceServiceStatusRecord> {
+  const coordRoot = resolve(input.coordRoot);
+  const paths = resourcePaths(coordRoot);
+  const intervalMs = normalizeInterval(input.intervalMs);
+  const now = input.now ?? (() => new Date());
+  const sample = input.sample ?? sampleResources;
+  const wait = input.wait ?? delay;
+  const lease = acquireLease(coordRoot, now());
+  rmSync(paths.stop, { force: true });
+  const startedAt = now().toISOString();
+  const daemonStartToken = processStartToken(process.pid);
+  const status: ResourceServiceStatusRecord = {
+    schema_version: RESOURCE_SERVICE_STATUS_SCHEMA_VERSION,
+    pid: process.pid,
+    ...(daemonStartToken ? { start_token: daemonStartToken } : {}),
+    host: hostname(),
+    nonce: lease.nonce,
+    state: "starting",
+    started_at: startedAt,
+    heartbeat_at: startedAt,
+    interval_ms: intervalMs,
+    sample_count: 0,
+  };
+  let stopping = false;
+  let previous: ResourceSamplerState | undefined;
+  const writeStatus = () => {
+    status.heartbeat_at = now().toISOString();
+    writePrivateJsonAtomic(paths.service, status);
+  };
+  const requestStop = () => {
+    stopping = true;
+    status.state = "stopping";
+    writeStatus();
+  };
+  process.on("SIGINT", requestStop);
+  process.on("SIGTERM", requestStop);
+  status.state = "running";
+  writeStatus();
+  log(coordRoot, "resource_observer.started", { pid: process.pid, interval_ms: intervalMs });
+  try {
+    while (!stopping && !existsSync(paths.stop)) {
+      const cycleStarted = Date.now();
+      try {
+        const result = sample(coordRoot, previous);
+        previous = result.state;
+        writePrivateJsonAtomic(paths.snapshot, result.snapshot);
+        status.sample_count += 1;
+        status.last_sample_at = result.snapshot.sampled_at;
+        status.last_error_code = undefined;
+        log(coordRoot, "resource_observer.sample", {
+          sample_count: status.sample_count,
+          cpu_percent: result.snapshot.machine.cpu_percent ?? -1,
+          memory_percent: result.snapshot.machine.memory_percent ?? -1,
+          visible_processes: result.snapshot.visible_process_count,
+          unattributed_processes: result.snapshot.unattributed_process_count,
+          duration_ms: result.snapshot.sample_duration_ms,
+          collector_cpu_ms: result.snapshot.collector_cpu_ms,
+        });
+      } catch (error) {
+        status.last_error_code = resourceErrorCode(error);
+        status.state = "error";
+        log(
+          coordRoot,
+          "resource_observer.sample_error",
+          { reason_code: status.last_error_code },
+          error,
+        );
+      }
+      if (status.state === "error") status.state = "running";
+      writeStatus();
+      if (input.maxSamples !== undefined && status.sample_count >= input.maxSamples) break;
+      const remaining = Math.max(0, intervalMs - (Date.now() - cycleStarted));
+      if (!stopping && !existsSync(paths.stop)) await wait(remaining);
+    }
+  } finally {
+    process.off("SIGINT", requestStop);
+    process.off("SIGTERM", requestStop);
+    status.state = "stopped";
+    status.stopped_at = now().toISOString();
+    writeStatus();
+    log(coordRoot, "resource_observer.stopped", { sample_count: status.sample_count });
+    try {
+      await closeProcessLoggers();
+    } finally {
+      releaseLease(paths.lease, lease.nonce);
+    }
+  }
+  return status;
+}
+
+function acquireLease(coordRoot: string, now: Date): ResourceServiceLease {
+  const paths = resourcePaths(coordRoot);
+  mkdirSync(paths.root, { recursive: true, mode: 0o700 });
+  for (let attempt = 0; attempt < 2; attempt++) {
+    const daemonStartToken = processStartToken(process.pid);
+    const lease: ResourceServiceLease = {
+      pid: process.pid,
+      ...(daemonStartToken ? { start_token: daemonStartToken } : {}),
+      host: hostname(),
+      nonce: randomUUID(),
+      created_at: now.toISOString(),
+    };
+    try {
+      const fd = openSync(paths.lease, "wx", 0o600);
+      try {
+        writeFileSync(fd, `${JSON.stringify(lease)}\n`, "utf8");
+      } finally {
+        closeSync(fd);
+      }
+      return lease;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      const existing = readLease(paths.lease);
+      if (existing && leaseIsLive(existing)) {
+        throw new Error(`resource observer lease is held by pid ${existing.pid}`);
+      }
+      try {
+        unlinkSync(paths.lease);
+      } catch {
+        // A concurrent starter may have replaced or removed the stale lease.
+      }
+    }
+  }
+  throw new Error("resource observer could not acquire its singleton lease");
+}
+
+function releaseLease(path: string, nonce: string): void {
+  const current = readLease(path);
+  if (current?.nonce !== nonce) return;
+  try {
+    unlinkSync(path);
+  } catch {
+    // A missing lease is already released.
+  }
+}
+
+function readLease(path: string): ResourceServiceLease | undefined {
+  try {
+    const value = JSON.parse(readFileSync(path, "utf8")) as ResourceServiceLease;
+    return value && Number.isInteger(value.pid) && typeof value.nonce === "string"
+      ? value
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function leaseIsLive(lease: ResourceServiceLease): boolean {
+  if (lease.host !== hostname()) return true;
+  try {
+    process.kill(lease.pid, 0);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
+  }
+  return checkPidToken(lease.pid, lease.start_token) !== "mismatch";
+}
+
+function log(
+  coordRoot: string,
+  event: string,
+  fields: Record<string, unknown>,
+  error?: unknown,
+): void {
+  try {
+    const logger = processLogger(coordRoot, "resource-observer");
+    if (error) logger.error(event, legacyLogFields(fields), error);
+    else logger.info(event, legacyLogFields(fields));
+  } catch {
+    // Observability must never break the observer loop.
+  }
+}
+
+function resourceErrorCode(error: unknown): string {
+  const message = error instanceof Error ? error.message : String(error);
+  if (/procfs|\/proc\//i.test(message)) return "procfs_read_failed";
+  if (/permission|EACCES|EPERM/i.test(message)) return "permission_denied";
+  return "sample_failed";
+}
+
+function normalizeInterval(value: number | undefined): number {
+  const interval = value ?? RESOURCE_SERVICE_DEFAULT_INTERVAL_MS;
+  if (
+    !Number.isFinite(interval) ||
+    interval < RESOURCE_SERVICE_MIN_INTERVAL_MS ||
+    interval > RESOURCE_SERVICE_MAX_INTERVAL_MS
+  ) {
+    throw new Error(
+      `resource observer interval must be ${RESOURCE_SERVICE_MIN_INTERVAL_MS}-${RESOURCE_SERVICE_MAX_INTERVAL_MS} ms`,
+    );
+  }
+  return Math.round(interval);
+}
+
+function delay(milliseconds: number): Promise<void> {
+  return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds));
+}

package/src/core/resources/storage.ts

@@ -0,0 +1,44 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import type { ResourceServiceStatusRecord, ResourceSnapshot } from "./contract.ts";
+
+export { writePrivateJsonAtomic } from "../storage/atomic-json.ts";
+
+export interface ResourcePaths {
+  root: string;
+  service: string;
+  lease: string;
+  stop: string;
+  snapshot: string;
+}
+
+export function resourcePaths(coordRootRaw: string): ResourcePaths {
+  const root = join(resolve(coordRootRaw), ".harnery", "resources");
+  return {
+    root,
+    service: join(root, "service.json"),
+    lease: join(root, "service.lease.json"),
+    stop: join(root, "stop.json"),
+    snapshot: join(root, "snapshot.json"),
+  };
+}
+
+export function readResourceSnapshot(coordRoot: string): ResourceSnapshot | undefined {
+  return readJson<ResourceSnapshot>(resourcePaths(coordRoot).snapshot);
+}
+
+export function readResourceServiceRecord(
+  coordRoot: string,
+): ResourceServiceStatusRecord | undefined {
+  return readJson<ResourceServiceStatusRecord>(resourcePaths(coordRoot).service);
+}
+
+function readJson<T>(path: string): T | undefined {
+  if (!existsSync(path)) return undefined;
+  try {
+    const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
+    return value && typeof value === "object" ? (value as T) : undefined;
+  } catch {
+    return undefined;
+  }
+}

package/src/core/storage/atomic-json.ts

@@ -0,0 +1,15 @@
+import { chmodSync, mkdirSync, renameSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+/** Write one private JSON projection without exposing a partial frame to readers. */
+export function writePrivateJsonAtomic(path: string, value: unknown): void {
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  const temporary = `${path}.tmp.${process.pid}.${Date.now()}`;
+  writeFileSync(temporary, `${JSON.stringify(value)}\n`, { encoding: "utf8", mode: 0o600 });
+  try {
+    chmodSync(temporary, 0o600);
+  } catch {
+    // Windows does not expose POSIX modes. The atomic write still applies.
+  }
+  renameSync(temporary, path);
+}

package/src/core/storage/builtins.ts

@@ -35,6 +35,8 @@ const LOG_PARTITIONS = [
   "agent-operational",
   "web-performance",
   "semantic-service",
+  "resource-observer",
+  "supervisor",
   "governor-service",
   "presence-relay",
   "tunnel-process",
@@ -59,8 +61,8 @@ const V3_ARCHIVE_PROVIDER: HarneryStorageProvider = {
   provider_id: "event-v3-archive-provider",
   kind: "filesystem",
   inventory: "filesystem",
-  maintenance: "none",
-  lifecycle_authority: "Event Ledger V3 logical authority reader",
+  maintenance: "delegated",
+  lifecycle_authority: "Event Ledger V3 closed-epoch retention policy",
   partitions: ["canonical", "support"],
 };
 
@@ -81,7 +83,7 @@ function eventFamilies(): HarneryStorageFamily[] {
     family({
       id: "event-v3-canonical-active",
       owner: "Event Ledger V3 authority",
-      storage_class: "canonical-authority",
+      storage_class: "durable-object-history",
       roots: (context) => [
         exact(context, ".harnery/ledgers/v3/active.ndjson", "file"),
         exact(context, ".harnery/ledgers/v3/catalog.json", "file"),
@@ -112,14 +114,14 @@ function eventFamilies(): HarneryStorageFamily[] {
       format: "canonical-ndjson",
       durability: "immutable",
       writer_model: "object-owned",
-      policy: authorityPolicy("event-v3-archive-canonical-v1"),
+      policy: closedEpochPolicy("event-v3-archive-canonical-v2"),
       consumers: ["Event Ledger V3 logical authority reader", "recovery verifier"],
       provider: V3_ARCHIVE_PROVIDER,
     }),
     family({
       id: "event-v3-support-active",
       owner: "Event Ledger V3 producer recovery",
-      storage_class: "recovery-state",
+      storage_class: "durable-object-history",
       roots: (context) => [
         subtree(context, ".harnery/ledgers/v3/diagnostics", OWNER_PROTOCOL_LINKS),
         subtree(context, ".harnery/ledgers/v3/diagnostic-summaries", OWNER_PROTOCOL_LINKS),
@@ -167,7 +169,7 @@ function eventFamilies(): HarneryStorageFamily[] {
       format: "files",
       durability: "immutable",
       writer_model: "object-owned",
-      policy: recoveryPolicy("event-v3-archive-support-v1"),
+      policy: closedEpochPolicy("event-v3-archive-support-v2"),
       consumers: ["Event Ledger V3 support reader", "recovery verifier", "agents health"],
       provider: V3_ARCHIVE_PROVIDER,
     }),
@@ -304,6 +306,14 @@ function logFamilies(): HarneryStorageFamily[] {
       ),
       (context) => [exact(context, ".harnery/semantic/v2/service.log", "file")],
     ),
+    operationalFamily(
+      "resource-observer-log",
+      "resource observer",
+      "resource-observer",
+      14,
+      64 * MIB,
+    ),
+    operationalFamily("supervisor-log", "local supervisor", "supervisor", 14, 64 * MIB),
     withCurrentRoots(
       operationalFamily(
         "governor-service-log",
@@ -463,6 +473,39 @@ function cacheFamilies(): HarneryStorageFamily[] {
         "canonical events and source documents",
       ),
     }),
+    family({
+      id: "resource-observer-cache",
+      owner: "resource observer",
+      storage_class: "repairable-cache",
+      roots: (context) => [subtree(context, ".harnery/resources")],
+      format: "json",
+      durability: "reconstructable",
+      writer_model: "object-owned",
+      policy: cachePolicy("resource-observer-cache-v1", "local process and machine samples"),
+      consumers: ["resource observer", "dashboard", "storage inventory", "storage health"],
+      provider: filesystemProvider(
+        "resource-observer-cache-provider",
+        "local process and machine samples",
+      ),
+    }),
+    family({
+      id: "supervisor-cache",
+      owner: "local supervisor",
+      storage_class: "repairable-cache",
+      roots: (context) => [subtree(context, ".harnery/supervisor")],
+      format: "json",
+      durability: "reconstructable",
+      writer_model: "object-owned",
+      policy: cachePolicy(
+        "supervisor-cache-v1",
+        "local resources, service health, log cursors, and finding projections",
+      ),
+      consumers: ["local supervisor", "dashboard", "storage inventory", "storage health"],
+      provider: filesystemProvider(
+        "supervisor-cache-provider",
+        "bounded local diagnostic observations",
+      ),
+    }),
     family({
       id: "structured-log-metrics",
       owner: "structured log provider",
@@ -770,6 +813,19 @@ function ownerHistoryPolicy(version: string): HarneryStoragePolicy {
   });
 }
 
+function closedEpochPolicy(version: string): HarneryStoragePolicy {
+  return policy(version, "active", "private", "owner-content", {
+    status: "active",
+    mode: "delegated",
+    maxAge: bounded(7 * DAY_MS, "milliseconds"),
+    maxBytes: bounded(1024 * MIB, "bytes"),
+    maxFiles: unbounded("files", "complete epoch directories are the deletion unit"),
+    maxRecords: unbounded("records", "complete epoch directories are the deletion unit"),
+    reason:
+      "The Event Ledger owner keeps the newest two complete epochs and revalidates each closed epoch before removal.",
+  });
+}
+
 function delegatedHistoryPolicy(version: string): HarneryStoragePolicy {
   const result = ownerHistoryPolicy(version);
   result.retention.mode = "delegated";

package/src/core/storage/logger.ts

@@ -517,6 +517,8 @@ const PROCESS_LOGGER_BINDINGS = {
   "agent-hook": "agent-hook-debug-log",
   "agent-coord": "agent-coord-debug-log",
   "semantic-service": "semantic-service-log",
+  "resource-observer": "resource-observer-log",
+  supervisor: "supervisor-log",
   "governor-service": "governor-service-log",
   "presence-relay": "presence-relay-log",
 } as const;

package/src/core/storage/query.ts

@@ -42,6 +42,12 @@ export interface HarneryLogFollowCursor {
   active_offset: number;
 }
 
+export interface HarneryRecentActiveLogResult {
+  records: readonly HarneryLogRecordV1[];
+  bytes_examined: number;
+  truncated: boolean;
+}
+
 const LEVELS: readonly HarneryLogLevel[] = ["trace", "debug", "info", "warn", "error", "fatal"];
 const MAX_UNBOUNDED_RECORD_BYTES = 1024 * 1024;
 const STREAM_CHUNK_BYTES = 64 * 1024;
@@ -109,6 +115,58 @@ export function rotationFollowCursor(
   };
 }
 
+/**
+ * Read the newest complete records from one family's active segment without
+ * walking sealed history. Dashboard-style live views need a cheap bounded
+ * snapshot; the full `queryLogs` surface remains the historical query path.
+ */
+export function readRecentActiveLogs(
+  family: HarneryRegisteredStorageFamily,
+  options: { max_records: number; max_bytes: number },
+): HarneryRecentActiveLogResult {
+  if (
+    !Number.isSafeInteger(options.max_records) ||
+    options.max_records <= 0 ||
+    !Number.isSafeInteger(options.max_bytes) ||
+    options.max_bytes <= 0
+  ) {
+    throw new Error("recent log read requires positive budgets");
+  }
+  const active = join(familyLogDirectory(family), "active.jsonl");
+  if (!existsSync(active)) return { records: [], bytes_examined: 0, truncated: false };
+
+  const opened = openRegularNoFollow(active);
+  try {
+    const size = fstatSync(opened.fd).size;
+    const start = Math.max(0, size - options.max_bytes);
+    const length = size - start;
+    if (length === 0) return { records: [], bytes_examined: 0, truncated: false };
+    const buffer = Buffer.allocUnsafe(length);
+    let consumed = 0;
+    while (consumed < length) {
+      const count = readSync(opened.fd, buffer, consumed, length - consumed, start + consumed);
+      if (count === 0) break;
+      consumed += count;
+    }
+    assertOpenFileIdentity(active, opened.fd, opened.parent);
+    const text = buffer.subarray(0, consumed).toString("utf8");
+    let lines = text.split("\n");
+    // A bounded tail can begin in the middle of a record. The active writer
+    // can also leave a not-yet-complete final record while this snapshot reads.
+    if (start > 0) lines = lines.slice(1);
+    if (!text.endsWith("\n")) lines = lines.slice(0, -1);
+    const records = lines.filter((line) => line.trim()).map(parseLogRecord);
+    const truncated = start > 0 || records.length > options.max_records;
+    return {
+      records: records.slice(-options.max_records),
+      bytes_examined: consumed,
+      truncated,
+    };
+  } finally {
+    closeSync(opened.fd);
+  }
+}
+
 export async function readLogFollow(
   family: HarneryRegisteredStorageFamily,
   cursor: HarneryLogFollowCursor,