harnery 0.35.0 → 0.37.0

package/src/core/supervisor/services.ts

@@ -0,0 +1,203 @@
+import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
+import { hostname } from "node:os";
+import { join } from "node:path";
+import { checkPidToken, processStartToken } from "../agents/state/proc-start.ts";
+import { writePrivateJsonAtomic } from "../storage/atomic-json.ts";
+import {
+  type ObservedServiceHealth,
+  SUPERVISOR_CONSUMER_SCHEMA_VERSION,
+  type SupervisorConsumerRecord,
+  type SupervisorServiceStatusRecord,
+} from "./contract.ts";
+import { readSupervisorConsumer, supervisorConsumerPath, supervisorPaths } from "./storage.ts";
+
+export function registerSupervisorConsumer(
+  coordRoot: string,
+  input: { id: string; pid?: number },
+): SupervisorConsumerRecord {
+  const pid = input.pid ?? process.pid;
+  const startToken = processStartToken(pid);
+  const record: SupervisorConsumerRecord = {
+    schema_version: SUPERVISOR_CONSUMER_SCHEMA_VERSION,
+    id: input.id,
+    pid,
+    ...(startToken ? { start_token: startToken } : {}),
+    registered_at: new Date().toISOString(),
+  };
+  writePrivateJsonAtomic(supervisorConsumerPath(coordRoot, input.id), record);
+  return record;
+}
+
+export function unregisterSupervisorConsumer(coordRoot: string, id: string): void {
+  rmSync(supervisorConsumerPath(coordRoot, id), { force: true });
+}
+
+export function collectServiceHealth(
+  coordRoot: string,
+  supervisor: SupervisorServiceStatusRecord,
+): { services: ObservedServiceHealth[]; consumers: SupervisorConsumerRecord[] } {
+  const consumers = readLiveConsumers(coordRoot);
+  const semantic = readDetachedService(
+    join(coordRoot, ".harnery", "semantic", "v2", "service.json"),
+    "semantic-reader",
+  );
+  const governor = readDetachedService(
+    join(coordRoot, ".harnery", "governor-service", "service.json"),
+    "governor",
+  );
+  const relay = readPresenceRelay(join(coordRoot, ".harnery", "presence", "relay-daemon.json"));
+  const dashboard = consumers.find((consumer) => consumer.id === "dashboard");
+  const services: ObservedServiceHealth[] = [
+    {
+      id: "supervisor",
+      state:
+        supervisor.state === "running"
+          ? "running"
+          : supervisor.state === "error"
+            ? "stale"
+            : "stopped",
+      pid: supervisor.pid,
+      started_at: supervisor.started_at,
+      heartbeat_at: supervisor.heartbeat_at,
+      ...(supervisor.state === "error" ? { reason: "supervisor-reported-error" } : {}),
+    },
+    semantic,
+    governor,
+    relay,
+    dashboard
+      ? {
+          id: "dashboard",
+          state: "running",
+          pid: dashboard.pid,
+          started_at: dashboard.registered_at,
+        }
+      : { id: "dashboard", state: "stopped" },
+  ];
+  return { services, consumers };
+}
+
+function readLiveConsumers(coordRoot: string): SupervisorConsumerRecord[] {
+  const directory = supervisorPaths(coordRoot).consumers;
+  if (!existsSync(directory)) return [];
+  const live: SupervisorConsumerRecord[] = [];
+  for (const entry of readdirSync(directory)) {
+    if (!entry.endsWith(".json")) continue;
+    const path = `${directory}/${entry}`;
+    const record = readSupervisorConsumer(path);
+    if (!record || !consumerAlive(record)) {
+      rmSync(path, { force: true });
+      continue;
+    }
+    live.push(record);
+  }
+  return live.sort((left, right) => left.id.localeCompare(right.id));
+}
+
+function consumerAlive(record: SupervisorConsumerRecord): boolean {
+  if (
+    record.schema_version !== SUPERVISOR_CONSUMER_SCHEMA_VERSION ||
+    !Number.isSafeInteger(record.pid) ||
+    record.pid < 1 ||
+    typeof record.id !== "string"
+  ) {
+    return false;
+  }
+  try {
+    process.kill(record.pid, 0);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
+  }
+  return checkPidToken(record.pid, record.start_token) !== "mismatch";
+}
+
+interface DetachedServiceRecord {
+  pid: number;
+  host?: string;
+  state?: string;
+  started_at?: string;
+  heartbeat_at?: string;
+}
+
+function readDetachedService(
+  path: string,
+  id: "semantic-reader" | "governor",
+): ObservedServiceHealth {
+  const record = readBoundedJson<DetachedServiceRecord>(path);
+  if (!record || !Number.isSafeInteger(record.pid) || record.pid < 1) {
+    return { id, state: "stopped" };
+  }
+  if (record.state === "stopping" || record.state === "stopped") {
+    return projection(id, "stopped", record);
+  }
+  if (record.host && record.host !== hostname()) {
+    const age = record.heartbeat_at
+      ? Date.now() - Date.parse(record.heartbeat_at)
+      : Number.POSITIVE_INFINITY;
+    const state = Number.isFinite(age) && age < 120_000 ? "running" : "stale";
+    return projection(
+      id,
+      state,
+      record,
+      state === "stale" ? "remote-heartbeat-expired" : undefined,
+    );
+  }
+  const state = pidAlive(record.pid) ? "running" : "stale";
+  return projection(
+    id,
+    state,
+    record,
+    state === "stale" ? "recorded-running-process-missing" : undefined,
+  );
+}
+
+function readPresenceRelay(path: string): ObservedServiceHealth {
+  const record = readBoundedJson<{ pid: number; started_at?: string }>(path);
+  if (!record || !Number.isSafeInteger(record.pid) || record.pid < 1) {
+    return { id: "presence-relay", state: "stopped" };
+  }
+  const running = pidAlive(record.pid);
+  return {
+    id: "presence-relay",
+    state: running ? "running" : "stale",
+    pid: record.pid,
+    ...(record.started_at ? { started_at: record.started_at } : {}),
+    ...(!running ? { reason: "recorded-running-process-missing" } : {}),
+  };
+}
+
+function projection(
+  id: "semantic-reader" | "governor",
+  state: ObservedServiceHealth["state"],
+  record: DetachedServiceRecord,
+  reason?: string,
+): ObservedServiceHealth {
+  return {
+    id,
+    state,
+    pid: record.pid,
+    ...(record.started_at ? { started_at: record.started_at } : {}),
+    ...(record.heartbeat_at ? { heartbeat_at: record.heartbeat_at } : {}),
+    ...(reason ? { reason } : {}),
+  };
+}
+
+function readBoundedJson<T>(path: string): T | undefined {
+  if (!existsSync(path)) return undefined;
+  try {
+    const size = statSync(path).size;
+    if (size < 2 || size > 512 * 1_024) return undefined;
+    const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
+    return value && typeof value === "object" && !Array.isArray(value) ? (value as T) : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function pidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code === "EPERM";
+  }
+}

package/src/core/supervisor/status.ts

@@ -0,0 +1,59 @@
+import { hostname } from "node:os";
+import { checkPidToken } from "../agents/state/proc-start.ts";
+import {
+  SUPERVISOR_STATUS_SCHEMA_VERSION,
+  type SupervisorServiceStatusRecord,
+  type SupervisorStatus,
+} from "./contract.ts";
+import { readSupervisorServiceRecord, supervisorPaths } from "./storage.ts";
+
+const STALE_HEARTBEAT_MS = 15_000;
+
+export function readSupervisorStatus(coordRoot: string, nowMs = Date.now()): SupervisorStatus {
+  const paths = supervisorPaths(coordRoot);
+  const record = readSupervisorServiceRecord(coordRoot);
+  if (!validRecord(record)) {
+    return {
+      running: false,
+      stale: false,
+      status_path: paths.service,
+      snapshot_path: paths.snapshot,
+    };
+  }
+  const heartbeatAt = Date.parse(record.heartbeat_at);
+  const stale = !Number.isFinite(heartbeatAt) || nowMs - heartbeatAt > STALE_HEARTBEAT_MS;
+  const running =
+    record.host === hostname() &&
+    !stale &&
+    (record.state === "starting" || record.state === "running" || record.state === "stopping") &&
+    pidStillMatches(record.pid, record.start_token);
+  return {
+    running,
+    stale,
+    record,
+    status_path: paths.service,
+    snapshot_path: paths.snapshot,
+  };
+}
+
+function validRecord(
+  value: SupervisorServiceStatusRecord | undefined,
+): value is SupervisorServiceStatusRecord {
+  return (
+    value?.schema_version === SUPERVISOR_STATUS_SCHEMA_VERSION &&
+    Number.isSafeInteger(value.pid) &&
+    value.pid > 0 &&
+    typeof value.host === "string" &&
+    typeof value.heartbeat_at === "string" &&
+    typeof value.keep_alive === "boolean"
+  );
+}
+
+function pidStillMatches(pid: number, startToken: string | undefined): boolean {
+  try {
+    process.kill(pid, 0);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
+  }
+  return checkPidToken(pid, startToken) !== "mismatch";
+}

package/src/core/supervisor/storage.ts

@@ -0,0 +1,126 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import type { CoordinationHealthSnapshot } from "../agents/health.ts";
+import type {
+  SupervisorActivitySnapshot,
+  SupervisorConsumerRecord,
+  SupervisorFindingExplanation,
+  SupervisorFindings,
+  SupervisorHistory,
+  SupervisorLogFeed,
+  SupervisorServiceStatusRecord,
+  SupervisorSnapshot,
+  SupervisorTimeline,
+} from "./contract.ts";
+
+export interface SupervisorPaths {
+  root: string;
+  service: string;
+  lease: string;
+  stop: string;
+  snapshot: string;
+  history: string;
+  activity: string;
+  findings: string;
+  timelines: string;
+  explanations: string;
+  coordination_health: string;
+  log_feed: string;
+  consumers: string;
+}
+
+export function supervisorPaths(coordRootRaw: string): SupervisorPaths {
+  const root = join(resolve(coordRootRaw), ".harnery", "supervisor");
+  return {
+    root,
+    service: join(root, "service.json"),
+    lease: join(root, "service.lease.json"),
+    stop: join(root, "stop.json"),
+    snapshot: join(root, "snapshot.json"),
+    history: join(root, "history.json"),
+    activity: join(root, "activity.json"),
+    findings: join(root, "findings.json"),
+    timelines: join(root, "timelines"),
+    explanations: join(root, "explanations"),
+    coordination_health: join(root, "coordination-health.json"),
+    log_feed: join(root, "log-feed.json"),
+    consumers: join(root, "consumers"),
+  };
+}
+
+export function supervisorConsumerPath(coordRoot: string, id: string): string {
+  return join(supervisorPaths(coordRoot).consumers, `${safeId(id)}.json`);
+}
+
+export function readSupervisorServiceRecord(
+  coordRoot: string,
+): SupervisorServiceStatusRecord | undefined {
+  return readJson<SupervisorServiceStatusRecord>(supervisorPaths(coordRoot).service);
+}
+
+export function readSupervisorSnapshot(coordRoot: string): SupervisorSnapshot | undefined {
+  return readJson<SupervisorSnapshot>(supervisorPaths(coordRoot).snapshot);
+}
+
+export function readSupervisorHistory(coordRoot: string): SupervisorHistory | undefined {
+  return readJson<SupervisorHistory>(supervisorPaths(coordRoot).history);
+}
+
+export function readSupervisorActivity(coordRoot: string): SupervisorActivitySnapshot | undefined {
+  return readJson<SupervisorActivitySnapshot>(supervisorPaths(coordRoot).activity);
+}
+
+export function readSupervisorFindings(coordRoot: string): SupervisorFindings | undefined {
+  return readJson<SupervisorFindings>(supervisorPaths(coordRoot).findings);
+}
+
+export function readSupervisorTimeline(
+  coordRoot: string,
+  findingId: string,
+): SupervisorTimeline | undefined {
+  return readJson<SupervisorTimeline>(
+    join(supervisorPaths(coordRoot).timelines, `${safeId(findingId)}.json`),
+  );
+}
+
+export function readSupervisorExplanation(
+  coordRoot: string,
+  findingId: string,
+): SupervisorFindingExplanation | undefined {
+  return readJson<SupervisorFindingExplanation>(
+    join(supervisorPaths(coordRoot).explanations, `${safeId(findingId)}.json`),
+  );
+}
+
+export function readCoordinationHealthSnapshot(
+  coordRoot: string,
+): CoordinationHealthSnapshot | undefined {
+  return readJson<CoordinationHealthSnapshot>(supervisorPaths(coordRoot).coordination_health);
+}
+
+export function readSupervisorLogFeed(coordRoot: string): SupervisorLogFeed | undefined {
+  return readJson<SupervisorLogFeed>(supervisorPaths(coordRoot).log_feed);
+}
+
+export function readSupervisorConsumer(path: string): SupervisorConsumerRecord | undefined {
+  return readJson<SupervisorConsumerRecord>(path);
+}
+
+function readJson<T>(path: string): T | undefined {
+  if (!existsSync(path)) return undefined;
+  try {
+    const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
+    return value && typeof value === "object" && !Array.isArray(value) ? (value as T) : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function safeId(value: string): string {
+  const normalized = value
+    .toLowerCase()
+    .replace(/[^a-z0-9_-]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  if (!normalized) throw new Error("supervisor consumer id must contain a letter or number");
+  return normalized.slice(0, 80);
+}

package/src/core/supervisor/timeline.ts

@@ -0,0 +1,133 @@
+import { createHash } from "node:crypto";
+import {
+  SUPERVISOR_DIAGNOSTIC_LIMITS,
+  SUPERVISOR_TIMELINE_SCHEMA_VERSION,
+  type SupervisorFinding,
+  type SupervisorSourceReference,
+  type SupervisorTimeline,
+  type SupervisorTimelineEntry,
+} from "./contract.ts";
+
+export function buildSupervisorTimeline(
+  finding: SupervisorFinding,
+  relatedSources: readonly SupervisorSourceReference[] = [],
+): SupervisorTimeline {
+  const relatedEntries = clusterRelatedSources(relatedSources);
+  const entries: SupervisorTimelineEntry[] = [
+    entry("opened", finding.opened_at, finding.summary, finding.primary_source),
+    ...finding.evidence.map((evidence) => ({
+      ...entry("observed", evidence.source.observed_at, evidence.summary, evidence.source),
+      evidence_id: evidence.id,
+    })),
+    ...relatedEntries,
+    ...finding.capabilities
+      .filter((capability) => capability.state !== "supported")
+      .map((capability) =>
+        entry(
+          "capability",
+          finding.observed_at,
+          `${capability.source_kind} capability is ${capability.state}${capability.reason_code ? ` (${capability.reason_code})` : ""}.`,
+          {
+            id: stableId("src", `${finding.id}:${capability.source_kind}:${capability.state}`),
+            source_kind: capability.source_kind,
+            source_id: capability.reason_code ?? capability.state,
+            observed_at: finding.observed_at,
+            capability: capability.state,
+          },
+        ),
+      ),
+    ...(finding.state === "resolved" && finding.resolved_at
+      ? [
+          entry(
+            "resolved",
+            finding.resolved_at,
+            "The observed condition returned below its threshold.",
+            finding.primary_source,
+          ),
+        ]
+      : []),
+  ].sort(
+    (left, right) =>
+      left.occurred_at.localeCompare(right.occurred_at) || left.id.localeCompare(right.id),
+  );
+  const maxEntries = SUPERVISOR_DIAGNOSTIC_LIMITS.max_timeline_entries;
+  const bounded = entries.slice(-maxEntries);
+  return {
+    schema_version: SUPERVISOR_TIMELINE_SCHEMA_VERSION,
+    finding_id: finding.id,
+    start_at: bounded[0]?.occurred_at ?? finding.opened_at,
+    end_at: bounded[bounded.length - 1]?.occurred_at ?? finding.observed_at,
+    max_entries: maxEntries,
+    omitted_entries: Math.max(0, entries.length - bounded.length),
+    compacted_entries: Math.max(0, relatedSources.length - relatedEntries.length),
+    entries: bounded,
+    capabilities: finding.capabilities.slice(0, SUPERVISOR_DIAGNOSTIC_LIMITS.max_capabilities),
+  };
+}
+
+function entry(
+  relation: SupervisorTimelineEntry["relation"],
+  occurredAt: string,
+  summary: string,
+  source: SupervisorSourceReference,
+): SupervisorTimelineEntry {
+  return {
+    id: stableId("tl", `${relation}\u0000${occurredAt}\u0000${source.id}\u0000${summary}`),
+    occurred_at: occurredAt,
+    first_occurred_at: occurredAt,
+    last_occurred_at: occurredAt,
+    occurrence_count: 1,
+    relation,
+    summary: summary.slice(0, SUPERVISOR_DIAGNOSTIC_LIMITS.max_summary_chars),
+    source,
+  };
+}
+
+function clusterRelatedSources(
+  sources: readonly SupervisorSourceReference[],
+): SupervisorTimelineEntry[] {
+  const groups = new Map<
+    string,
+    {
+      first: SupervisorSourceReference;
+      last: SupervisorSourceReference;
+      count: number;
+    }
+  >();
+  const ordered = [...sources].sort(
+    (left, right) =>
+      left.observed_at.localeCompare(right.observed_at) || left.id.localeCompare(right.id),
+  );
+  for (const source of ordered) {
+    const key = `${source.source_kind}\u0000${source.source_id}\u0000${source.capability}`;
+    const group = groups.get(key);
+    if (group) {
+      group.last = source;
+      group.count += 1;
+    } else {
+      groups.set(key, { first: source, last: source, count: 1 });
+    }
+  }
+  return [...groups.entries()].map(([key, group]) => {
+    const firstAt = group.first.observed_at;
+    const lastAt = group.last.observed_at;
+    const summary =
+      group.count === 1
+        ? `Related ${group.last.source_kind} activity was observed in the same time range.`
+        : `${group.count} related ${group.last.source_kind} ${group.last.source_id} observations were recorded in the same time range.`;
+    return {
+      id: stableId("tl", `related\u0000${key}\u0000${firstAt}\u0000${lastAt}\u0000${group.count}`),
+      occurred_at: lastAt,
+      first_occurred_at: firstAt,
+      last_occurred_at: lastAt,
+      occurrence_count: group.count,
+      relation: "related",
+      summary: summary.slice(0, SUPERVISOR_DIAGNOSTIC_LIMITS.max_summary_chars),
+      source: group.last,
+    };
+  });
+}
+
+function stableId(prefix: string, value: string): string {
+  return `${prefix}_${createHash("sha256").update(value).digest("hex").slice(0, 24)}`;
+}

package/src/core/work/state.ts

@@ -1,6 +1,5 @@
 import { createHash, randomBytes, randomUUID } from "node:crypto";
 import {
-  appendFileSync,
   chmodSync,
   closeSync,
   existsSync,
@@ -14,6 +13,10 @@ import {
 } from "node:fs";
 import { hostname } from "node:os";
 import { dirname, isAbsolute, join, resolve } from "node:path";
+import {
+  appendSegmentedJsonlFile,
+  readSegmentedJsonlFileSync,
+} from "../storage/durable-history.ts";
 import { readWorkflowApproval } from "../workflow/approvals.ts";
 import { stableDigest } from "../workflow/durable-record.ts";
 import { readWorkflowProof } from "../workflow/proof.ts";
@@ -988,8 +991,6 @@ function appendWorkEventUnderLease(
     });
     return appendWorkEventUnderLease(coordRoot, workId, input);
   }
-  if (events.length >= MAX_EVENTS)
-    throw new Error(`work item ${workId} exceeds ${MAX_EVENTS} events`);
   const event: WorkEvent = {
     schema_version: WORK_EVENT_SCHEMA_VERSION,
     work_id: workId,
@@ -1002,13 +1003,10 @@ function appendWorkEventUnderLease(
   if (Buffer.byteLength(line) > MAX_EVENT_BYTES) {
     throw new Error(`work event exceeds ${MAX_EVENT_BYTES} bytes`);
   }
-  const path = join(workDir(coordRoot, workId), "events.jsonl");
-  const existingBytes = existsSync(path) ? statSync(path).size : 0;
-  if (existingBytes + Buffer.byteLength(line) > MAX_EVENTS_BYTES) {
-    throw new Error("work event log would exceed its limit");
-  }
-  appendFileSync(path, line, { encoding: "utf8", mode: 0o600 });
-  chmodSync(path, 0o600);
+  appendSegmentedJsonlFile(join(workDir(coordRoot, workId), "events.jsonl"), event, {
+    max_record_bytes: MAX_EVENT_BYTES,
+    max_segment_bytes: MAX_EVENTS_BYTES,
+  });
   return event;
 }
 
@@ -1029,17 +1027,10 @@ function readWorkIntent(coordRoot: string, workId: string): WorkIntent {
 function readWorkEvents(coordRoot: string, workId: string): WorkEvent[] {
   assertWorkId(workId);
   const path = join(workDir(coordRoot, workId), "events.jsonl");
-  if (!existsSync(path)) return [];
-  const events = readBoundedLines(
-    path,
-    MAX_EVENTS_BYTES,
-    MAX_EVENT_BYTES,
-    `work item ${workId} events`,
-  ).map((line, index) => {
-    const event = parseObject(
-      line,
-      `work item ${workId} event ${index + 1}`,
-    ) as unknown as WorkEvent;
+  const events = readSegmentedJsonlFileSync<WorkEvent>(path, {
+    max_record_bytes: MAX_EVENT_BYTES,
+    max_records: 1_000_000,
+  }).map((event, index) => {
     validateWorkEvent(event, workId, index + 1);
     return event;
   });