harnery 0.14.0 → 0.16.0

package/src/core/supervisor/runner.ts

@@ -0,0 +1,278 @@
+import { resolve } from "node:path";
+import {
+  acceptWorkItem,
+  type RunWorkItemInput,
+  reconcileWorkItem,
+  runWorkItem,
+  type WorkRecord,
+} from "../work/index.ts";
+import {
+  acquireSupervisorLease,
+  readSupervisorIgnoringLease,
+  type SupervisorProjection,
+  type SupervisorRecord,
+} from "./state.ts";
+
+export type SupervisorStopReason =
+  | "succeeded"
+  | "awaiting_attention"
+  | "blocked"
+  | "budget_exhausted"
+  | "no_progress"
+  | "tick_complete";
+
+export interface SupervisorDispatchOutcome {
+  work_id: string;
+  action: "run" | "resume" | "retry";
+  status: "completed" | "parked" | "failed";
+  run_id?: string;
+  error?: string;
+}
+
+export interface SupervisorRunReport {
+  goal_id: string;
+  mode: "tick" | "run";
+  stop_reason: SupervisorStopReason;
+  reason: string;
+  started_at: string;
+  ended_at: string;
+  duration_ms: number;
+  cycles: number;
+  dispatches: number;
+  acceptances: number;
+  outcomes: SupervisorDispatchOutcome[];
+  projection: SupervisorProjection;
+}
+
+export interface RunSupervisorInput {
+  coordRoot: string;
+  goalId: string;
+  mode?: "tick" | "run";
+  actor?: string;
+  engine: Omit<RunWorkItemInput["engine"], "maxAgents" | "concurrency" | "specialists">;
+  onLog?: (line: string) => void;
+}
+
+interface Candidate {
+  record: WorkRecord;
+  action: "run" | "resume" | "retry";
+  consumesAttempt: boolean;
+}
+
+export async function runSupervisor(input: RunSupervisorInput): Promise<SupervisorRunReport> {
+  const coordRoot = resolve(input.coordRoot);
+  const mode = input.mode ?? "run";
+  const actor = input.actor?.trim() || `supervisor:${input.goalId}`;
+  const log = input.onLog ?? (() => undefined);
+  const startedAt = new Date().toISOString();
+  const startedMs = Date.now();
+  const release = acquireSupervisorLease(coordRoot, input.goalId);
+  let cycles = 0;
+  let dispatches = 0;
+  let acceptances = 0;
+  const outcomes: SupervisorDispatchOutcome[] = [];
+  try {
+    const initial = readSupervisorIgnoringLease(coordRoot, input.goalId);
+    const cycleLimit = mode === "tick" ? 1 : initial.intent.limits.max_cycles;
+    const deadline = startedMs + initial.intent.limits.max_runtime_ms;
+    while (cycles < cycleLimit) {
+      cycles++;
+      let record = refreshGraph(coordRoot, input.goalId, actor);
+      log(
+        `[${record.intent.title}] cycle ${cycles}: ${record.projection.state}; ` +
+          `${record.projection.ready_work.length} ready, ${record.projection.attention_work.length} attention`,
+      );
+      if (record.projection.state === "succeeded") {
+        return report("succeeded", record.projection.reason, record);
+      }
+      if (Date.now() >= deadline) {
+        return report("budget_exhausted", "supervisor wall-time limit reached", record);
+      }
+
+      const before = graphFingerprint(record);
+      let changed = false;
+      if (record.intent.automation.accept_passing_proof) {
+        for (const work of record.work) {
+          if (work.projection.state !== "in_review") continue;
+          acceptWorkItem(coordRoot, work.intent.id, {
+            actor,
+            reason: "passing proof accepted by frozen supervisor policy",
+          });
+          acceptances++;
+          changed = true;
+          log(`[${record.intent.title}] accepted ${work.intent.id} from passing proof`);
+        }
+        if (changed) record = refreshGraph(coordRoot, input.goalId, actor);
+        if (record.projection.state === "succeeded") {
+          return report("succeeded", record.projection.reason, record);
+        }
+      }
+
+      const candidates = selectCandidates(record);
+      const selected = selectWithinAttemptBudget(record, candidates).slice(
+        0,
+        record.intent.limits.max_parallel_work,
+      );
+      if (selected.length > 0 && Date.now() < deadline) {
+        const settled = await Promise.all(
+          selected.map(async (candidate) => {
+            dispatches++;
+            log(`[${record.intent.title}] ${candidate.action} ${candidate.record.intent.id}`);
+            try {
+              const result = await runWorkItem({
+                coordRoot,
+                workId: candidate.record.intent.id,
+                retry: candidate.action === "retry",
+                actor,
+                engine: {
+                  ...input.engine,
+                  specialists: record.intent.specialists,
+                  maxAgents: record.intent.limits.max_agents_per_work,
+                  concurrency: record.intent.limits.agent_concurrency,
+                },
+              });
+              return {
+                work_id: candidate.record.intent.id,
+                action: candidate.action,
+                status: "completed",
+                run_id: result.runId,
+              } satisfies SupervisorDispatchOutcome;
+            } catch (error) {
+              const latest = reconcileWorkItem(coordRoot, candidate.record.intent.id, actor);
+              return {
+                work_id: candidate.record.intent.id,
+                action: candidate.action,
+                status: latest.projection.state === "awaiting_approval" ? "parked" : "failed",
+                run_id: latest.projection.latest_run_id,
+                error: boundedError((error as Error).message),
+              } satisfies SupervisorDispatchOutcome;
+            }
+          }),
+        );
+        outcomes.push(...settled);
+        changed = true;
+      }
+
+      record = refreshGraph(coordRoot, input.goalId, actor);
+      const after = graphFingerprint(record);
+      if (record.projection.state === "succeeded") {
+        return report("succeeded", record.projection.reason, record);
+      }
+      if (mode === "tick") {
+        return report(
+          before === after ? "no_progress" : "tick_complete",
+          before === after
+            ? "the tick found no legal progress action"
+            : "one bounded supervisor cycle completed",
+          record,
+        );
+      }
+      if (Date.now() >= deadline) {
+        return report("budget_exhausted", "supervisor wall-time limit reached", record);
+      }
+      if (record.projection.state === "budget_exhausted") {
+        return report("budget_exhausted", record.projection.reason, record);
+      }
+      if (selected.length === 0 && !changed) {
+        return report(stopForProjection(record.projection), record.projection.reason, record);
+      }
+      if (selected.length === 0 && record.projection.state !== "ready") {
+        return report(stopForProjection(record.projection), record.projection.reason, record);
+      }
+      if (before === after) {
+        return report("no_progress", "a supervisor cycle produced no durable state change", record);
+      }
+    }
+    const final = refreshGraph(coordRoot, input.goalId, actor);
+    return report("budget_exhausted", `supervisor cycle limit reached (${cycleLimit})`, final);
+  } finally {
+    release();
+  }
+
+  function report(
+    stopReason: SupervisorStopReason,
+    reason: string,
+    record: SupervisorRecord,
+  ): SupervisorRunReport {
+    const endedAt = new Date().toISOString();
+    return {
+      goal_id: input.goalId,
+      mode,
+      stop_reason: stopReason,
+      reason,
+      started_at: startedAt,
+      ended_at: endedAt,
+      duration_ms: Date.now() - startedMs,
+      cycles,
+      dispatches,
+      acceptances,
+      outcomes,
+      projection: record.projection,
+    };
+  }
+}
+
+function refreshGraph(coordRoot: string, goalId: string, actor: string): SupervisorRecord {
+  const before = readSupervisorIgnoringLease(coordRoot, goalId);
+  for (const work of before.work) reconcileWorkItem(coordRoot, work.intent.id, actor);
+  return readSupervisorIgnoringLease(coordRoot, goalId);
+}
+
+function selectCandidates(record: SupervisorRecord): Candidate[] {
+  const candidates: Candidate[] = [];
+  for (const work of record.work) {
+    if (work.projection.state === "ready") {
+      candidates.push({ record: work, action: "run", consumesAttempt: true });
+      continue;
+    }
+    if (
+      work.projection.state === "awaiting_approval" &&
+      work.projection.next_action === "resume" &&
+      record.intent.automation.resume_approved
+    ) {
+      candidates.push({ record: work, action: "resume", consumesAttempt: false });
+      continue;
+    }
+    if (
+      work.projection.state === "blocked" &&
+      work.projection.next_action === "retry" &&
+      record.intent.automation.retry_blocked
+    ) {
+      candidates.push({ record: work, action: "retry", consumesAttempt: true });
+    }
+  }
+  return candidates;
+}
+
+function selectWithinAttemptBudget(record: SupervisorRecord, candidates: Candidate[]): Candidate[] {
+  let remaining = record.projection.attempts_remaining;
+  return candidates.filter((candidate) => {
+    if (!candidate.consumesAttempt) return true;
+    if (remaining <= 0) return false;
+    remaining--;
+    return true;
+  });
+}
+
+function graphFingerprint(record: SupervisorRecord): string {
+  return JSON.stringify(
+    record.work.map((work) => [
+      work.intent.id,
+      work.projection.state,
+      work.projection.next_action,
+      work.projection.attempts_used,
+      work.events.length,
+    ]),
+  );
+}
+
+function stopForProjection(projection: SupervisorProjection): SupervisorStopReason {
+  if (projection.state === "budget_exhausted") return "budget_exhausted";
+  if (projection.state === "blocked") return "blocked";
+  return "awaiting_attention";
+}
+
+function boundedError(message: string): string {
+  const normalized = message.replace(/\s+/g, " ").trim();
+  return normalized.length > 2_000 ? `${normalized.slice(0, 1_999)}…` : normalized;
+}

package/src/core/supervisor/service-read.ts

@@ -0,0 +1,144 @@
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { hostname } from "node:os";
+import { join, resolve } from "node:path";
+import type {
+  SupervisorServiceConfig,
+  SupervisorServiceRuntime,
+  SupervisorServiceStatus,
+  SupervisorServiceStatusRecord,
+} from "./service.ts";
+
+const MAX_FILE_BYTES = 512 * 1024;
+const MAX_GOALS = 100;
+const FOREIGN_STATUS_STALE_MS = 30_000;
+
+export function readSupervisorServiceConfig(coordRootRaw: string): SupervisorServiceConfig {
+  const coordRoot = resolve(coordRootRaw);
+  const config = readBoundedJson<SupervisorServiceConfig>(
+    join(serviceDir(coordRoot), "config.json"),
+    "supervisor service configuration",
+  );
+  if (
+    config.schema_version !== 1 ||
+    !validTimestamp(config.created_at) ||
+    !Array.isArray(config.goal_ids) ||
+    config.goal_ids.length < 1 ||
+    config.goal_ids.length > MAX_GOALS ||
+    config.goal_ids.some((goalId) => typeof goalId !== "string" || !goalId.trim()) ||
+    !positive(config.wake_interval_ms) ||
+    !positive(config.heartbeat_interval_ms) ||
+    !positive(config.error_backoff_base_ms) ||
+    !positive(config.error_backoff_max_ms) ||
+    config.error_backoff_max_ms < config.error_backoff_base_ms ||
+    !config.engine ||
+    typeof config.engine !== "object" ||
+    typeof config.engine.subscription_only !== "boolean" ||
+    typeof config.engine.allow_api_billing !== "boolean"
+  ) {
+    throw new Error("supervisor service configuration has an unsupported schema");
+  }
+  return config;
+}
+
+export function readSupervisorServiceRuntime(
+  coordRootRaw: string,
+): SupervisorServiceRuntime | undefined {
+  const coordRoot = resolve(coordRootRaw);
+  const path = join(serviceDir(coordRoot), "runtime.json");
+  if (!existsSync(path)) return undefined;
+  const runtime = readBoundedJson<SupervisorServiceRuntime>(path, "supervisor service runtime");
+  if (
+    runtime.schema_version !== 1 ||
+    !validTimestamp(runtime.config_created_at) ||
+    !validTimestamp(runtime.updated_at) ||
+    !runtime.goals ||
+    typeof runtime.goals !== "object" ||
+    Array.isArray(runtime.goals)
+  ) {
+    throw new Error("supervisor service runtime has an unsupported schema");
+  }
+  return runtime;
+}
+
+export function readSupervisorServiceStatus(coordRootRaw: string): SupervisorServiceStatus {
+  const coordRoot = resolve(coordRootRaw);
+  const record = readStatusRecord(coordRoot);
+  let config: SupervisorServiceConfig | undefined;
+  let runtime: SupervisorServiceRuntime | undefined;
+  try {
+    config = readSupervisorServiceConfig(coordRoot);
+  } catch {
+    // Unconfigured or corrupt service state is represented without throwing.
+  }
+  try {
+    runtime = readSupervisorServiceRuntime(coordRoot);
+  } catch {
+    // Runtime is recoverable and must not make the dashboard unreadable.
+  }
+  if (!record) return { running: false, stale: false, config, runtime };
+  const running = statusOwnerIsLive(record);
+  return { running, stale: !running && record.state !== "stopped", record, config, runtime };
+}
+
+export function supervisorServiceLogPath(coordRoot: string): string {
+  return join(serviceDir(resolve(coordRoot)), "service.log");
+}
+
+function readStatusRecord(coordRoot: string): SupervisorServiceStatusRecord | undefined {
+  const path = join(serviceDir(coordRoot), "status.json");
+  if (!existsSync(path)) return undefined;
+  try {
+    const value = readBoundedJson<SupervisorServiceStatusRecord>(path, "supervisor service status");
+    if (
+      value.schema_version !== 1 ||
+      !Number.isSafeInteger(value.pid) ||
+      value.pid < 1 ||
+      typeof value.host !== "string" ||
+      typeof value.nonce !== "string" ||
+      !validTimestamp(value.started_at) ||
+      !validTimestamp(value.heartbeat_at)
+    ) {
+      return undefined;
+    }
+    return value;
+  } catch {
+    return undefined;
+  }
+}
+
+function statusOwnerIsLive(record: SupervisorServiceStatusRecord): boolean {
+  if (record.state === "stopped") return false;
+  if (record.host === hostname()) {
+    try {
+      process.kill(record.pid, 0);
+      return true;
+    } catch (error) {
+      return (error as NodeJS.ErrnoException).code === "EPERM";
+    }
+  }
+  const age = Date.now() - Date.parse(record.heartbeat_at);
+  return Number.isFinite(age) && age < FOREIGN_STATUS_STALE_MS;
+}
+
+function readBoundedJson<T>(path: string, label: string): T {
+  if (!existsSync(path)) throw new Error(`${label} does not exist`);
+  const size = statSync(path).size;
+  if (size <= 0 || size > MAX_FILE_BYTES) throw new Error(`${label} has invalid size ${size}`);
+  try {
+    return JSON.parse(readFileSync(path, "utf8")) as T;
+  } catch (error) {
+    throw new Error(`cannot parse ${label}: ${(error as Error).message}`);
+  }
+}
+
+function serviceDir(coordRoot: string): string {
+  return join(coordRoot, ".harnery", "supervisor-service");
+}
+
+function positive(value: unknown): value is number {
+  return Number.isSafeInteger(value) && (value as number) > 0;
+}
+
+function validTimestamp(value: unknown): value is string {
+  return typeof value === "string" && value.length <= 40 && Number.isFinite(Date.parse(value));
+}