harnery 0.32.0 → 0.34.0

package/src/core/guard/config.ts

@@ -0,0 +1,206 @@
+import { createHash } from "node:crypto";
+import { coordRunQualityConfigSource } from "../config.ts";
+import type { RunQualityConfig, RunQualityConfigResult, RunQualityMode } from "./types.ts";
+
+export const DEFAULT_RUN_QUALITY_CONFIG: RunQualityConfig = {
+  mode: "off",
+  evaluation_interval_seconds: 30,
+  snapshot_ttl_seconds: 120,
+  max_tail_bytes: 2 * 1024 * 1024,
+  evaluation_timeout_seconds: 30,
+  lock_stale_seconds: 60,
+  supervised_roots_per_sweep: 8,
+  thresholds: {
+    repeated_tool_calls: 8,
+    consecutive_failures: 5,
+    context_growth_per_minute: 60_000,
+    compaction_grace_seconds: 300,
+    no_progress_evaluations: 2,
+  },
+};
+
+const RANGES = {
+  evaluation_interval_seconds: [5, 3600],
+  snapshot_ttl_seconds: [10, 86_400],
+  max_tail_bytes: [64 * 1024, 64 * 1024 * 1024],
+  evaluation_timeout_seconds: [1, 30],
+  lock_stale_seconds: [5, 3600],
+  supervised_roots_per_sweep: [1, 100],
+  repeated_tool_calls: [2, 10_000],
+  consecutive_failures: [1, 10_000],
+  context_growth_per_minute: [1, 10_000_000],
+  compaction_grace_seconds: [0, 86_400],
+  no_progress_evaluations: [1, 10_000],
+} as const;
+
+/** Read and validate the project-owned coord.run_quality object as one unit. */
+export function readRunQualityConfig(coordRoot: string): RunQualityConfigResult {
+  const source = coordRunQualityConfigSource(coordRoot);
+  if (source.invalid) return invalid(source.digest_seed, "config_json_invalid", "off");
+  const value = source.value;
+  if (value === undefined) return result(DEFAULT_RUN_QUALITY_CONFIG, true, []);
+  const rq = record(value);
+  const mode = modeOf(rq?.mode);
+  if (!rq || !mode) return invalid(value, "run_quality_object_invalid", mode ?? "off");
+
+  const config: RunQualityConfig = {
+    mode,
+    evaluation_interval_seconds: numberOrDefault(
+      rq.evaluation_interval_seconds,
+      DEFAULT_RUN_QUALITY_CONFIG.evaluation_interval_seconds,
+    ),
+    snapshot_ttl_seconds: numberOrDefault(
+      rq.snapshot_ttl_seconds,
+      DEFAULT_RUN_QUALITY_CONFIG.snapshot_ttl_seconds,
+    ),
+    max_tail_bytes: numberOrDefault(rq.max_tail_bytes, DEFAULT_RUN_QUALITY_CONFIG.max_tail_bytes),
+    evaluation_timeout_seconds: numberOrDefault(
+      rq.evaluation_timeout_seconds,
+      DEFAULT_RUN_QUALITY_CONFIG.evaluation_timeout_seconds,
+    ),
+    lock_stale_seconds: numberOrDefault(
+      rq.lock_stale_seconds,
+      DEFAULT_RUN_QUALITY_CONFIG.lock_stale_seconds,
+    ),
+    supervised_roots_per_sweep: numberOrDefault(
+      rq.supervised_roots_per_sweep,
+      DEFAULT_RUN_QUALITY_CONFIG.supervised_roots_per_sweep,
+    ),
+    thresholds: {
+      repeated_tool_calls: numberOrDefault(
+        record(rq.thresholds)?.repeated_tool_calls,
+        DEFAULT_RUN_QUALITY_CONFIG.thresholds.repeated_tool_calls,
+      ),
+      consecutive_failures: numberOrDefault(
+        record(rq.thresholds)?.consecutive_failures,
+        DEFAULT_RUN_QUALITY_CONFIG.thresholds.consecutive_failures,
+      ),
+      context_growth_per_minute: numberOrDefault(
+        record(rq.thresholds)?.context_growth_per_minute,
+        DEFAULT_RUN_QUALITY_CONFIG.thresholds.context_growth_per_minute,
+      ),
+      compaction_grace_seconds: numberOrDefault(
+        record(rq.thresholds)?.compaction_grace_seconds,
+        DEFAULT_RUN_QUALITY_CONFIG.thresholds.compaction_grace_seconds,
+      ),
+      no_progress_evaluations: numberOrDefault(
+        record(rq.thresholds)?.no_progress_evaluations,
+        DEFAULT_RUN_QUALITY_CONFIG.thresholds.no_progress_evaluations,
+      ),
+    },
+  };
+  const reasons = validate(config, rq);
+  return reasons.length > 0 ? invalid(value, reasons, mode) : result(config, true, []);
+}
+
+function validate(config: RunQualityConfig, raw: Record<string, unknown>): string[] {
+  const reasons: string[] = [];
+  const allowedTop = new Set([
+    "mode",
+    "evaluation_interval_seconds",
+    "snapshot_ttl_seconds",
+    "max_tail_bytes",
+    "evaluation_timeout_seconds",
+    "lock_stale_seconds",
+    "supervised_roots_per_sweep",
+    "thresholds",
+  ]);
+  if (Object.keys(raw).some((key) => !allowedTop.has(key))) reasons.push("unknown_field");
+  const top = [
+    "evaluation_interval_seconds",
+    "snapshot_ttl_seconds",
+    "max_tail_bytes",
+    "evaluation_timeout_seconds",
+    "lock_stale_seconds",
+    "supervised_roots_per_sweep",
+  ] as const;
+  for (const key of top) validateInteger(key, config[key], raw[key], reasons);
+  if (config.lock_stale_seconds <= config.evaluation_timeout_seconds) {
+    reasons.push("lock_stale_not_above_timeout");
+  }
+  if (config.snapshot_ttl_seconds < config.evaluation_interval_seconds) {
+    reasons.push("snapshot_ttl_below_interval");
+  }
+  const thresholds = record(raw.thresholds);
+  if (raw.thresholds !== undefined && !thresholds) reasons.push("thresholds_invalid");
+  const allowedThresholds = new Set(Object.keys(config.thresholds));
+  if (thresholds && Object.keys(thresholds).some((key) => !allowedThresholds.has(key))) {
+    reasons.push("thresholds_unknown_field");
+  }
+  for (const key of Object.keys(config.thresholds) as Array<keyof RunQualityConfig["thresholds"]>) {
+    validateInteger(key, config.thresholds[key], thresholds?.[key], reasons);
+  }
+  return reasons;
+}
+
+function validateInteger(
+  key: keyof typeof RANGES,
+  value: number,
+  supplied: unknown,
+  reasons: string[],
+): void {
+  if (supplied !== undefined && (!Number.isSafeInteger(supplied) || supplied !== value)) {
+    reasons.push(`${key}_not_integer`);
+    return;
+  }
+  const [min, max] = RANGES[key];
+  if (value < min || value > max) reasons.push(`${key}_out_of_range`);
+}
+
+function numberOrDefault(value: unknown, fallback: number): number {
+  return typeof value === "number" ? value : fallback;
+}
+
+function modeOf(value: unknown): RunQualityMode | null {
+  return value === "off" || value === "shadow" || value === "report" ? value : null;
+}
+
+function result(
+  config: RunQualityConfig,
+  valid: boolean,
+  reasonCodes: string[],
+): RunQualityConfigResult {
+  return {
+    config,
+    digest: digest(config),
+    valid,
+    requested_mode: config.mode,
+    reason_codes: reasonCodes,
+  };
+}
+
+function invalid(
+  value: unknown,
+  reasons: string | string[],
+  mode: RunQualityMode,
+): RunQualityConfigResult {
+  return {
+    config: null,
+    digest: digest(value),
+    valid: false,
+    requested_mode: mode,
+    reason_codes: Array.isArray(reasons) ? reasons : [reasons],
+  };
+}
+
+function digest(value: unknown): string {
+  const body = typeof value === "string" ? value : stableJson(value);
+  return createHash("sha256").update(body).digest("hex");
+}
+
+function stableJson(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
+  if (value && typeof value === "object") {
+    return `{${Object.entries(value as Record<string, unknown>)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([key, child]) => `${JSON.stringify(key)}:${stableJson(child)}`)
+      .join(",")}}`;
+  }
+  return JSON.stringify(value) ?? "null";
+}
+
+function record(value: unknown): Record<string, unknown> | null {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}

package/src/core/guard/coordinator.ts

@@ -0,0 +1,227 @@
+import { mkdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { readEventV3ControlState } from "../events/v3/control.ts";
+import { liveInstanceIdV3 } from "../events/v3/live-routing.ts";
+import { readRunQualityConfig } from "./config.ts";
+import { evaluateRunQuality } from "./evaluator.ts";
+import { readRunQualityLiveSourceV3 } from "./live-source-v3.ts";
+import {
+  acquireEvaluationLock,
+  cleanupOrphanSnapshots,
+  evaluationLockOwned,
+  guardDir,
+  readGuardCursor,
+  readRunQualitySnapshot,
+  releaseEvaluationLock,
+  writeAtomicJson,
+  writeGuardCursor,
+  writeRunQualitySnapshot,
+} from "./storage.ts";
+import type { RunQualityConfigResult, RunQualitySnapshot } from "./types.ts";
+
+export interface RunQualityEvaluationResult {
+  config: RunQualityConfigResult;
+  evaluated: boolean;
+  busy: boolean;
+  timed_out: boolean;
+  snapshot: RunQualitySnapshot | null;
+}
+
+/**
+ * Lazy, root-scoped evaluator. Its due check is config/cursor-only; ledger I/O
+ * begins only after the non-blocking writer lock is acquired.
+ */
+export function evaluateRunQualityIfDue(
+  coordRoot: string,
+  now: Date = new Date(),
+  instanceId?: string,
+): RunQualityEvaluationResult {
+  const configResult = readRunQualityConfig(coordRoot);
+  const canonicalInstanceId = instanceId ? liveInstanceIdV3(instanceId) : undefined;
+  const previousForCaller = canonicalInstanceId
+    ? readRunQualitySnapshot(coordRoot, canonicalInstanceId)
+    : null;
+  if (!configResult.valid || !configResult.config) {
+    maybeEmitInvalidConfig(coordRoot, configResult, now);
+    return {
+      config: configResult,
+      evaluated: false,
+      busy: false,
+      timed_out: false,
+      snapshot: null,
+    };
+  }
+  const config = configResult.config;
+  if (config.mode === "off") {
+    return {
+      config: configResult,
+      evaluated: false,
+      busy: false,
+      timed_out: false,
+      snapshot: previousForCaller,
+    };
+  }
+
+  const cursor = readGuardCursor(coordRoot);
+  if (
+    cursor?.config_digest === configResult.digest &&
+    Date.parse(cursor.next_eligible_at) > now.getTime()
+  ) {
+    return {
+      config: configResult,
+      evaluated: false,
+      busy: false,
+      timed_out: false,
+      snapshot: previousForCaller,
+    };
+  }
+
+  const lock = acquireEvaluationLock(coordRoot, now, config.lock_stale_seconds);
+  if (!lock) {
+    return {
+      config: configResult,
+      evaluated: false,
+      busy: true,
+      timed_out: false,
+      snapshot: previousForCaller,
+    };
+  }
+
+  const deadline = Date.now() + config.evaluation_timeout_seconds * 1000;
+  let timedOut = false;
+  try {
+    const control = readEventV3ControlState(coordRoot);
+    if (control.state !== "candidate" && control.state !== "active") {
+      throw new Error(`run_quality_v3_control_${control.state}`);
+    }
+    return evaluateRunQualityV3({
+      coordRoot,
+      now,
+      instanceId: canonicalInstanceId,
+      configResult,
+      deadline,
+      lockNonce: lock.nonce,
+    });
+  } catch (error) {
+    timedOut = (error as Error).message === "run_quality_evaluation_timeout";
+    return {
+      config: configResult,
+      evaluated: false,
+      busy: false,
+      timed_out: timedOut,
+      snapshot: previousForCaller,
+    };
+  } finally {
+    releaseEvaluationLock(coordRoot, lock.nonce);
+  }
+}
+
+function evaluateRunQualityV3(input: {
+  coordRoot: string;
+  now: Date;
+  instanceId?: string;
+  configResult: Extract<RunQualityConfigResult, { valid: true }> | RunQualityConfigResult;
+  deadline: number;
+  lockNonce: string;
+}): RunQualityEvaluationResult {
+  const config = input.configResult.config;
+  if (!config) throw new Error("run_quality_v3_config_unavailable");
+  const source = readRunQualityLiveSourceV3(input.coordRoot, input.now);
+  const snapshots: RunQualitySnapshot[] = [];
+  for (const generation of source.generations) {
+    const previous = readRunQualitySnapshot(input.coordRoot, generation.instance_id) ?? undefined;
+    const previousIndex = previous?.evidence.last_event_id
+      ? generation.events.findIndex((event) => event.event_id === previous.evidence.last_event_id)
+      : -1;
+    const events =
+      previousIndex >= 0 ? generation.events.slice(previousIndex + 1) : generation.events;
+    const snapshot = evaluateRunQuality({
+      instance_id: generation.instance_id,
+      session_id: generation.session_id,
+      session_generation: generation.generation_id,
+      adapter: generation.adapter,
+      now: input.now.toISOString(),
+      config,
+      config_digest: input.configResult.digest,
+      events,
+      previous,
+      role_wait: generation.role_wait,
+      evidence: generation.evidence,
+      sufficient_history:
+        generation.sufficient_history &&
+        (!previous ||
+          previous.session_generation !== generation.generation_id ||
+          previousIndex >= 0),
+      live: true,
+    });
+    snapshots.push(snapshot);
+  }
+
+  assertCanPublish(input.coordRoot, input.lockNonce, input.deadline);
+  const liveIds = new Set(snapshots.map((snapshot) => snapshot.instance_id));
+  for (const snapshot of snapshots) {
+    assertCanPublish(input.coordRoot, input.lockNonce, input.deadline);
+    writeRunQualitySnapshot(input.coordRoot, snapshot, input.lockNonce);
+  }
+  cleanupOrphanSnapshots(input.coordRoot, liveIds);
+
+  const last = source.generations
+    .flatMap((generation) => generation.events)
+    .sort(
+      (left, right) =>
+        left.ts.localeCompare(right.ts) || left.event_id.localeCompare(right.event_id),
+    )
+    .at(-1);
+  writeGuardCursor(
+    input.coordRoot,
+    {
+      schema_version: 1,
+      segment: last ? `v3:${source.genesis_id}` : `v3:${source.genesis_id}:empty`,
+      last_event_id: last?.event_id,
+      next_eligible_at: new Date(
+        input.now.getTime() + config.evaluation_interval_seconds * 1000,
+      ).toISOString(),
+      config_digest: input.configResult.digest,
+      updated_at: input.now.toISOString(),
+    },
+    input.lockNonce,
+  );
+
+  return {
+    config: input.configResult,
+    evaluated: true,
+    busy: false,
+    timed_out: false,
+    snapshot: input.instanceId
+      ? (snapshots.find((snapshot) => snapshot.instance_id === input.instanceId) ?? null)
+      : null,
+  };
+}
+
+function maybeEmitInvalidConfig(
+  coordRoot: string,
+  config: RunQualityConfigResult,
+  now: Date,
+): void {
+  if (config.valid) return;
+  const dir = guardDir(coordRoot);
+  const markerPath = join(dir, "config-invalid.json");
+  let previousDigest: string | undefined;
+  try {
+    previousDigest = (JSON.parse(readFileSync(markerPath, "utf8")) as { digest?: string }).digest;
+  } catch {
+    // First invalid observation.
+  }
+  if (previousDigest === config.digest) return;
+  mkdirSync(dir, { recursive: true });
+  writeAtomicJson(
+    markerPath,
+    { digest: config.digest, emitted_at: now.toISOString() },
+    config.digest,
+  );
+}
+
+function assertCanPublish(coordRoot: string, nonce: string, deadline: number): void {
+  if (Date.now() > deadline) throw new Error("run_quality_evaluation_timeout");
+  if (!evaluationLockOwned(coordRoot, nonce)) throw new Error("run_quality_lock_stolen");
+}