harnery 0.33.0 → 0.34.0

package/src/core/guard/evaluator.ts

@@ -0,0 +1,343 @@
+import type {
+  EvaluateRunQualityInput,
+  RunQualityEvaluatorState,
+  RunQualityEvidenceEvent,
+  RunQualitySignal,
+  RunQualitySnapshot,
+  RunQualityStatus,
+} from "./types.ts";
+import { RUN_QUALITY_SCHEMA_VERSION } from "./types.ts";
+
+const LEVELS: RunQualityStatus[] = ["unknown", "healthy", "attention", "critical"];
+
+/** Pure report-only transition. It has no filesystem, process, or verdict dependencies. */
+export function evaluateRunQuality(input: EvaluateRunQualityInput): RunQualitySnapshot {
+  const configChanged = !!input.previous && input.previous.config_digest !== input.config_digest;
+  const sameGeneration =
+    !input.previous || input.previous.session_generation === input.session_generation;
+  const priorState = !configChanged && sameGeneration ? input.previous?.state : undefined;
+  const state = cloneState(priorState);
+  const events = [...input.events].sort(compareEvidence);
+  let progressSeen = false;
+  let newEvidence = false;
+
+  for (const event of events) {
+    if (input.previous?.evidence.last_event_id === event.event_id) continue;
+    newEvidence = true;
+    progressSeen = applyEvidence(state, event, input) || progressSeen;
+  }
+
+  const legitimateWait = input.role_wait.fresh && input.role_wait.wait_kind !== "none";
+  const deadlineEpoch = !newEvidence && input.live && !legitimateWait;
+  const epoch = newEvidence ? "evidence" : deadlineEpoch ? "deadline" : "read";
+
+  const inGrace = graceActive(state, input.now);
+  const signals = buildSignals(state, input, inGrace);
+  const precursorFamilies = new Set(
+    signals
+      .filter(
+        (signal) =>
+          signal.state === "active" &&
+          signal.id !== "no_progress" &&
+          signal.id !== "compaction_grace",
+      )
+      .map((signal) =>
+        signal.id === "repeated_tool_calls" || signal.id === "target_stagnation"
+          ? "tool_behavior"
+          : signal.id,
+      ),
+  ).size;
+
+  if (progressSeen) {
+    state.no_progress_epochs = 0;
+    state.no_progress_deadline_epochs = 0;
+  } else if (precursorFamilies >= 2 && (newEvidence || deadlineEpoch)) {
+    state.no_progress_epochs++;
+    if (deadlineEpoch) state.no_progress_deadline_epochs++;
+  }
+  replaceSignal(signals, noProgressSignal(state, input, inGrace, legitimateWait));
+
+  let desired = desiredStatus(signals, input.sufficient_history, input.evidence.truncated);
+  if (progressSeen && desired === "critical") desired = "attention";
+  const previousStatus = input.previous?.status ?? "unknown";
+  const transitionBase = configChanged || !sameGeneration ? "unknown" : previousStatus;
+  const status = stepOneLevel(transitionBase, desired);
+  const evaluatedAt = input.now;
+  const nextEligibleAt = addSeconds(evaluatedAt, input.config.evaluation_interval_seconds);
+  const expiresAt = addSeconds(evaluatedAt, input.config.snapshot_ttl_seconds);
+
+  return {
+    schema_version: RUN_QUALITY_SCHEMA_VERSION,
+    instance_id: input.instance_id,
+    session_id: input.session_id,
+    session_generation: input.session_generation,
+    adapter: input.adapter,
+    config_digest: input.config_digest,
+    mode: input.config.mode === "report" ? "report" : "shadow",
+    status,
+    previous_status: previousStatus,
+    evaluated_at: evaluatedAt,
+    next_eligible_at: nextEligibleAt,
+    expires_at: expiresAt,
+    evidence: input.evidence,
+    signals,
+    role_wait: input.role_wait,
+    state,
+    epoch,
+    reason: configChanged
+      ? "config_changed"
+      : !input.sufficient_history || input.evidence.truncated
+        ? "insufficient_evidence"
+        : deadlineEpoch
+          ? "deadline"
+          : "evidence",
+  };
+}
+
+function applyEvidence(
+  state: RunQualityEvaluatorState,
+  event: RunQualityEvidenceEvent,
+  input: EvaluateRunQualityInput,
+): boolean {
+  switch (event.kind) {
+    case "tool_call":
+      state.work_since_progress++;
+      if (event.target_hash) {
+        state.target_hash_seen = true;
+        if (state.target_hash === event.target_hash) state.target_streak++;
+        else {
+          state.target_hash = event.target_hash;
+          state.target_streak = 1;
+        }
+      }
+      if (!event.input_hash) {
+        state.missing_hash_seen = true;
+        return false;
+      }
+      state.exact_hash_seen = true;
+      if (state.repeated_hash === event.input_hash) state.repeated_count++;
+      else {
+        state.repeated_hash = event.input_hash;
+        state.repeated_count = 1;
+      }
+      return false;
+    case "tool_success":
+      state.failure_streak = 0;
+      return false;
+    case "tool_failure":
+      state.failure_streak++;
+      return false;
+    case "progress":
+      state.work_since_progress = 0;
+      state.target_streak = 0;
+      return true;
+    case "context_sample":
+      applyContextSample(state, event, input.adapter);
+      return false;
+    case "compaction_started":
+      state.compaction_grace_until = addSeconds(
+        event.ts,
+        input.config.thresholds.compaction_grace_seconds,
+      );
+      return false;
+    case "compaction_completed":
+      state.compaction_grace_until = addSeconds(
+        event.ts,
+        input.config.thresholds.compaction_grace_seconds,
+      );
+      return true;
+    case "tool_pairing_incomplete":
+    case "command_pairing_incomplete":
+    case "recovered_terminal":
+      return false;
+  }
+}
+
+function applyContextSample(
+  state: RunQualityEvaluatorState,
+  event: RunQualityEvidenceEvent,
+  adapter: string,
+): void {
+  if (
+    event.used_tokens === undefined ||
+    !Number.isFinite(event.used_tokens) ||
+    !attestedContextSource(adapter, event.telemetry_source, event.confidence)
+  ) {
+    return;
+  }
+  const previous = state.last_context;
+  if (previous) {
+    const minutes = (Date.parse(event.ts) - Date.parse(previous.ts)) / 60_000;
+    if (Number.isFinite(minutes) && minutes > 0) {
+      state.context_growth_per_minute = (event.used_tokens - previous.used_tokens) / minutes;
+    }
+  }
+  state.last_context = { used_tokens: event.used_tokens, ts: event.ts };
+}
+
+function attestedContextSource(
+  adapter: string,
+  source: string | undefined,
+  confidence: RunQualityEvidenceEvent["confidence"],
+): boolean {
+  if (confidence !== "exact" && confidence !== "reported") return false;
+  const allowed: Record<string, readonly string[]> = {
+    "claude-code": ["hook", "native_event", "result"],
+    codex: ["native_event", "result", "transcript"],
+    cursor: ["native_event"],
+  };
+  return !!source && (allowed[adapter] ?? []).includes(source);
+}
+
+function buildSignals(
+  state: RunQualityEvaluatorState,
+  input: EvaluateRunQualityInput,
+  inGrace: boolean,
+): RunQualitySignal[] {
+  const t = input.config.thresholds;
+  return [
+    thresholdSignal(
+      "repeated_tool_calls",
+      state.repeated_count,
+      t.repeated_tool_calls,
+      state.exact_hash_seen ? "exact_input_hash" : "exact_input_hash_unavailable",
+      !state.exact_hash_seen && state.missing_hash_seen,
+    ),
+    thresholdSignal(
+      "consecutive_failures",
+      state.failure_streak,
+      t.consecutive_failures,
+      "consecutive_outcomes",
+      false,
+    ),
+    inGrace
+      ? signal("context_growth", "suppressed", "none", 0, "compaction_grace")
+      : state.context_growth_per_minute === undefined
+        ? signal("context_growth", "unknown", "none", 0, "attested_samples_unavailable")
+        : thresholdSignal(
+            "context_growth",
+            Math.max(0, Math.round(state.context_growth_per_minute)),
+            t.context_growth_per_minute,
+            "attested_context_velocity",
+            false,
+          ),
+    thresholdSignal(
+      "target_stagnation",
+      state.target_streak,
+      t.repeated_tool_calls,
+      "same_target_without_progress_event",
+      !state.target_hash_seen,
+    ),
+    signal("no_progress", "inactive", "none", state.no_progress_epochs, "not_corroborated"),
+    inGrace
+      ? signal("compaction_grace", "active", "none", 1, "within_grace_deadline")
+      : signal("compaction_grace", "inactive", "none", 0, "outside_grace_deadline"),
+  ];
+}
+
+function noProgressSignal(
+  state: RunQualityEvaluatorState,
+  input: EvaluateRunQualityInput,
+  inGrace: boolean,
+  legitimateWait: boolean,
+): RunQualitySignal {
+  if (inGrace) return signal("no_progress", "suppressed", "none", 0, "compaction_grace");
+  if (legitimateWait) return signal("no_progress", "suppressed", "none", 0, "legitimate_wait");
+  if (state.no_progress_deadline_epochs < 1) {
+    return signal(
+      "no_progress",
+      "inactive",
+      "none",
+      state.no_progress_epochs,
+      "deadline_epoch_required",
+    );
+  }
+  return thresholdSignal(
+    "no_progress",
+    state.no_progress_epochs,
+    input.config.thresholds.no_progress_evaluations,
+    "corroborated_epochs",
+    false,
+  );
+}
+
+function thresholdSignal(
+  id: RunQualitySignal["id"],
+  count: number,
+  threshold: number,
+  reason: string,
+  unknown: boolean,
+): RunQualitySignal {
+  if (unknown) return signal(id, "unknown", "none", count, reason);
+  if (count >= threshold * 2) return signal(id, "active", "critical", count, reason);
+  if (count >= threshold) return signal(id, "active", "attention", count, reason);
+  return signal(id, "inactive", "none", count, reason);
+}
+
+function signal(
+  id: RunQualitySignal["id"],
+  state: RunQualitySignal["state"],
+  severity: RunQualitySignal["severity"],
+  count: number,
+  reason_code: string,
+): RunQualitySignal {
+  return { id, state, severity, count, reason_code };
+}
+
+function replaceSignal(signals: RunQualitySignal[], next: RunQualitySignal): void {
+  const index = signals.findIndex((signal) => signal.id === next.id);
+  if (index >= 0) signals[index] = next;
+}
+
+function desiredStatus(
+  signals: RunQualitySignal[],
+  sufficientHistory: boolean,
+  truncated: boolean,
+): RunQualityStatus {
+  if (!sufficientHistory || truncated) return "unknown";
+  if (signals.some((signal) => signal.state === "active" && signal.severity === "critical")) {
+    return "critical";
+  }
+  if (signals.some((signal) => signal.state === "active" && signal.severity === "attention")) {
+    return "attention";
+  }
+  return "healthy";
+}
+
+function stepOneLevel(previous: RunQualityStatus, desired: RunQualityStatus): RunQualityStatus {
+  if (previous === desired) return previous;
+  if (previous === "unknown") return desired === "critical" ? "attention" : desired;
+  if (desired === "unknown") return "unknown";
+  const from = LEVELS.indexOf(previous);
+  const to = LEVELS.indexOf(desired);
+  return LEVELS[from + Math.sign(to - from)] ?? desired;
+}
+
+function graceActive(state: RunQualityEvaluatorState, now: string): boolean {
+  if (!state.compaction_grace_until) return false;
+  return Date.parse(now) < Date.parse(state.compaction_grace_until);
+}
+
+function cloneState(previous?: RunQualityEvaluatorState): RunQualityEvaluatorState {
+  return previous
+    ? structuredClone(previous)
+    : {
+        repeated_count: 0,
+        exact_hash_seen: false,
+        missing_hash_seen: false,
+        target_streak: 0,
+        target_hash_seen: false,
+        failure_streak: 0,
+        work_since_progress: 0,
+        no_progress_epochs: 0,
+        no_progress_deadline_epochs: 0,
+      };
+}
+
+function compareEvidence(a: RunQualityEvidenceEvent, b: RunQualityEvidenceEvent): number {
+  return a.event_id.localeCompare(b.event_id) || a.ts.localeCompare(b.ts);
+}
+
+function addSeconds(iso: string, seconds: number): string {
+  return new Date(Date.parse(iso) + seconds * 1000).toISOString();
+}

package/src/core/guard/evidence-v3.ts

@@ -0,0 +1,227 @@
+import { createHash } from "node:crypto";
+import { canonicalJsonV3, type EventV3 } from "../events/v3/index.ts";
+import type { RunQualityCorpusCategoryV3, RunQualityEvidenceEvent } from "./types.ts";
+
+interface GenerationScope {
+  root_id: string;
+  instance_id: string;
+  session_id: string;
+  generation_id: string;
+  workflow_id?: string;
+}
+
+interface GenerationLinks {
+  caused_by: string[];
+  parent_generation_id?: string;
+}
+
+/**
+ * Convert one validated V3 row into the evaluator's privacy-safe evidence.
+ * Both the live guard and offline corpus use this exact adapter.
+ */
+export function normalizeRunQualityEventV3(
+  event: EventV3,
+  priorEvents: ReadonlyMap<string, EventV3> = new Map(),
+): RunQualityEvidenceEvent[] {
+  const base = { event_id: event.event_id, ts: event.time.observed_at };
+  const recovery = recoveryCategory(event, base);
+  switch (event.event_type) {
+    case "tool.requested":
+      return [
+        {
+          ...base,
+          kind: "tool_call",
+          input_hash: bareDigest(event.payload.exact_input.digest),
+          target_hash: combinedTargetHash(
+            event.payload.targets.map(({ fingerprint }) => fingerprint.digest),
+          ),
+        },
+        ...recovery,
+      ];
+    case "tool.completed":
+      return [
+        {
+          ...base,
+          kind: event.payload.outcome === "succeeded" ? "tool_success" : "tool_failure",
+        },
+        ...recovery,
+      ];
+    case "command.completed":
+      return recovery;
+    case "turn.started":
+      return [{ ...base, kind: "progress" }];
+    case "progress.observed":
+      return trustedProgress(event, priorEvents) ? [{ ...base, kind: "progress" }] : [];
+    case "context.observed": {
+      const measurement = event.payload.measurement;
+      if (measurement.state !== "observed") return [];
+      return [
+        {
+          ...base,
+          kind: "context_sample",
+          used_tokens: measurement.value.used_tokens,
+          confidence: evidenceConfidence(measurement.attestation, measurement.confidence),
+          telemetry_source: telemetrySource(measurement.value.method),
+        },
+      ];
+    }
+    case "context.compaction_started":
+      return [{ ...base, kind: "compaction_started" }];
+    case "context.compaction_completed":
+      return [{ ...base, kind: "compaction_completed" }];
+    default:
+      return [];
+  }
+}
+
+/**
+ * Derive generation-level pairing markers from validated events. The markers
+ * are audit metadata, not behavioral guard evidence: callers keep them in a
+ * separate corpus-category dimension.
+ */
+export function normalizeRunQualityPairingV3(
+  events: readonly EventV3[],
+): RunQualityEvidenceEvent[] {
+  const spans = new Map<
+    string,
+    {
+      tool_requested: EventV3[];
+      tool_completed: EventV3[];
+      started_commands: EventV3[];
+      completed_commands: EventV3[];
+    }
+  >();
+  for (const event of events) {
+    if (
+      event.event_type !== "tool.requested" &&
+      event.event_type !== "tool.completed" &&
+      event.event_type !== "command.started" &&
+      event.event_type !== "command.completed"
+    ) {
+      continue;
+    }
+    const spanId = (event.links as { span_id?: string }).span_id;
+    if (!spanId) continue;
+    const span = spans.get(spanId) ?? {
+      tool_requested: [],
+      tool_completed: [],
+      started_commands: [],
+      completed_commands: [],
+    };
+    if (event.event_type === "tool.requested") span.tool_requested.push(event);
+    else if (event.event_type === "tool.completed") span.tool_completed.push(event);
+    else if (event.event_type === "command.started") span.started_commands.push(event);
+    else span.completed_commands.push(event);
+    spans.set(spanId, span);
+  }
+
+  const markers: RunQualityEvidenceEvent[] = [];
+  for (const span of spans.values()) {
+    if (
+      (span.tool_requested.length > 0 || span.tool_completed.length > 0) &&
+      (span.tool_requested.length !== 1 || span.tool_completed.length !== 1)
+    ) {
+      markers.push(
+        pairingMarker("tool_pairing_incomplete", span.tool_requested, span.tool_completed),
+      );
+    }
+    if (
+      (span.started_commands.length > 0 || span.completed_commands.length > 0) &&
+      (span.started_commands.length !== 1 || span.completed_commands.length !== 1)
+    ) {
+      markers.push(
+        pairingMarker("command_pairing_incomplete", span.started_commands, span.completed_commands),
+      );
+    }
+  }
+  return markers;
+}
+
+export function isRunQualityCorpusCategoryV3(
+  kind: RunQualityEvidenceEvent["kind"],
+): kind is RunQualityCorpusCategoryV3 {
+  return (
+    kind === "tool_pairing_incomplete" ||
+    kind === "command_pairing_incomplete" ||
+    kind === "recovered_terminal"
+  );
+}
+
+function recoveryCategory(
+  event: EventV3,
+  base: Pick<RunQualityEvidenceEvent, "event_id" | "ts">,
+): RunQualityEvidenceEvent[] {
+  if (!("recovery" in event.payload) || event.payload.recovery === undefined) return [];
+  return [{ ...base, kind: "recovered_terminal" }];
+}
+
+function pairingMarker(
+  kind: Extract<
+    RunQualityCorpusCategoryV3,
+    "tool_pairing_incomplete" | "command_pairing_incomplete"
+  >,
+  left: EventV3[],
+  right: EventV3[],
+): RunQualityEvidenceEvent {
+  const witness = [...left, ...right].sort(
+    (a, b) =>
+      a.time.observed_at.localeCompare(b.time.observed_at) || a.event_id.localeCompare(b.event_id),
+  )[0]!;
+  return { event_id: witness.event_id, ts: witness.time.observed_at, kind };
+}
+
+function trustedProgress(
+  event: Extract<EventV3, { event_type: "progress.observed" }>,
+  priorEvents: ReadonlyMap<string, EventV3>,
+): boolean {
+  const links = event.links as GenerationLinks;
+  const scope = event.scope as GenerationScope;
+  if (!event.payload.evidence_event_ids.every((eventId) => links.caused_by.includes(eventId))) {
+    return false;
+  }
+  return event.payload.evidence_event_ids.every((eventId) => {
+    const evidence = priorEvents.get(eventId);
+    if (!evidence || !("generation_id" in evidence.scope)) return false;
+    if (evidence.scope.generation_id !== scope.generation_id) return false;
+    switch (evidence.event_type) {
+      case "tool.completed":
+      case "turn.completed":
+      case "run.completed":
+      case "command.completed":
+      case "agent.completed":
+        return evidence.payload.outcome === "succeeded";
+      case "artifact.observed":
+        return evidence.payload.operation !== "viewed";
+      default:
+        return false;
+    }
+  });
+}
+
+function combinedTargetHash(digests: string[]): string | undefined {
+  const unique = [...new Set(digests.map(bareDigest))].sort();
+  if (unique.length === 0) return undefined;
+  if (unique.length === 1) return unique[0];
+  return createHash("sha256").update(canonicalJsonV3(unique)).digest("hex");
+}
+
+function bareDigest(digest: string): string {
+  return digest.slice("sha256:".length);
+}
+
+function evidenceConfidence(
+  attestation: "native" | "derived" | "inferred",
+  confidence: "exact" | "high" | "medium" | "low",
+): NonNullable<RunQualityEvidenceEvent["confidence"]> {
+  if (attestation === "native" && confidence === "exact") return "exact";
+  if (attestation === "inferred") return "estimated";
+  return "reported";
+}
+
+function telemetrySource(method: string): string {
+  if (method.endsWith("_hook")) return "hook";
+  if (method.includes("transcript")) return "transcript";
+  if (method.includes("result")) return "result";
+  if (method.includes("estimate")) return "estimate";
+  return "native_event";
+}

package/src/core/guard/index.ts

@@ -0,0 +1,35 @@
+export { DEFAULT_RUN_QUALITY_CONFIG, readRunQualityConfig } from "./config.ts";
+export type { RunQualityEvaluationResult } from "./coordinator.ts";
+export { evaluateRunQualityIfDue } from "./coordinator.ts";
+export { evaluateRunQuality } from "./evaluator.ts";
+export {
+  isRunQualityCorpusCategoryV3,
+  normalizeRunQualityEventV3,
+  normalizeRunQualityPairingV3,
+} from "./evidence-v3.ts";
+export type {
+  RunQualityLiveGenerationV3,
+  RunQualityLiveSourceV3,
+} from "./live-source-v3.ts";
+export {
+  projectRunQualityLiveSourceV3,
+  RunQualityLiveSourceV3Error,
+  readRunQualityLiveSourceV3,
+} from "./live-source-v3.ts";
+export {
+  readFreshRunQualitySnapshot,
+  readRunQualitySnapshot,
+} from "./storage.ts";
+export type {
+  EvaluateRunQualityInput,
+  RunQualityConfig,
+  RunQualityConfigResult,
+  RunQualityCorpusCategoryV3,
+  RunQualityEvidenceEvent,
+  RunQualityMode,
+  RunQualityRoleWait,
+  RunQualitySignal,
+  RunQualitySnapshot,
+  RunQualityStatus,
+  RunQualityWaitKind,
+} from "./types.ts";