harnery 0.43.0 → 0.45.0

package/src/core/workflow/proof.ts

@@ -1,7 +1,7 @@
 import { createHash } from "node:crypto";
 import { existsSync, readFileSync, statSync } from "node:fs";
-import { join } from "node:path";
-import type { RepoSnapshot } from "../context/index.ts";
+import { isAbsolute, join } from "node:path";
+import { observationsComparable, type RepoSnapshot } from "../context/observation.ts";
 import { DIAGNOSTIC_ADVICE_SCHEMA_VERSION } from "../diagnostics/contract.ts";
 import type {
   NormalizedPolicy,
@@ -12,6 +12,7 @@ import type {
 import { policyDigest } from "../policy/index.ts";
 import { isCanonicalWorkflowAttemptContext } from "./attempt-context.ts";
 import { stableDigest, writeImmutableJson } from "./durable-record.ts";
+import { failureEvidenceAllowsUncharged, isSpawnFailureEvidence } from "./failure-evidence.ts";
 import { readWorkflowRunManifest } from "./run-state.ts";
 import type {
   AcceptanceCriterion,
@@ -242,17 +243,25 @@ export function digestResult(value: unknown, kind?: "text" | "json"): ResultDige
  *    attempt was informative about the work — charge it. This also keeps a
  *    resumed run whose earlier segment did real work from being written off by
  *    a later environment failure.
- * 3. Otherwise, among the failed agents, environment wins over upstream: a
- *    missing binary means nothing ran at all, and it is the operator-chosen
- *    hard stop.
+ * 3. Every participant must have failed with positive no-effect evidence,
+ *    observed absent output and no refusal. A class alone is insufficient.
+ *    Among those eligible failures, environment is the hard stop.
  * 4. Anything else is undefined ⇒ a charged work failure, exactly as today.
  */
 export function deriveRunFailureClass(
   status: "succeeded" | "failed",
-  agents: readonly Pick<WorkflowAgentProof, "status" | "class">[],
+  agents: readonly Pick<WorkflowAgentProof, "status" | "class" | "failure_evidence">[],
 ): SpawnFailureClass | undefined {
   if (status !== "failed") return undefined;
-  if (agents.some((agent) => agent.status === "succeeded" || agent.status === "cached")) {
+  if (
+    agents.length === 0 ||
+    agents.some(
+      (agent) =>
+        agent.status !== "failed" ||
+        (agent.class !== "environment" && agent.class !== "upstream") ||
+        !failureEvidenceAllowsUncharged(agent.failure_evidence),
+    )
+  ) {
     return undefined;
   }
   const failed = agents.filter((agent) => agent.status === "failed");
@@ -385,6 +394,13 @@ export function readWorkflowProof(coordRoot: string, runId: string): WorkflowPro
       proof.run.class !== "environment" &&
       proof.run.class !== "upstream" &&
       proof.run.class !== "decision") ||
+    !Array.isArray(proof.agents) ||
+    proof.agents.some(
+      (agent) =>
+        !agent ||
+        typeof agent !== "object" ||
+        (agent.failure_evidence !== undefined && !isSpawnFailureEvidence(agent.failure_evidence)),
+    ) ||
     (proof.run.work_context !== undefined &&
       (!proof.run.work_item_id ||
         proof.run.work_context.id !== proof.run.work_item_id ||
@@ -399,6 +415,22 @@ export function readWorkflowProof(coordRoot: string, runId: string): WorkflowPro
   ) {
     throw new Error(`workflow proof at ${path} has an unsupported or mismatched schema`);
   }
+  if (
+    repoComparisonProblem(proof.repository.before, proof.repository.after) !== undefined ||
+    proof.repository.before.dirty_paths_truncated ||
+    proof.repository.after.dirty_paths_truncated ||
+    !proof.repository.before.branch?.trim() ||
+    !proof.repository.after.branch?.trim() ||
+    !proof.repository.before.head?.trim() ||
+    !proof.repository.after.head?.trim()
+  ) {
+    proof.repository = buildRepoEvidence(proof.repository.before, proof.repository.after);
+    proof.unknowns = proof.unknowns.filter((item) => item.code !== "repository_drift_incomplete");
+    proof.unknowns.push({
+      code: "repository_drift_incomplete",
+      message: proof.repository.drift.note!,
+    });
+  }
   const manifestPath = join(coordRoot, ".harnery", "workflows", runId, "run.json");
   if (proof.execution === undefined && !existsSync(manifestPath)) return proof;
   const manifest = readWorkflowRunManifest(coordRoot, runId);
@@ -512,13 +544,31 @@ export function renderWorkflowProof(proof: WorkflowProof): string {
         : "admission: shadow not needed; no real child dispatch",
     );
   }
-  const repo = proof.repository;
+  const comparable =
+    repoComparisonProblem(proof.repository.before, proof.repository.after) === undefined;
+  const repo =
+    comparable &&
+    !proof.repository.before.dirty_paths_truncated &&
+    !proof.repository.after.dirty_paths_truncated &&
+    proof.repository.before.branch?.trim() &&
+    proof.repository.after.branch?.trim() &&
+    proof.repository.before.head?.trim() &&
+    proof.repository.after.head?.trim()
+      ? proof.repository
+      : buildRepoEvidence(proof.repository.before, proof.repository.after);
   const drift = repo.drift;
-  lines.push(
-    `repository: branch ${repo.before.branch ?? "unknown"} -> ${repo.after.branch ?? "unknown"}; ` +
-      `HEAD ${short(repo.before.head)} -> ${short(repo.after.head)}; ` +
-      `${drift.dirty_paths_added.length} dirty added, ${drift.dirty_paths_cleared.length} cleared`,
-  );
+  if (!comparable) {
+    lines.push(`repository: drift unknown. ${drift.note}`);
+  } else {
+    const paths =
+      drift.dirty_paths_added === null || drift.dirty_paths_cleared === null
+        ? "dirty-path drift unknown"
+        : `${drift.dirty_paths_added.length} dirty added, ${drift.dirty_paths_cleared.length} cleared`;
+    lines.push(
+      `repository: branch ${repo.before.branch ?? "unknown"} -> ${repo.after.branch ?? "unknown"}; ` +
+        `HEAD ${short(repo.before.head)} -> ${short(repo.after.head)}; ${paths}`,
+    );
+  }
   if (proof.unknowns.length > 0) {
     lines.push(`unknowns: ${proof.unknowns.length}`);
     for (const unknown of proof.unknowns) lines.push(`  - ${unknown.message}`);
@@ -530,26 +580,56 @@ export function renderWorkflowProof(proof: WorkflowProof): string {
 function buildRepoEvidence(beforeRaw: RepoSnapshot, afterRaw: RepoSnapshot): WorkflowRepoEvidence {
   const before = normalizeRepoSnapshot(beforeRaw);
   const after = normalizeRepoSnapshot(afterRaw);
+  const problem = repoComparisonProblem(before, after);
+  if (problem !== undefined) {
+    return {
+      source: "engine",
+      before,
+      after,
+      drift: {
+        branch_changed: null,
+        head_changed: null,
+        dirty_paths_added: null,
+        dirty_paths_cleared: null,
+        dirty_paths_retained: null,
+        incomplete: true,
+        note: `Repository drift could not be verified: ${problem}. Re-check before editing.`,
+      },
+    };
+  }
+  const pathsComparable = !before.dirty_paths_truncated && !after.dirty_paths_truncated;
+  const branchKnown = Boolean(before.branch && after.branch);
+  const headKnown = Boolean(before.head && after.head);
   const beforeDirty = new Set(before.dirty_paths);
   const afterDirty = new Set(after.dirty_paths);
   const retained = after.dirty_paths.filter((path) => beforeDirty.has(path));
-  const incomplete = Boolean(
-    before.dirty_paths_truncated || after.dirty_paths_truncated || retained.length > 0,
-  );
+  const notes = [
+    ...(!pathsComparable
+      ? ["Dirty-path drift is unknown because a snapshot list was truncated."]
+      : []),
+    ...(!branchKnown ? ["Branch drift is unknown because a snapshot has no branch."] : []),
+    ...(!headKnown ? ["HEAD drift is unknown because a snapshot has no HEAD."] : []),
+    ...(pathsComparable && retained.length > 0
+      ? ["Snapshots cannot prove whether retained dirty paths changed during the run."]
+      : []),
+  ];
+  const incomplete = notes.length > 0;
   return {
     source: "engine",
     before,
     after,
     drift: {
-      branch_changed: before.branch !== after.branch,
-      head_changed: before.head !== after.head,
-      dirty_paths_added: after.dirty_paths.filter((path) => !beforeDirty.has(path)),
-      dirty_paths_cleared: before.dirty_paths.filter((path) => !afterDirty.has(path)),
-      dirty_paths_retained: retained,
+      branch_changed: branchKnown ? before.branch !== after.branch : null,
+      head_changed: headKnown ? before.head !== after.head : null,
+      dirty_paths_added: pathsComparable
+        ? after.dirty_paths.filter((path) => !beforeDirty.has(path))
+        : null,
+      dirty_paths_cleared: pathsComparable
+        ? before.dirty_paths.filter((path) => !afterDirty.has(path))
+        : null,
+      dirty_paths_retained: pathsComparable ? retained : null,
       incomplete,
-      note: incomplete
-        ? "Snapshots cannot prove whether retained dirty paths changed during the run, and truncated lists may omit paths."
-        : undefined,
+      note: incomplete ? notes.join(" ") : undefined,
     },
   };
 }
@@ -557,14 +637,42 @@ function buildRepoEvidence(beforeRaw: RepoSnapshot, afterRaw: RepoSnapshot): Wor
 function normalizeRepoSnapshot(snapshot: RepoSnapshot): WorkflowRepoSnapshot {
   return {
     cwd: snapshot.cwd,
+    observation: snapshot.observation ?? "unavailable",
+    observation_detail: snapshot.observation
+      ? clippedOptional(snapshot.observation_detail, 500)
+      : "No repository observation was recorded.",
     root: snapshot.root,
-    branch: snapshot.branch,
-    head: snapshot.head,
+    branch: snapshot.branch?.trim() || undefined,
+    head: snapshot.head?.trim() || undefined,
     dirty_paths: snapshot.dirty_paths,
     dirty_paths_truncated: snapshot.dirty_paths_truncated,
   };
 }
 
+function snapshotFailure(label: string, snapshot: WorkflowRepoSnapshot): string {
+  return `${label} snapshot is ${snapshot.observation}${snapshot.observation_detail ? ` (${snapshot.observation_detail})` : ""}`;
+}
+
+function repoComparisonProblem(
+  before: WorkflowRepoSnapshot,
+  after: WorkflowRepoSnapshot,
+): string | undefined {
+  if (!observationsComparable(before, after)) {
+    return [
+      ...(before.observation !== "ok" ? [snapshotFailure("before", before)] : []),
+      ...(after.observation !== "ok" ? [snapshotFailure("after", after)] : []),
+    ].join("; ");
+  }
+  const unknownRoots = [
+    ...(!before.root || !isAbsolute(before.root) ? ["before"] : []),
+    ...(!after.root || !isAbsolute(after.root) ? ["after"] : []),
+  ];
+  if (unknownRoots.length > 0)
+    return `${unknownRoots.join(" and ")} snapshot repository root is unknown`;
+  if (before.root !== after.root) return "snapshots name different repository roots";
+  return undefined;
+}
+
 function buildAdapterCoverage(
   agents: WorkflowAgentProof[],
   claims: Readonly<Record<string, AdapterEvidenceCapability | undefined>> | undefined,

package/src/core/workflow/recovery-audit.ts

@@ -0,0 +1,181 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, lstatSync, mkdirSync, readdirSync } from "node:fs";
+import { hostname } from "node:os";
+import { dirname, join } from "node:path";
+import { stateDirMode } from "../storage/modes.ts";
+import { assertWorkflowRunId } from "./approvals.ts";
+import {
+  fsyncParentDirectory,
+  readJsonRecord,
+  stableDigest,
+  writeImmutableJson,
+} from "./durable-record.ts";
+
+export interface WorkflowRecoveryAuthorization {
+  actor: string;
+  reason: string;
+}
+
+export interface WorkflowRecoveryIntent extends WorkflowRecoveryAuthorization {
+  schema_version: 1;
+  operation_id: string;
+  run_id: string;
+  action: "reclaim_unknown_liveness" | "unlock_resume";
+  target_sha256: string;
+  target: Record<string, unknown>;
+  recorded_at: string;
+  pid: number;
+  host: string;
+  outcome: "unknown";
+}
+
+export interface WorkflowRecoveryRecord {
+  intent: WorkflowRecoveryIntent;
+  outcome: "unknown" | "completed";
+}
+
+export function validateWorkflowRecoveryAuthorization(
+  authorization: WorkflowRecoveryAuthorization,
+): void {
+  for (const [field, limit] of [
+    ["actor", 200],
+    ["reason", 2_000],
+  ] as const) {
+    const value = authorization[field];
+    if (
+      typeof value !== "string" ||
+      !value.trim() ||
+      value.length > limit ||
+      Array.from(value).some(
+        (character) => character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127,
+      )
+    ) {
+      throw new Error(
+        `workflow recovery requires a nonempty ${field} of at most ${limit} characters without control characters`,
+      );
+    }
+  }
+}
+
+/** Canonical recovery ledger: immutable intent is durable before effects.
+ * Completion cites that exact intent; absence always means unknown. */
+export function beginWorkflowRecovery(
+  coordRoot: string,
+  input: Omit<
+    WorkflowRecoveryIntent,
+    "schema_version" | "operation_id" | "recorded_at" | "pid" | "host" | "outcome"
+  >,
+): { intent: WorkflowRecoveryIntent; assertCurrent(): void; complete(): void } {
+  assertWorkflowRunId(input.run_id);
+  validateWorkflowRecoveryAuthorization(input);
+  if (!/^[a-f0-9]{64}$/.test(input.target_sha256))
+    throw new Error("workflow recovery requires an exact preview digest");
+  const intent: WorkflowRecoveryIntent = {
+    ...input,
+    schema_version: 1,
+    operation_id: `recovery-${randomUUID()}`,
+    recorded_at: new Date().toISOString(),
+    pid: process.pid,
+    host: hostname(),
+    outcome: "unknown",
+  };
+  const directory = join(
+    coordRoot,
+    ".harnery",
+    "workflows",
+    input.run_id,
+    "recovery",
+    intent.operation_id,
+  );
+  // Sync each newly published directory name too. Syncing only the intent's
+  // immediate parent cannot make a newly created ancestor durable.
+  const runDirectory = join(coordRoot, ".harnery", "workflows", input.run_id);
+  ensureDurableDirectory(runDirectory);
+  const ledgerDirectory = join(runDirectory, "recovery");
+  ensureDurableDirectory(ledgerDirectory);
+  fsyncParentDirectory(ledgerDirectory);
+  mkdirSync(directory, { mode: stateDirMode() });
+  fsyncParentDirectory(directory);
+  const intentPath = join(directory, "intent.json");
+  writeImmutableJson(intentPath, intent);
+  const assertCurrent = () => {
+    if (stableDigest(readJsonRecord(intentPath, "recovery intent")) !== stableDigest(intent))
+      throw new Error("workflow recovery audit authority changed");
+  };
+  return {
+    intent,
+    assertCurrent,
+    complete: () => {
+      assertCurrent();
+      writeImmutableJson(join(directory, "completion.json"), {
+        schema_version: 1,
+        operation_id: intent.operation_id,
+        intent_sha256: stableDigest(intent),
+        outcome: "completed",
+        recorded_at: new Date().toISOString(),
+      });
+    },
+  };
+}
+
+function ensureDurableDirectory(path: string): void {
+  if (existsSync(path)) {
+    if (!lstatSync(path).isDirectory())
+      throw new Error("workflow recovery ledger is not a directory");
+    return;
+  }
+  ensureDurableDirectory(dirname(path));
+  try {
+    mkdirSync(path, { mode: stateDirMode() });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EEXIST" || !lstatSync(path).isDirectory())
+      throw error;
+  }
+  fsyncParentDirectory(path);
+}
+
+export function readWorkflowRecoveryLedger(
+  coordRoot: string,
+  runId: string,
+): WorkflowRecoveryRecord[] {
+  assertWorkflowRunId(runId);
+  const directory = join(coordRoot, ".harnery", "workflows", runId, "recovery");
+  if (!existsSync(directory)) return [];
+  return readdirSync(directory)
+    .sort()
+    .map((entry) => {
+      if (!/^recovery-[a-f0-9-]{36}$/.test(entry))
+        throw new Error("invalid workflow recovery ledger entry");
+      const intent = readJsonRecord<WorkflowRecoveryIntent>(
+        join(directory, entry, "intent.json"),
+        "recovery intent",
+      );
+      if (
+        intent.schema_version !== 1 ||
+        intent.operation_id !== entry ||
+        intent.run_id !== runId ||
+        intent.outcome !== "unknown" ||
+        !["reclaim_unknown_liveness", "unlock_resume"].includes(intent.action) ||
+        !/^[a-f0-9]{64}$/.test(intent.target_sha256)
+      ) {
+        throw new Error("invalid workflow recovery intent authority");
+      }
+      validateWorkflowRecoveryAuthorization(intent);
+      const completionPath = join(directory, entry, "completion.json");
+      if (!existsSync(completionPath)) return { intent, outcome: "unknown" };
+      const completion = readJsonRecord<{
+        schema_version: number;
+        operation_id: string;
+        intent_sha256: string;
+        outcome: string;
+      }>(completionPath, "recovery completion");
+      if (
+        completion.schema_version !== 1 ||
+        completion.operation_id !== entry ||
+        completion.intent_sha256 !== stableDigest(intent) ||
+        completion.outcome !== "completed"
+      )
+        throw new Error("invalid workflow recovery completion authority");
+      return { intent, outcome: "completed" };
+    });
+}