harnery 0.24.0 → 0.26.0

package/src/core/harnesses/attest.ts

@@ -0,0 +1,181 @@
+/**
+ * The opt-in live probe that produces an attestation (ADR 0038).
+ *
+ * One bounded turn per harness, through the same `spawn` the workflow engine
+ * uses, so what gets attested is the path production takes rather than a
+ * parallel test rig.
+ */
+
+import type { SpawnResult } from "../workflow/types.ts";
+import type { AttestableDimension, HarnessAttestation } from "./attestation.ts";
+import {
+  ATTESTATION_SCHEMA_VERSION,
+  profileDigest,
+  sealAttestation,
+  writeAttestation,
+} from "./attestation.ts";
+import { probeBinaryVersion } from "./bench.ts";
+import type { HarnessRegistry } from "./registry.ts";
+import type { CapabilitySupport, HarnessId } from "./types.ts";
+
+/** Fixed and content-free, so nothing user-supplied can reach the record and
+ * the turn stays as cheap as a turn can be. */
+export const ATTESTATION_PROMPT = "Reply with the single word: ok";
+
+export const DEFAULT_ATTESTATION_TIMEOUT_MS = 120_000;
+
+/** Vendor failures arrive as whole console transcripts, banner and prompt echo
+ * included. Reports are bounded evidence, so the reason is collapsed to one
+ * short line and the echoed prompt is removed. */
+const MAX_NOTE_REASON_CHARS = 200;
+
+/** Keep the TAIL, not the head. A CLI prints its banner, config, and startup
+ * warnings first and the reason it actually failed last, so truncating from the
+ * front reliably preserves the noise and discards the answer. Learned the hard
+ * way: a head-truncated note once surfaced a cosmetic startup warning while
+ * hiding the real "out of credits" failure on the final line. */
+function boundedReason(reason: string | undefined): string {
+  if (!reason) return "no error reported";
+  const collapsed = reason.split(ATTESTATION_PROMPT).join("<prompt>").replace(/\s+/g, " ").trim();
+  if (!collapsed) return "no error reported";
+  return collapsed.length > MAX_NOTE_REASON_CHARS
+    ? `…${collapsed.slice(-MAX_NOTE_REASON_CHARS)}`
+    : collapsed;
+}
+
+export type AttestationOutcome = "recorded" | "skipped" | "unreachable" | "failed";
+
+export interface HarnessAttestationResult {
+  harness: HarnessId;
+  outcome: AttestationOutcome;
+  binaryVersion?: string;
+  observations?: Partial<Record<AttestableDimension, CapabilitySupport>>;
+  durationMs?: number;
+  note: string;
+}
+
+export interface HarnessAttestationReport {
+  generatedAt: string;
+  harnesses: HarnessId[];
+  results: HarnessAttestationResult[];
+  recorded: number;
+  /** True when at least one selected harness could not be attested, so a
+   * caller can tell a partial sweep from a complete one. */
+  incomplete: boolean;
+}
+
+export interface RunHarnessAttestationOptions {
+  harnesses?: readonly string[];
+  timeoutMs?: number;
+  cwd?: string;
+  coordRoot?: string;
+  /** Scrub API-key vars from the child so it can only use its stored login,
+   * matching `workflow run --subscription-only`. The observation is recorded
+   * against this mode, because a child that may fall back to an API key can
+   * behave differently from one that may not. */
+  subscriptionOnly?: boolean;
+  /** Test seam and alternate host probe. A null version means unavailable. */
+  versionProbe?: (binary: string) => string | null;
+  /** Test seam. Defaults to the adapter's production spawner. */
+  spawn?: (harness: HarnessId, prompt: string, timeoutMs: number) => Promise<SpawnResult>;
+  /** Test seam. Defaults to writing under the coord root. */
+  persist?: (record: HarnessAttestation) => void;
+  now?: () => Date;
+}
+
+export async function runHarnessAttestation(
+  registry: HarnessRegistry,
+  opts: RunHarnessAttestationOptions = {},
+): Promise<HarnessAttestationReport> {
+  const ids = opts.harnesses?.length ? [...opts.harnesses] : registry.ids();
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_ATTESTATION_TIMEOUT_MS;
+  const cwd = opts.cwd ?? process.cwd();
+  const subscriptionOnly = opts.subscriptionOnly === true;
+  const versionProbe = opts.versionProbe ?? probeBinaryVersion;
+  const now = opts.now ?? (() => new Date());
+  const results: HarnessAttestationResult[] = [];
+
+  for (const id of ids) {
+    const adapter = registry.require(id);
+    const binaryVersion = versionProbe(adapter.profile.binary);
+    if (!binaryVersion) {
+      results.push({
+        harness: id,
+        outcome: "skipped",
+        note: `${adapter.profile.binary} not found on PATH`,
+      });
+      continue;
+    }
+
+    let result: SpawnResult;
+    try {
+      result = opts.spawn
+        ? await opts.spawn(id, ATTESTATION_PROMPT, timeoutMs)
+        : await adapter.spawn({
+            prompt: ATTESTATION_PROMPT,
+            timeoutMs,
+            maxTurns: 1,
+            cwd,
+            subscriptionOnly,
+          });
+    } catch (error) {
+      results.push({
+        harness: id,
+        outcome: "failed",
+        binaryVersion,
+        note: `probe threw: ${boundedReason((error as Error).message)}`,
+      });
+      continue;
+    }
+
+    // Prerequisite rule: an unreachable subject cannot evidence anything, so a
+    // failed turn records nothing at all rather than a page of `unsupported`.
+    if (!result.ok) {
+      results.push({
+        harness: id,
+        outcome: "unreachable",
+        binaryVersion,
+        durationMs: result.durationMs,
+        note: `the probe turn did not complete (${boundedReason(result.error)}); nothing recorded`,
+      });
+      continue;
+    }
+
+    const observations: Partial<Record<AttestableDimension, CapabilitySupport>> = {
+      invocation: "supported",
+      finalResult: result.text.trim().length > 0 ? "supported" : "unsupported",
+      sessionId: result.sessionId ? "supported" : "unsupported",
+      cost: result.costUsd !== undefined ? "supported" : "unsupported",
+    };
+
+    const record = sealAttestation({
+      schema_version: ATTESTATION_SCHEMA_VERSION,
+      harness: id,
+      binary_version: binaryVersion,
+      profile_digest: profileDigest(adapter.profile),
+      subscription_only: subscriptionOnly,
+      observed_at: now().toISOString(),
+      observations,
+    });
+
+    if (opts.persist) opts.persist(record);
+    else writeAttestation(record, { coordRoot: opts.coordRoot });
+
+    results.push({
+      harness: id,
+      outcome: "recorded",
+      binaryVersion,
+      observations,
+      durationMs: result.durationMs,
+      note: `observed on ${binaryVersion}`,
+    });
+  }
+
+  return {
+    generatedAt: now().toISOString(),
+    harnesses: ids,
+    results,
+    recorded: results.filter((row) => row.outcome === "recorded").length,
+    incomplete: results.some((row) => row.outcome !== "recorded"),
+  };
+}

package/src/core/harnesses/attestation.ts

@@ -0,0 +1,249 @@
+/**
+ * Durable record of what an installed vendor CLI was actually observed doing
+ * (ADR 0038).
+ *
+ * The conformance bench proves Harnery's own planner and normalizer against a
+ * committed fixture. That is an adapter check, not a vendor check
+ * (ADR 0037). An attestation is the other half: one bounded live turn, its
+ * observations, and the vendor version they were observed on.
+ *
+ * A record stores structural facts only. No prompt text, no completion text,
+ * no host paths. It lives under the host's coordination directory and is never
+ * published.
+ */
+
+import {
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { resolve } from "node:path";
+import { monorepoRoot } from "../agents/coord-client.ts";
+import { stableDigest } from "../workflow/durable-record.ts";
+import type { CapabilitySupport, HarnessId, HarnessProfile } from "./types.ts";
+
+export const ATTESTATION_SCHEMA_VERSION = 2;
+
+/** Dimensions one minimal live turn can honestly establish. Everything else
+ * needs a purpose-built scenario and stays outside the record rather than
+ * being guessed at. */
+export const ATTESTABLE_DIMENSIONS = ["invocation", "finalResult", "sessionId", "cost"] as const;
+
+export type AttestableDimension = (typeof ATTESTABLE_DIMENSIONS)[number];
+
+export interface HarnessAttestation {
+  schema_version: number;
+  harness: HarnessId;
+  /** What the vendor binary reported when this was recorded. Staleness is
+   * keyed on this, so a vendor upgrade invalidates the record automatically. */
+  binary_version: string;
+  /** Digest of the capability declaration at record time, so an edited
+   * declaration also invalidates the record. */
+  profile_digest: string;
+  /** The billing policy the probe ran under. A child launched with API keys
+   * scrubbed can behave differently from one that can fall back to them, so an
+   * observation only speaks for the mode it was made in. */
+  subscription_only: boolean;
+  observed_at: string;
+  /** Only what the probe actually saw. A dimension absent from this map was
+   * not observed, which is not the same as unsupported. */
+  observations: Partial<Record<AttestableDimension, CapabilitySupport>>;
+  /** Over every field above. A hand-edited record fails to load. */
+  record_digest: string;
+}
+
+export interface AttestationStoreOptions {
+  /** Test seam and alternate host. Defaults to the resolved coord root. */
+  coordRoot?: string;
+}
+
+export function attestationsDir(opts: AttestationStoreOptions = {}): string {
+  const root = opts.coordRoot ?? monorepoRoot();
+  if (!root) throw new Error("Not in a coord-aware repo (coord root resolved to null).");
+  return resolve(root, ".harnery", "harnesses", "attestations");
+}
+
+/** Stable identity of a declaration, so editing a claim invalidates the
+ * attestation that was recorded against the old one. */
+export function profileDigest(profile: HarnessProfile): string {
+  return stableDigest({
+    id: profile.id,
+    binary: profile.binary,
+    capabilities: profile.capabilities,
+  });
+}
+
+function digestOf(record: Omit<HarnessAttestation, "record_digest">): string {
+  return stableDigest(record);
+}
+
+export function sealAttestation(
+  record: Omit<HarnessAttestation, "record_digest">,
+): HarnessAttestation {
+  return { ...record, record_digest: digestOf(record) };
+}
+
+function attestationPath(harness: HarnessId, opts: AttestationStoreOptions): string {
+  if (!/^[a-z0-9][a-z0-9._-]*$/i.test(harness)) {
+    throw new Error(`unsafe harness id for an attestation path: ${harness}`);
+  }
+  return resolve(attestationsDir(opts), `${harness}.json`);
+}
+
+/** Replace-in-place write. Unlike a workflow record an attestation is meant to
+ * be re-recorded, so this is a mutable atomic swap rather than an immutable
+ * create. */
+export function writeAttestation(
+  record: HarnessAttestation,
+  opts: AttestationStoreOptions = {},
+): string {
+  const path = attestationPath(record.harness, opts);
+  mkdirSync(attestationsDir(opts), { recursive: true, mode: 0o700 });
+  const temporary = `${path}.tmp-${process.pid}`;
+  try {
+    writeFileSync(temporary, `${JSON.stringify(record, null, 2)}\n`, {
+      encoding: "utf8",
+      mode: 0o600,
+    });
+    renameSync(temporary, path);
+  } catch (error) {
+    try {
+      unlinkSync(temporary);
+    } catch {
+      // The temp file may never have been created; the original error wins.
+    }
+    throw error;
+  }
+  return path;
+}
+
+/** Null for absent, unreadable, malformed, wrong-schema, or tampered records.
+ * A record that fails its own digest is discarded rather than trusted, because
+ * the whole point of the file is that it was not hand-written. */
+export function readAttestation(
+  harness: HarnessId,
+  opts: AttestationStoreOptions = {},
+): HarnessAttestation | null {
+  let raw: string;
+  try {
+    raw = readFileSync(attestationPath(harness, opts), "utf8");
+  } catch {
+    return null;
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  return validateAttestation(parsed, harness);
+}
+
+export function validateAttestation(
+  value: unknown,
+  harness?: HarnessId,
+): HarnessAttestation | null {
+  if (!value || typeof value !== "object") return null;
+  const record = value as HarnessAttestation;
+  if (record.schema_version !== ATTESTATION_SCHEMA_VERSION) return null;
+  if (typeof record.harness !== "string" || (harness && record.harness !== harness)) return null;
+  if (typeof record.binary_version !== "string" || typeof record.profile_digest !== "string") {
+    return null;
+  }
+  if (typeof record.observed_at !== "string" || typeof record.record_digest !== "string")
+    return null;
+  if (!record.observations || typeof record.observations !== "object") return null;
+
+  const { record_digest, ...body } = record;
+  if (digestOf(body) !== record_digest) return null;
+  return record;
+}
+
+/** An attestation speaks only for the vendor version and declaration it was
+ * recorded against. */
+export function isAttestationCurrent(
+  record: HarnessAttestation | null,
+  binaryVersion: string | null,
+  profile: HarnessProfile,
+  subscriptionOnly?: boolean,
+): record is HarnessAttestation {
+  if (!record || !binaryVersion) return false;
+  if (record.binary_version !== binaryVersion) return false;
+  if (subscriptionOnly !== undefined && record.subscription_only !== subscriptionOnly) return false;
+  return record.profile_digest === profileDigest(profile);
+}
+
+export function listAttestations(opts: AttestationStoreOptions = {}): HarnessAttestation[] {
+  let names: string[];
+  try {
+    names = readdirSync(attestationsDir(opts));
+  } catch {
+    return [];
+  }
+  const records: HarnessAttestation[] = [];
+  for (const name of names) {
+    if (!name.endsWith(".json")) continue;
+    const record = readAttestation(name.slice(0, -".json".length), opts);
+    if (record) records.push(record);
+  }
+  return records.sort((a, b) => a.harness.localeCompare(b.harness));
+}
+
+/** Both harness-derived proof inputs, read once, for a workflow run
+ * (ADR 0038). Callers inject the result so the engine performs no capability
+ * lookups of its own. A harness with no current attestation simply has no
+ * citation; that absence is not a proof unknown. */
+export function harnessProofInputs(
+  profiles: readonly HarnessProfile[],
+  opts: AttestationStoreOptions & {
+    versionProbe: (binary: string) => string | null;
+    /** Billing policy this run will use, so a record made under the other mode
+     * is not cited as if it applied. */
+    subscriptionOnly?: boolean;
+  },
+): {
+  harnessEvidence: Record<string, { toolEvidence: HarnessProfile["capabilities"]["toolEvidence"] }>;
+  harnessAttestations: Record<
+    string,
+    { binary_version: string; observed_at: string; record_digest: string }
+  >;
+} {
+  const harnessEvidence: Record<
+    string,
+    { toolEvidence: HarnessProfile["capabilities"]["toolEvidence"] }
+  > = {};
+  const harnessAttestations: Record<
+    string,
+    { binary_version: string; observed_at: string; record_digest: string }
+  > = {};
+
+  for (const profile of profiles) {
+    harnessEvidence[profile.id] = { toolEvidence: profile.capabilities.toolEvidence };
+    let record: HarnessAttestation | null = null;
+    try {
+      record = readAttestation(profile.id, opts);
+    } catch {
+      // No coord root or unreadable store: run unattested rather than fail.
+      continue;
+    }
+    if (
+      !isAttestationCurrent(
+        record,
+        opts.versionProbe(profile.binary),
+        profile,
+        opts.subscriptionOnly,
+      )
+    ) {
+      continue;
+    }
+    harnessAttestations[profile.id] = {
+      binary_version: record.binary_version,
+      observed_at: record.observed_at,
+      record_digest: record.record_digest,
+    };
+  }
+  return { harnessEvidence, harnessAttestations };
+}