harnery 0.9.0 → 0.10.0

package/src/core/harnesses/bench.ts

@@ -0,0 +1,214 @@
+import { spawnSync } from "node:child_process";
+import { HARNESS_SPECS } from "../hooks/harness/events.ts";
+import type { SpawnRequest, SpawnResult } from "../workflow/types.ts";
+import type { HarnessRegistry } from "./registry.ts";
+import type {
+  CapabilitySupport,
+  HarnessAdapter,
+  HarnessCapabilityDimension,
+  HarnessId,
+} from "./types.ts";
+import { HARNESS_CAPABILITY_DIMENSIONS } from "./types.ts";
+
+export type BenchVerdict =
+  | "supported"
+  | "partial"
+  | "unsupported"
+  | "not_applicable"
+  | "unknown"
+  | "skipped"
+  | "drift";
+
+export type BenchDimension = "registration" | "binary" | HarnessCapabilityDimension;
+
+export interface BenchResult {
+  harness: HarnessId;
+  dimension: BenchDimension;
+  declared: CapabilitySupport | "not_applicable";
+  observed: BenchVerdict;
+  verdict: BenchVerdict;
+  note?: string;
+}
+
+export interface HarnessBenchReport {
+  generatedAt: string;
+  mode: "offline";
+  harnesses: HarnessId[];
+  results: BenchResult[];
+  summary: Record<BenchVerdict, number>;
+  drift: boolean;
+  skipped: boolean;
+}
+
+export interface HarnessBenchOptions {
+  harnesses?: readonly string[];
+  dimensions?: readonly HarnessCapabilityDimension[];
+  /** Test seam and alternate host probe. A null version means unavailable. */
+  versionProbe?: (binary: string) => string | null;
+}
+
+const EMPTY_SUMMARY: Record<BenchVerdict, number> = {
+  supported: 0,
+  partial: 0,
+  unsupported: 0,
+  not_applicable: 0,
+  unknown: 0,
+  skipped: 0,
+  drift: 0,
+};
+
+export function runHarnessBench(
+  registry: HarnessRegistry,
+  opts: HarnessBenchOptions = {},
+): HarnessBenchReport {
+  const ids = opts.harnesses?.length ? [...opts.harnesses] : registry.ids();
+  const dimensions = opts.dimensions?.length
+    ? [...new Set(opts.dimensions)]
+    : [...HARNESS_CAPABILITY_DIMENSIONS];
+  const versionProbe = opts.versionProbe ?? probeVersion;
+  const results: BenchResult[] = [];
+
+  for (const id of ids) {
+    const adapter = registry.require(id);
+    results.push({
+      harness: id,
+      dimension: "registration",
+      declared: "supported",
+      observed: "supported",
+      verdict: "supported",
+      note: `${adapter.profile.integrationMode}; ${adapter.profile.authModel}`,
+    });
+
+    const version = versionProbe(adapter.profile.binary);
+    results.push({
+      harness: id,
+      dimension: "binary",
+      declared: "supported",
+      observed: version ? "supported" : "skipped",
+      verdict: version ? "supported" : "skipped",
+      note: version ?? `${adapter.profile.binary} not found on PATH`,
+    });
+
+    const observations = observeAdapter(adapter);
+    for (const dimension of dimensions) {
+      const claim = adapter.profile.capabilities[dimension];
+      const observed = observations[dimension];
+      results.push({
+        harness: id,
+        dimension,
+        declared: claim.support,
+        observed,
+        verdict: reconcile(claim.support, observed),
+        note: claim.note,
+      });
+    }
+  }
+
+  const summary = { ...EMPTY_SUMMARY };
+  for (const result of results) summary[result.verdict]++;
+  return {
+    generatedAt: new Date().toISOString(),
+    mode: "offline",
+    harnesses: ids,
+    results,
+    summary,
+    drift: summary.drift > 0,
+    skipped: summary.skipped > 0,
+  };
+}
+
+function observeAdapter(adapter: HarnessAdapter): Record<HarnessCapabilityDimension, BenchVerdict> {
+  const profile = adapter.profile;
+  const effort = profile.effortValues[0];
+  const request: SpawnRequest = {
+    prompt: "HARNERY_BENCH_PROMPT",
+    model: "harnery-bench-model",
+    effort,
+    timeoutMs: 1_000,
+    maxTurns: 7,
+    cwd: "/harnery-bench",
+  };
+  let argv: string[] = [];
+  let planningFailed = false;
+  try {
+    argv = adapter.buildInvocation(request, "/harnery-bench/final.txt").argv;
+  } catch {
+    planningFailed = true;
+  }
+
+  let normalized: SpawnResult | null = null;
+  try {
+    normalized = adapter.normalizeResult(adapter.fixture.raw);
+  } catch {
+    normalized = null;
+  }
+
+  const fixture = adapter.fixture.expected;
+  const finalMatches = normalized?.ok === fixture.ok && normalized?.text === fixture.text;
+  const sessionObserved = normalized?.sessionId !== undefined ? "supported" : "unsupported";
+  const costObserved = normalized?.costUsd !== undefined ? "supported" : "unsupported";
+  const hookSpec =
+    adapter.profile.id in HARNESS_SPECS
+      ? HARNESS_SPECS[adapter.profile.id as keyof typeof HARNESS_SPECS]
+      : undefined;
+  const hookSubcommands = new Set(hookSpec?.events.map((event) => event.subcommand) ?? []);
+
+  return {
+    invocation:
+      !planningFailed && argv[0] === profile.binary && argv.includes(request.prompt)
+        ? "supported"
+        : "unsupported",
+    modelSelection:
+      !planningFailed && argv.includes(request.model ?? "") ? "supported" : "unsupported",
+    effortSelection:
+      effort === undefined
+        ? "unsupported"
+        : !planningFailed && argv.some((arg) => arg.includes(effort))
+          ? "supported"
+          : "unsupported",
+    maxTurns:
+      !planningFailed && argv.includes(String(request.maxTurns)) ? "supported" : "unsupported",
+    finalResult: finalMatches ? "supported" : "unsupported",
+    sessionId:
+      sessionObserved === "supported" && normalized?.sessionId !== fixture.sessionId
+        ? "unsupported"
+        : sessionObserved,
+    cost:
+      costObserved === "supported" && normalized?.costUsd !== fixture.costUsd
+        ? "unsupported"
+        : costObserved,
+    toolEvidence: normalized && "toolEvidence" in normalized ? "supported" : "unsupported",
+    policyMapping: "unknown",
+    interruption: "unknown",
+    streaming: "unknown",
+    steering: "unknown",
+    resume: "unknown",
+    images: "unknown",
+    contextTelemetry: "unknown",
+    preCompactionSignal: hookSpec
+      ? hookSubcommands.has("pre-compact")
+        ? "supported"
+        : "unsupported"
+      : "unknown",
+    postCompactionSignal: hookSpec
+      ? hookSubcommands.has("post-compact") ||
+        (adapter.profile.id === "claude-code" && hookSubcommands.has("session-start"))
+        ? "supported"
+        : "unsupported"
+      : "unknown",
+    compaction: "unknown",
+  };
+}
+
+function reconcile(declared: CapabilitySupport, observed: BenchVerdict): BenchVerdict {
+  if (observed === "unknown" || observed === "skipped" || observed === "not_applicable") {
+    return observed;
+  }
+  return declared === observed ? observed : "drift";
+}
+
+function probeVersion(binary: string): string | null {
+  const result = spawnSync(binary, ["--version"], { encoding: "utf8", timeout: 5_000 });
+  if (result.error || result.status !== 0) return null;
+  return (result.stdout || result.stderr).trim().split("\n")[0] || "installed";
+}

package/src/core/harnesses/index.ts

@@ -0,0 +1,31 @@
+export type { Spawner, SpawnRequest, SpawnResult } from "../workflow/types.ts";
+export type {
+  BenchDimension,
+  BenchResult,
+  BenchVerdict,
+  HarnessBenchOptions,
+  HarnessBenchReport,
+} from "./bench.ts";
+export { runHarnessBench } from "./bench.ts";
+export type { BuiltinHarnessId } from "./profiles.ts";
+export {
+  BUILTIN_HARNESS_IDS,
+  BUILTIN_HARNESS_PROFILES,
+  builtinHarnessProfile,
+  isBuiltinHarness,
+  validateHarnessEffort,
+} from "./profiles.ts";
+export { createBuiltinHarnessRegistry, HarnessRegistry } from "./registry.ts";
+export type {
+  CapabilityClaim,
+  CapabilitySupport,
+  HarnessAdapter,
+  HarnessBenchFixture,
+  HarnessCapabilities,
+  HarnessCapabilityDimension,
+  HarnessId,
+  HarnessInvocation,
+  HarnessProfile,
+  HarnessRawResult,
+} from "./types.ts";
+export { HARNESS_CAPABILITY_DIMENSIONS } from "./types.ts";

package/src/core/harnesses/profiles.ts

@@ -0,0 +1,141 @@
+import type { CapabilityClaim, HarnessCapabilities, HarnessProfile } from "./types.ts";
+
+const supported = (note?: string): CapabilityClaim => ({ support: "supported", note });
+const unsupported = (note?: string): CapabilityClaim => ({ support: "unsupported", note });
+const partial = (note: string): CapabilityClaim => ({ support: "partial", note });
+const unknown = (note: string): CapabilityClaim => ({ support: "unknown", note });
+
+function capabilities(overrides: Partial<HarnessCapabilities>): HarnessCapabilities {
+  return {
+    invocation: supported("Headless CLI subprocess."),
+    modelSelection: supported("Explicit model flag is mapped by the adapter."),
+    effortSelection: unsupported(),
+    maxTurns: unsupported(),
+    finalResult: supported("Normalized to SpawnResult.text."),
+    sessionId: unsupported(),
+    cost: unsupported(),
+    toolEvidence: unsupported("The final-result adapter does not retain tool events."),
+    policyMapping: unsupported("No ALLOW/DENY/ASK translation at the workflow boundary."),
+    interruption: partial("Timeout kills the subprocess; no caller-driven interrupt handle."),
+    streaming: unsupported("Workflow children return one normalized final result."),
+    steering: unsupported("One prompt is fixed at subprocess launch."),
+    resume: unsupported("Workflow children always start a new vendor session."),
+    images: unsupported("SpawnRequest currently carries text only."),
+    contextTelemetry: unknown("The workflow adapter does not expose live context usage."),
+    preCompactionSignal: unknown("No compaction lifecycle probe has certified this adapter."),
+    postCompactionSignal: unknown("No compaction lifecycle probe has certified this adapter."),
+    compaction: unsupported("Harnery does not initiate native harness compaction."),
+    ...overrides,
+  };
+}
+
+/** The one built-in profile catalog. Adding a fourth harness starts here; CLI
+ * choices, doctor metadata, workflow dispatch, and the bench derive from it. */
+export const BUILTIN_HARNESS_PROFILES = {
+  "claude-code": {
+    id: "claude-code",
+    displayName: "Claude Code",
+    binary: "claude",
+    installHint: "npm install -g @anthropic-ai/claude-code",
+    loginHint: "run `claude` and use /login",
+    apiKeyEnv: "ANTHROPIC_API_KEY",
+    integrationMode: "cli-subprocess",
+    authModel: "own-auth",
+    modelFamily: "claude",
+    effortValues: ["low", "medium", "high", "xhigh", "max"],
+    verified: { date: "2026-07-20", version: "current CLI contract" },
+    capabilities: capabilities({
+      effortSelection: supported("Mapped to `--effort <level>`."),
+      maxTurns: supported("Mapped to `--max-turns <n>`."),
+      sessionId: supported("Read from the JSON result envelope."),
+      cost: supported("Read from total_cost_usd in the JSON result envelope."),
+      contextTelemetry: partial(
+        "Normalized when a hook payload reports context_window; no dedicated statusline bridge.",
+      ),
+      preCompactionSignal: supported("Claude Code PreCompact is wired to a durable checkpoint."),
+      postCompactionSignal: supported(
+        "SessionStart source=compact completes recovery and injects a verified briefing.",
+      ),
+    }),
+  },
+  codex: {
+    id: "codex",
+    displayName: "Codex",
+    binary: "codex",
+    installHint: "npm install -g @openai/codex",
+    loginHint: "codex login",
+    apiKeyEnv: "OPENAI_API_KEY",
+    integrationMode: "cli-subprocess",
+    authModel: "own-auth",
+    modelFamily: "gpt",
+    effortValues: ["none", "minimal", "low", "medium", "high", "xhigh"],
+    verified: { date: "2026-07-21", version: "codex-cli 0.145.0-alpha.18" },
+    capabilities: capabilities({
+      effortSelection: supported('Mapped to `-c model_reasoning_effort="<level>"`.'),
+      maxTurns: unsupported("codex exec exposes no turn-ceiling flag."),
+      sessionId: unsupported("--output-last-message carries no session id."),
+      cost: unsupported("The final-message path carries no usage or cost."),
+      contextTelemetry: partial(
+        "Normalized when native hook payloads report context_window; unavailable otherwise.",
+      ),
+      preCompactionSignal: supported("Codex PreCompact is wired to a durable checkpoint."),
+      postCompactionSignal: supported(
+        "Codex PostCompact is observed; recovery is injected on the next submitted prompt.",
+      ),
+    }),
+  },
+  cursor: {
+    id: "cursor",
+    displayName: "Cursor Agent",
+    binary: "cursor-agent",
+    installHint: "curl https://cursor.com/install -fsS | bash",
+    loginHint: "cursor-agent login",
+    apiKeyEnv: "CURSOR_API_KEY",
+    integrationMode: "cli-subprocess",
+    authModel: "own-auth",
+    modelFamily: "multi",
+    effortValues: [],
+    verified: { date: "2026-07-21", version: "2026.07.16-899851b" },
+    capabilities: capabilities({
+      effortSelection: unsupported(
+        "Cursor embeds effort in some parameterized model ids; Harnery does not rewrite model ids.",
+      ),
+      maxTurns: unsupported("cursor-agent exposes no turn-ceiling flag."),
+      sessionId: supported("Read from the JSON result envelope."),
+      cost: unsupported("The JSON result envelope carries no cost."),
+      contextTelemetry: unknown("No stable Cursor context-usage payload has been certified."),
+      preCompactionSignal: unsupported(
+        "Harnery does not wire an unverified Cursor preCompact hook.",
+      ),
+      postCompactionSignal: unsupported(
+        "Harnery does not wire an unverified Cursor postCompact hook.",
+      ),
+    }),
+  },
+} as const satisfies Record<string, HarnessProfile>;
+
+export type BuiltinHarnessId = keyof typeof BUILTIN_HARNESS_PROFILES;
+
+export const BUILTIN_HARNESS_IDS = Object.freeze(
+  Object.keys(BUILTIN_HARNESS_PROFILES) as BuiltinHarnessId[],
+);
+
+export function isBuiltinHarness(id: string): id is BuiltinHarnessId {
+  return Object.hasOwn(BUILTIN_HARNESS_PROFILES, id);
+}
+
+export function builtinHarnessProfile(id: string): HarnessProfile | undefined {
+  return isBuiltinHarness(id) ? BUILTIN_HARNESS_PROFILES[id] : undefined;
+}
+
+export function validateHarnessEffort(id: string, effort: string | undefined): void {
+  if (!effort) return;
+  const profile = builtinHarnessProfile(id);
+  if (!profile) throw new Error(`unknown harness ${JSON.stringify(id)}`);
+  if (!profile.effortValues.includes(effort)) {
+    const supportedValues = profile.effortValues.length ? profile.effortValues.join(", ") : "none";
+    throw new Error(
+      `effort ${JSON.stringify(effort)} is not supported by ${id}; supported values: ${supportedValues}`,
+    );
+  }
+}

package/src/core/harnesses/registry.ts

@@ -0,0 +1,140 @@
+import {
+  buildClaudeInvocation,
+  claudeCodeSpawner,
+  normalizeClaudeResult,
+} from "../workflow/spawn-claude.ts";
+import {
+  buildCodexInvocation,
+  codexSpawner,
+  normalizeCodexResult,
+} from "../workflow/spawn-codex.ts";
+import {
+  buildCursorInvocation,
+  cursorSpawner,
+  normalizeCursorResult,
+} from "../workflow/spawn-cursor.ts";
+import {
+  BUILTIN_HARNESS_IDS,
+  BUILTIN_HARNESS_PROFILES,
+  type BuiltinHarnessId,
+} from "./profiles.ts";
+import type { HarnessAdapter, HarnessId } from "./types.ts";
+
+const BUILTIN_ADAPTERS: Record<BuiltinHarnessId, HarnessAdapter> = {
+  "claude-code": {
+    profile: BUILTIN_HARNESS_PROFILES["claude-code"],
+    spawn: claudeCodeSpawner,
+    buildInvocation: buildClaudeInvocation,
+    normalizeResult: normalizeClaudeResult,
+    fixture: {
+      raw: {
+        stdout: JSON.stringify({
+          type: "result",
+          result: "HARNERY_BENCH_OK",
+          session_id: "claude-bench-session",
+          total_cost_usd: 0.0012,
+        }),
+        stderr: "",
+        exitCode: 0,
+        durationMs: 12,
+      },
+      expected: {
+        ok: true,
+        text: "HARNERY_BENCH_OK",
+        sessionId: "claude-bench-session",
+        costUsd: 0.0012,
+      },
+    },
+  },
+  codex: {
+    profile: BUILTIN_HARNESS_PROFILES.codex,
+    spawn: codexSpawner,
+    buildInvocation: buildCodexInvocation,
+    normalizeResult: normalizeCodexResult,
+    fixture: {
+      raw: {
+        stdout: "event stream is not the final answer",
+        stderr: "",
+        exitCode: 0,
+        durationMs: 14,
+        resultFileText: "HARNERY_BENCH_OK\n",
+      },
+      expected: { ok: true, text: "HARNERY_BENCH_OK" },
+    },
+  },
+  cursor: {
+    profile: BUILTIN_HARNESS_PROFILES.cursor,
+    spawn: cursorSpawner,
+    buildInvocation: buildCursorInvocation,
+    normalizeResult: normalizeCursorResult,
+    fixture: {
+      raw: {
+        stdout: JSON.stringify({
+          type: "result",
+          result: "HARNERY_BENCH_OK",
+          session_id: "cursor-bench-session",
+        }),
+        stderr: "",
+        exitCode: 0,
+        durationMs: 9,
+      },
+      expected: {
+        ok: true,
+        text: "HARNERY_BENCH_OK",
+        sessionId: "cursor-bench-session",
+      },
+    },
+  },
+};
+
+/** Mutable registry for embedders, initialized empty unless adapters are
+ * provided. Duplicate ids fail loud instead of silently replacing behavior. */
+export class HarnessRegistry {
+  readonly #adapters = new Map<HarnessId, HarnessAdapter>();
+
+  constructor(adapters: Iterable<HarnessAdapter> = []) {
+    for (const adapter of adapters) this.register(adapter);
+  }
+
+  register(adapter: HarnessAdapter): this {
+    const id = adapter.profile.id.trim();
+    if (!id) throw new Error("harness adapter id cannot be empty");
+    if (this.#adapters.has(id))
+      throw new Error(`harness adapter ${JSON.stringify(id)} is already registered`);
+    if (adapter.profile.binary.trim() === "") {
+      throw new Error(`harness adapter ${JSON.stringify(id)} has no binary`);
+    }
+    this.#adapters.set(id, adapter);
+    return this;
+  }
+
+  get(id: HarnessId): HarnessAdapter | undefined {
+    return this.#adapters.get(id);
+  }
+
+  require(id: HarnessId): HarnessAdapter {
+    const adapter = this.get(id);
+    if (!adapter) {
+      throw new Error(
+        `unknown harness ${JSON.stringify(id)} (registered: ${this.ids().join(", ") || "none"})`,
+      );
+    }
+    return adapter;
+  }
+
+  list(): HarnessAdapter[] {
+    return Array.from(this.#adapters.values());
+  }
+
+  ids(): HarnessId[] {
+    return Array.from(this.#adapters.keys());
+  }
+
+  spawners(): Record<HarnessId, HarnessAdapter["spawn"]> {
+    return Object.fromEntries(this.list().map((adapter) => [adapter.profile.id, adapter.spawn]));
+  }
+}
+
+export function createBuiltinHarnessRegistry(): HarnessRegistry {
+  return new HarnessRegistry(BUILTIN_HARNESS_IDS.map((id) => BUILTIN_ADAPTERS[id]));
+}

package/src/core/harnesses/types.ts

@@ -0,0 +1,90 @@
+import type { Spawner, SpawnRequest, SpawnResult } from "../workflow/types.ts";
+
+/** Open harness identifier. Built-ins are registered at runtime rather than
+ * repeated through closed unions in every consumer. */
+export type HarnessId = string;
+
+export type CapabilitySupport = "supported" | "partial" | "unsupported" | "unknown";
+
+export const HARNESS_CAPABILITY_DIMENSIONS = [
+  "invocation",
+  "modelSelection",
+  "effortSelection",
+  "maxTurns",
+  "finalResult",
+  "sessionId",
+  "cost",
+  "toolEvidence",
+  "policyMapping",
+  "interruption",
+  "streaming",
+  "steering",
+  "resume",
+  "images",
+  "contextTelemetry",
+  "preCompactionSignal",
+  "postCompactionSignal",
+  "compaction",
+] as const;
+
+export type HarnessCapabilityDimension = (typeof HARNESS_CAPABILITY_DIMENSIONS)[number];
+
+export interface CapabilityClaim {
+  support: CapabilitySupport;
+  /** Operational qualification. Required for partial/unknown claims and useful
+   * whenever "supported" has a narrower meaning than the vendor CLI itself. */
+  note?: string;
+}
+
+export type HarnessCapabilities = Record<HarnessCapabilityDimension, CapabilityClaim>;
+
+export interface HarnessProfile {
+  id: HarnessId;
+  displayName: string;
+  binary: string;
+  installHint: string;
+  loginHint: string;
+  apiKeyEnv: string;
+  integrationMode: "cli-subprocess";
+  authModel: "own-auth";
+  modelFamily: "claude" | "gpt" | "multi";
+  effortValues: readonly string[];
+  capabilities: HarnessCapabilities;
+  /** The last real vendor CLI contract used to validate this declaration. */
+  verified?: { date: string; version: string };
+}
+
+/** Fully planned child invocation. `resultFile` is used by adapters such as
+ * Codex that deliver the authoritative final answer out-of-band. */
+export interface HarnessInvocation {
+  argv: string[];
+  resultFile?: string;
+}
+
+/** Vendor subprocess material passed through an adapter's normalizer. */
+export interface HarnessRawResult {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  durationMs: number;
+  resultFileText?: string;
+}
+
+export interface HarnessBenchFixture {
+  raw: HarnessRawResult;
+  expected: Pick<SpawnResult, "ok" | "text"> & {
+    sessionId?: string;
+    costUsd?: number;
+  };
+}
+
+/** One registered executable adapter. The same planner and normalizer power
+ * production workflow runs and the offline conformance bench, preventing the
+ * bench from testing a parallel mock implementation. */
+export interface HarnessAdapter {
+  profile: HarnessProfile;
+  spawn: Spawner;
+  buildInvocation: (request: SpawnRequest, resultFile?: string) => HarnessInvocation;
+  normalizeResult: (raw: HarnessRawResult) => SpawnResult;
+  fixture: HarnessBenchFixture;
+}