harnery 0.34.1 → 0.35.0

package/src/core/semantic/storage.ts

@@ -0,0 +1,286 @@
+import { randomBytes } from "node:crypto";
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { canonicalJsonV3, sha256V3 } from "../events/v3/canonical.ts";
+import type { LedgerCursorV3 } from "../events/v3/reader.ts";
+import {
+  SEMANTIC_EVIDENCE_CONTRACT_VERSION,
+  SEMANTIC_PROMPT_CONTRACT_VERSION,
+  type SemanticAgentReadModelV2,
+  type SemanticConfiguredModel,
+  type SemanticHarness,
+} from "./contract.ts";
+import { validateSemanticReadModel } from "./validate.ts";
+
+export const SEMANTIC_MANIFEST_SCHEMA_VERSION = 2 as const;
+const MAX_JSON_BYTES = 512 * 1024;
+const MAX_CACHE_FILES = 500;
+const MAX_CACHE_AGE_MS = 30 * 24 * 60 * 60 * 1_000;
+
+export interface SemanticReaderResolution {
+  harness: SemanticHarness;
+  configured_model: SemanticConfiguredModel;
+  resolved_model_id?: string;
+  model_attestation?: "verified" | "requested-only";
+  available: boolean;
+  reason_code?:
+    | "harness_unavailable"
+    | "authentication_unavailable"
+    | "model_unavailable"
+    | "model_mismatch";
+  discovered_at: string;
+}
+
+export interface SemanticPendingItem {
+  generation_id: string;
+  evidence_digest: `sha256:${string}`;
+  band: 1 | 2;
+  pending_since: string;
+}
+
+export interface SemanticCallReceipt {
+  generation_id: string;
+  started_at: string;
+}
+
+export interface SemanticManifestV2 {
+  schema_version: typeof SEMANTIC_MANIFEST_SCHEMA_VERSION;
+  ledger_genesis_id: string;
+  cursor?: LedgerCursorV3;
+  configuration_digest: `sha256:${string}`;
+  evidence_contract_version: typeof SEMANTIC_EVIDENCE_CONTRACT_VERSION;
+  prompt_contract_version: typeof SEMANTIC_PROMPT_CONTRACT_VERSION;
+  adapter_resolutions: Partial<Record<SemanticHarness, SemanticReaderResolution>>;
+  pending: SemanticPendingItem[];
+  call_history: SemanticCallReceipt[];
+  last_first_band_generation_id?: string;
+  newest_successful_pass?: string;
+  updated_at: string;
+}
+
+export interface SemanticCacheIdentity {
+  evidence_digest: `sha256:${string}`;
+  source_harness: SemanticHarness;
+  configured_model: SemanticConfiguredModel;
+  resolved_model_id: string;
+  evidence_contract_version: number;
+  prompt_contract_version: number;
+}
+
+export function semanticPaths(coordRootRaw: string) {
+  const root = join(resolve(coordRootRaw), ".harnery", "semantic", "v2");
+  return {
+    root,
+    manifest: join(root, "manifest.json"),
+    agents: join(root, "agents"),
+    cache: join(root, "cache"),
+    service: join(root, "service.json"),
+    stop: join(root, "stop.json"),
+    lease: join(root, "lease.json"),
+    log: join(root, "service.log"),
+  };
+}
+
+export function semanticConfigurationDigest(
+  resolutions: Partial<Record<SemanticHarness, SemanticReaderResolution>>,
+): `sha256:${string}` {
+  const stable = Object.fromEntries(
+    Object.entries(resolutions)
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([harness, resolution]) => [
+        harness,
+        resolution
+          ? {
+              configured_model: resolution.configured_model,
+              ...(resolution.resolved_model_id
+                ? { resolved_model_id: resolution.resolved_model_id }
+                : {}),
+              ...(resolution.model_attestation
+                ? { model_attestation: resolution.model_attestation }
+                : {}),
+              available: resolution.available,
+              ...(resolution.reason_code ? { reason_code: resolution.reason_code } : {}),
+            }
+          : null,
+      ]),
+  );
+  return sha256V3(canonicalJsonV3(stable));
+}
+
+export function semanticCacheKey(identity: SemanticCacheIdentity): `sha256:${string}` {
+  return sha256V3(canonicalJsonV3(identity));
+}
+
+export function readSemanticManifest(coordRootRaw: string): SemanticManifestV2 | undefined {
+  const path = semanticPaths(coordRootRaw).manifest;
+  if (!existsSync(path)) return undefined;
+  const value = readBoundedJson<SemanticManifestV2>(path, "semantic manifest");
+  if (
+    value.schema_version !== SEMANTIC_MANIFEST_SCHEMA_VERSION ||
+    value.evidence_contract_version !== SEMANTIC_EVIDENCE_CONTRACT_VERSION ||
+    value.prompt_contract_version !== SEMANTIC_PROMPT_CONTRACT_VERSION ||
+    !Array.isArray(value.pending) ||
+    !Array.isArray(value.call_history)
+  ) {
+    throw new Error("semantic manifest has an unsupported schema");
+  }
+  return value;
+}
+
+export function writeSemanticManifest(coordRootRaw: string, manifest: SemanticManifestV2): void {
+  writePrivateJsonAtomic(semanticPaths(coordRootRaw).manifest, manifest);
+}
+
+export function readSemanticAgentDocument(
+  coordRootRaw: string,
+  generationId: string,
+): SemanticAgentReadModelV2 | undefined {
+  requireGenerationId(generationId);
+  const path = join(semanticPaths(coordRootRaw).agents, `${generationId}.json`);
+  if (!existsSync(path)) return undefined;
+  const value = readBoundedJson<unknown>(path, "semantic agent document");
+  const verdict = validateSemanticReadModel(value);
+  if (!verdict.ok)
+    throw new Error(`semantic agent document is invalid: ${verdict.issues.join(",")}`);
+  return verdict.value;
+}
+
+export function listSemanticAgentDocuments(coordRootRaw: string): SemanticAgentReadModelV2[] {
+  const dir = semanticPaths(coordRootRaw).agents;
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((name) => /^gen_[a-zA-Z0-9-]+\.json$/.test(name))
+    .sort()
+    .flatMap((name) => {
+      try {
+        const value = readBoundedJson<unknown>(join(dir, name), "semantic agent document");
+        const verdict = validateSemanticReadModel(value);
+        return verdict.ok ? [verdict.value] : [];
+      } catch {
+        return [];
+      }
+    });
+}
+
+export function inspectSemanticDocument(
+  coordRootRaw: string,
+  instanceOrGeneration: string,
+): SemanticAgentReadModelV2 | undefined {
+  if (instanceOrGeneration.startsWith("gen_")) {
+    return readSemanticAgentDocument(coordRootRaw, instanceOrGeneration);
+  }
+  return listSemanticAgentDocuments(coordRootRaw)
+    .filter((document) => document.instance_id === instanceOrGeneration)
+    .sort((left, right) => right.generated_at.localeCompare(left.generated_at))[0];
+}
+
+export function writeSemanticAgentDocument(
+  coordRootRaw: string,
+  document: SemanticAgentReadModelV2,
+): void {
+  const verdict = validateSemanticReadModel(document);
+  if (!verdict.ok)
+    throw new Error(`refusing invalid semantic document: ${verdict.issues.join(",")}`);
+  writePrivateJsonAtomic(
+    join(semanticPaths(coordRootRaw).agents, `${document.generation_id}.json`),
+    document,
+  );
+}
+
+export function readSemanticCache(
+  coordRootRaw: string,
+  cacheKey: string,
+): SemanticAgentReadModelV2 | undefined {
+  requireDigest(cacheKey);
+  const path = join(semanticPaths(coordRootRaw).cache, `${cacheKey.slice("sha256:".length)}.json`);
+  if (!existsSync(path)) return undefined;
+  const value = readBoundedJson<unknown>(path, "semantic cache entry");
+  const verdict = validateSemanticReadModel(value);
+  return verdict.ok && verdict.value.reader_outcome === "accepted" ? verdict.value : undefined;
+}
+
+export function writeSemanticCache(
+  coordRootRaw: string,
+  cacheKey: string,
+  document: SemanticAgentReadModelV2,
+): void {
+  requireDigest(cacheKey);
+  if (document.reader_outcome !== "accepted") {
+    throw new Error("only accepted semantic documents may enter the cache");
+  }
+  writePrivateJsonAtomic(
+    join(semanticPaths(coordRootRaw).cache, `${cacheKey.slice("sha256:".length)}.json`),
+    document,
+  );
+}
+
+export function invalidateSemanticDerivedState(coordRootRaw: string): void {
+  const paths = semanticPaths(coordRootRaw);
+  rmSync(paths.agents, { recursive: true, force: true });
+  rmSync(paths.cache, { recursive: true, force: true });
+  rmSync(paths.manifest, { force: true });
+}
+
+export function pruneSemanticStorage(
+  coordRootRaw: string,
+  options: { keepGenerations: ReadonlySet<string>; now?: number; maxCacheFiles?: number },
+): void {
+  const paths = semanticPaths(coordRootRaw);
+  if (existsSync(paths.agents)) {
+    for (const name of readdirSync(paths.agents)) {
+      const generationId = name.endsWith(".json") ? name.slice(0, -5) : "";
+      if (generationId && !options.keepGenerations.has(generationId)) {
+        rmSync(join(paths.agents, name), { force: true });
+      }
+    }
+  }
+  if (!existsSync(paths.cache)) return;
+  const now = options.now ?? Date.now();
+  const maxFiles = Math.min(MAX_CACHE_FILES, Math.max(1, options.maxCacheFiles ?? MAX_CACHE_FILES));
+  const entries = readdirSync(paths.cache)
+    .filter((name) => /^[a-f0-9]{64}\.json$/.test(name))
+    .map((name) => ({ name, mtime: statSync(join(paths.cache, name)).mtimeMs }))
+    .sort((left, right) => right.mtime - left.mtime);
+  for (const [index, entry] of entries.entries()) {
+    if (index >= maxFiles || now - entry.mtime > MAX_CACHE_AGE_MS) {
+      rmSync(join(paths.cache, entry.name), { force: true });
+    }
+  }
+}
+
+function requireGenerationId(value: string): void {
+  if (!/^gen_[0-9a-zA-Z-]{1,128}$/.test(value)) throw new Error("invalid semantic generation id");
+}
+
+function requireDigest(value: string): void {
+  if (!/^sha256:[a-f0-9]{64}$/.test(value)) throw new Error("invalid semantic cache key");
+}
+
+function readBoundedJson<T>(path: string, label: string): T {
+  const size = statSync(path).size;
+  if (size <= 0 || size > MAX_JSON_BYTES) throw new Error(`${label} has invalid size ${size}`);
+  return JSON.parse(readFileSync(path, "utf8")) as T;
+}
+
+function writePrivateJsonAtomic(path: string, value: unknown): void {
+  const body = `${JSON.stringify(value, null, 2)}\n`;
+  if (Buffer.byteLength(body) > MAX_JSON_BYTES) {
+    throw new Error(`semantic file exceeds ${MAX_JSON_BYTES} bytes`);
+  }
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  chmodSync(dirname(path), 0o700);
+  const temporary = `${path}.tmp-${process.pid}-${randomBytes(4).toString("hex")}`;
+  writeFileSync(temporary, body, { encoding: "utf8", flag: "wx", mode: 0o600 });
+  renameSync(temporary, path);
+  chmodSync(path, 0o600);
+}

package/src/core/semantic/validate.ts

@@ -0,0 +1,212 @@
+import { Value } from "@sinclair/typebox/value";
+import {
+  SEMANTIC_TAGS,
+  type SemanticAgentReadModelV2,
+  SemanticAgentReadModelV2Schema,
+  type SemanticEvidenceKind,
+  type SemanticEvidenceV1,
+  type SemanticField,
+  type SemanticMeaningV2,
+  type SemanticModelReplyV2,
+  SemanticModelReplyV2Schema,
+} from "./contract.ts";
+
+export interface SemanticValidationSuccess<T> {
+  ok: true;
+  value: T;
+}
+
+export interface SemanticValidationFailure {
+  ok: false;
+  issues: string[];
+}
+
+export type SemanticValidationResult<T> = SemanticValidationSuccess<T> | SemanticValidationFailure;
+
+const PRIVATE_TEXT_PATTERNS: Array<[RegExp, string]> = [
+  [
+    /(?:^|\s)(?:[A-Za-z]:\\|\\\\|\/(?:[A-Za-z0-9._-]+\/)+[A-Za-z0-9._-]+|\.\.\/|\.harnery\/)/i,
+    "path",
+  ],
+  [/https?:\/\//i, "url"],
+  [/(?:^|\s)--[a-z][a-z0-9-]*(?:\s|=|$)/i, "command_flag"],
+  [/(?:^|\s)[A-Z][A-Z0-9_]{2,}=/, "environment_assignment"],
+  [/`[^`]+`/, "command_or_code"],
+  [/\b(?:api[_-]?key|secret|password|token)\s*[:=]\s*\S+/i, "secret_shape"],
+];
+
+const FORBIDDEN_CLAIM_PATTERNS: Array<[RegExp, string]> = [
+  [/\b\d{1,3}\s*%(?=\s|$|[.,])/, "percent_complete"],
+  [/\b(?:minutes?|hours?|days?)\s+(?:left|remaining)\b/i, "time_remaining"],
+  [/\b(?:frustrated|confused|happy|sad|excited|anxious|angry)\b/i, "emotion"],
+  [/\b(?:excellent|great|poor|bad)\s+(?:quality|work)\b/i, "quality"],
+];
+
+const RESULT_KINDS = new Set<SemanticEvidenceKind>(["progress", "artifact", "action", "terminal"]);
+const ATTENTION_KINDS = new Set<SemanticEvidenceKind>([
+  "wait",
+  "lifecycle",
+  "error",
+  "claim-conflict",
+]);
+const COMPLETION_LANGUAGE =
+  /\b(?:complete|completed|done|finished|shipped|passed|succeeded|deployed|published)\b/i;
+
+export function semanticPrivacyIssue(value: string): string | undefined {
+  for (const [pattern, issue] of PRIVATE_TEXT_PATTERNS) {
+    if (pattern.test(value)) return issue;
+  }
+  return undefined;
+}
+
+export function isSemanticPrivacySafe(value: string): boolean {
+  return semanticPrivacyIssue(value) === undefined;
+}
+
+export function validateSemanticEvidencePrivacy(
+  evidence: SemanticEvidenceV1,
+): SemanticValidationResult<SemanticEvidenceV1> {
+  const text: Array<[string, string | undefined]> = [
+    ["task", evidence.task?.value],
+    ["operation", evidence.operation?.label],
+    ["attention", evidence.attention?.label],
+    ...evidence.recent.map(
+      (item, index) => [`recent[${index}]`, item.label] as [string, string | undefined],
+    ),
+  ];
+  const issues = text.flatMap(([field, value]) => {
+    if (!value) return [];
+    const issue = semanticPrivacyIssue(value);
+    return issue ? [`${field}:${issue}`] : [];
+  });
+  return issues.length ? { ok: false, issues } : { ok: true, value: evidence };
+}
+
+export function validateSemanticModelReply(
+  input: unknown,
+  evidence: SemanticEvidenceV1,
+): SemanticValidationResult<SemanticModelReplyV2> {
+  if (!Value.Check(SemanticModelReplyV2Schema, input)) {
+    return {
+      ok: false,
+      issues: [...Value.Errors(SemanticModelReplyV2Schema, input)]
+        .slice(0, 12)
+        .map((error) => `schema:${error.path || "/"}:${error.message}`),
+    };
+  }
+  const reply = input as SemanticModelReplyV2;
+  const issues: string[] = [];
+  if (reply.generation_id !== evidence.generation_id) issues.push("generation_mismatch");
+  if (reply.evidence_digest !== evidence.evidence_digest) issues.push("evidence_digest_mismatch");
+
+  const knownIds = new Set(evidence.evidence_event_ids);
+  const kindById = evidenceKindByEventId(evidence);
+  for (const [name, field] of meaningFields(reply.meaning)) {
+    if (name === "tags" && Array.isArray(field.value) && field.value.length === 0) {
+      if (field.evidence_event_ids.length !== 0) issues.push("empty_tags_must_not_cite");
+    } else if (field.evidence_event_ids.length === 0) {
+      issues.push(`${name}:missing_citation`);
+    }
+    for (const eventId of field.evidence_event_ids) {
+      if (!knownIds.has(eventId)) issues.push(`${name}:unknown_citation`);
+    }
+    if (name === "next_step") {
+      if (field.basis !== "prediction" || field.confidence !== "low") {
+        issues.push("next_step:must_be_low_confidence_prediction");
+      }
+    } else if (field.basis !== "model-synthesis") {
+      issues.push(`${name}:must_be_model_synthesis`);
+    }
+    if (name === "expression_cue" && field.confidence === "high") {
+      issues.push("expression_cue:confidence_must_be_medium_or_low");
+    }
+    if (typeof field.value === "string") {
+      const privacyIssue = semanticPrivacyIssue(field.value);
+      if (privacyIssue) issues.push(`${name}:privacy_${privacyIssue}`);
+      for (const [pattern, issue] of FORBIDDEN_CLAIM_PATTERNS) {
+        if (pattern.test(field.value)) issues.push(`${name}:${issue}`);
+      }
+      if (/\bblocked\b/i.test(field.value) && name !== "attention") {
+        issues.push(`${name}:unsupported_blocker_language`);
+      }
+      if (
+        COMPLETION_LANGUAGE.test(field.value) &&
+        !citationsSupport(field, kindById, RESULT_KINDS)
+      ) {
+        issues.push(`${name}:unsupported_completion_language`);
+      }
+    }
+  }
+
+  if (
+    reply.meaning.recent_result &&
+    !citationsSupport(reply.meaning.recent_result, kindById, RESULT_KINDS)
+  ) {
+    issues.push("recent_result:unsupported_evidence_kind");
+  }
+  if (
+    reply.meaning.attention &&
+    !citationsSupport(reply.meaning.attention, kindById, ATTENTION_KINDS)
+  ) {
+    issues.push("attention:unsupported_evidence_kind");
+  }
+  if (reply.meaning.tags) {
+    for (const tag of reply.meaning.tags.value) {
+      if (!(SEMANTIC_TAGS as readonly string[]).includes(tag)) issues.push("tags:unsupported_tag");
+    }
+  }
+  return issues.length ? { ok: false, issues: [...new Set(issues)] } : { ok: true, value: reply };
+}
+
+export function validateSemanticReadModel(
+  input: unknown,
+  expected?: Pick<SemanticEvidenceV1, "instance_id" | "generation_id" | "evidence_digest">,
+): SemanticValidationResult<SemanticAgentReadModelV2> {
+  if (!Value.Check(SemanticAgentReadModelV2Schema, input)) {
+    return {
+      ok: false,
+      issues: [...Value.Errors(SemanticAgentReadModelV2Schema, input)]
+        .slice(0, 12)
+        .map((error) => `schema:${error.path || "/"}:${error.message}`),
+    };
+  }
+  const value = input as SemanticAgentReadModelV2;
+  const issues: string[] = [];
+  if (expected) {
+    if (value.instance_id !== expected.instance_id) issues.push("instance_mismatch");
+    if (value.generation_id !== expected.generation_id) issues.push("generation_mismatch");
+    if (value.source.evidence_digest !== expected.evidence_digest) {
+      issues.push("evidence_digest_mismatch");
+    }
+  }
+  return issues.length ? { ok: false, issues } : { ok: true, value };
+}
+
+function meaningFields(meaning: SemanticMeaningV2): Array<[string, SemanticField<unknown>]> {
+  return (Object.entries(meaning) as Array<[string, SemanticField<unknown>]>).filter(([, value]) =>
+    Boolean(value && typeof value === "object" && Array.isArray(value.evidence_event_ids)),
+  );
+}
+
+function evidenceKindByEventId(evidence: SemanticEvidenceV1): Map<string, SemanticEvidenceKind> {
+  const map = new Map<string, SemanticEvidenceKind>();
+  if (evidence.task) map.set(evidence.task.event_id, "task");
+  if (evidence.lifecycle) map.set(evidence.lifecycle.event_id, "lifecycle");
+  if (evidence.intent) map.set(evidence.intent.event_id, "intent");
+  if (evidence.operation) map.set(evidence.operation.event_id, "operation");
+  for (const wait of evidence.waits) map.set(wait.event_id, "wait");
+  for (const recent of evidence.recent) map.set(recent.event_id, recent.kind);
+  if (evidence.attention) map.set(evidence.attention.event_id, evidence.attention.kind);
+  return map;
+}
+
+function citationsSupport(
+  field: SemanticField<unknown>,
+  kindById: Map<string, SemanticEvidenceKind>,
+  allowed: Set<SemanticEvidenceKind>,
+): boolean {
+  return field.evidence_event_ids.some((eventId) => {
+    const kind = kindById.get(eventId);
+    return kind !== undefined && allowed.has(kind);
+  });
+}

package/src/lib/browser/client.ts

@@ -42,6 +42,12 @@ import {
   buildRuntsCheck,
   type RuntsResult,
 } from "./runts.js";
+import {
+  buildStandaloneHtmlScript,
+  DEFAULT_MAX_RESOURCE_BYTES,
+  DEFAULT_MAX_TOTAL_RESOURCE_BYTES,
+  type StandaloneHtmlResult,
+} from "./standalone-html.js";
 import {
   buildClearTargetSizeAnnotationsScript,
   buildTargetSizeAnnotateScript,
@@ -1011,6 +1017,21 @@ export class Browser {
     return await page.content();
   }
 
+  /**
+   * Self-contained snapshot of the page: stylesheets inlined, fonts and images
+   * embedded as data: URIs, everything else rewritten to absolute URLs on the
+   * captured origin. Use this for anything written to a file — a raw
+   * `htmlContent()` dump renders unstyled once it leaves the page's own origin.
+   */
+  async standaloneHtml(
+    opts: { maxResourceBytes?: number; maxTotalResourceBytes?: number } = {},
+  ): Promise<StandaloneHtmlResult> {
+    return await this.currentPage.evaluate(buildStandaloneHtmlScript(), {
+      maxResourceBytes: opts.maxResourceBytes ?? DEFAULT_MAX_RESOURCE_BYTES,
+      maxTotalResourceBytes: opts.maxTotalResourceBytes ?? DEFAULT_MAX_TOTAL_RESOURCE_BYTES,
+    });
+  }
+
   async click(selector: string): Promise<void> {
     await this.currentPage.click(selector);
   }

package/src/lib/browser/index.ts

@@ -140,6 +140,10 @@ export {
   sendBrowserSessionRequest,
   startBrowserSessionServer,
 } from "./session-control.js";
+export type {
+  StandaloneHtmlRequest,
+  StandaloneHtmlResult,
+} from "./standalone-html.js";
 export type {
   TargetSizeNode,
   TargetSizeOutcome,