@tangle-network/agent-eval 1.0.17 → 1.0.19

package/dist/app-grade/index.d.ts

@@ -0,0 +1,1042 @@
+import { m as CustomTokenPricing, t as CostLedger } from "../cost-ledger-Czro9wmo.js";
+import "../index-D4lT7aFu.js";
+import { w as ChatTransport, x as ChatClient } from "../types-0KglnQ9o.js";
+import { d as SystemOneRequest, g as SystemOneState, s as SystemOneQuestions } from "../systemone-protocol-C4t7JH4Y.js";
+import "../systemone-CpDppX8s.js";
+import { a as SystemOneReviewReport, r as SystemOneReviewCheck, t as PreparedSystemOneReview } from "../systemone-review-nbuMxxmi.js";
+import { IncomingHttpHeaders, OutgoingHttpHeaders } from "node:http";
+import { Browser, BrowserContext, Locator, Page } from "playwright";
+/**
+ * The tier ladder: a pure function of stored deterministic checks, stored SystemOne
+ * assessments and build gates. Gates, not a weighted blend: strong copy cannot
+ * buy back a failed journey, and an unresolved check never counts as a pass.
+ *
+ *   F  not served, or a floor check of the primary journey failed
+ *   U  unable to measure: no floor check failed, but one could not be observed
+ *      (the payment sits behind a passkey sign-in the grader cannot approve)
+ *   C  the floor passed, but a core SystemOne check is refuted or a journey check failed
+ *   B  every journey check passed and no SystemOne check is refuted
+ *   A  B, plus every core, senior and engineering check supported and the build gates pass
+ *   S  A, plus complete coverage, zero unresolved, and nothing left unmeasured
+ *   INVALID  a failed integrity check or a supported integrity question; the cell goes to human review
+ *
+ * F, U and C follow from deterministic checks alone, so `decidedWithoutReviews`
+ * lets the grader skip model calls that could not change the tier.
+ */
+export type Tier = 'F' | 'U' | 'C' | 'B' | 'A' | 'S';
+/** Ranked tiers. U is not on the ladder: nothing was learned about the payment. */
+export declare const TIER_ORDER: readonly Tier[];
+/** How a SystemOne check can move the tier. */
+export type TierRole = 'core' | 'senior' | 'engineering' | 'integrity';
+/**
+ * The fields of agent-eval's `SystemOneReviewAssessment` the ladder reads. Declared here so the
+ * ladder has no package dependency: the repository's admission gate re-derives stored
+ * verdicts with it.
+ */
+export interface VerdictAssessment {
+  question: string;
+  status: 'supported' | 'refuted' | 'unresolved';
+  reason: string;
+  supportProbability: number;
+  refutationProbability: number;
+}
+export interface VerdictCheck {
+  id: string;
+  status: 'pass' | 'fail' | 'not-applicable' | 'unmeasured';
+  gate: 'floor' | 'journey' | 'engineering' | 'integrity';
+  detail: string;
+}
+export interface VerdictReview {
+  id: string;
+  tierRole: Record<string, TierRole>;
+  assessments: VerdictAssessment[];
+}
+export interface EngineeringGates {
+  install: boolean;
+  typecheck: boolean;
+  build: boolean;
+  tests: 'pass' | 'fail' | 'absent';
+}
+export interface Verdict {
+  tier: Tier;
+  invalid: boolean;
+  reasons: string[];
+  jev: Record<string, {
+    status: VerdictAssessment['status'];
+    role: TierRole;
+    reason: string;
+    support: number;
+    refute: number;
+  }>;
+  unmeasured: string[];
+}
+/** The tier when deterministic checks settle it, before any model call; null when a review could still move it. */
+export declare function decidedWithoutReviews(input: {
+  served: boolean;
+  checks: VerdictCheck[];
+}): 'F' | 'U' | 'C' | null;
+export declare function deriveVerdict(input: {
+  served: boolean;
+  checks: VerdictCheck[];
+  reviews: VerdictReview[];
+  engineering: EngineeringGates;
+  unmeasured: string[];
+}): Verdict;
+export declare const CALIBRATION_SCHEMA = "app-grade-calibration/1";
+export declare const TASK_SCHEMA = "app-grade-task/1";
+export declare const PLANTS: readonly ["reference", "stale", "broken"];
+export type Plant = (typeof PLANTS)[number];
+/** The fewest runs per submission that can show a check separates them every time. */
+export declare const MIN_CALIBRATION_RUNS = 3;
+export interface CalibrationRun {
+  plant: Plant;
+  run: number;
+  served: boolean;
+  checks: VerdictCheck[];
+  reviews: VerdictReview[];
+  engineering: EngineeringGates;
+  unmeasured: string[];
+  /** The tier the grader stored. Admission re-derives it. */
+  tier: Tier;
+  invalid: boolean;
+  costUsd: number;
+  ms: number;
+  /** Run directory relative to the calibration directory. */
+  evidence: string;
+  /** Errors of earlier grading attempts of this run that threw before any verdict existed. */
+  failedAttempts?: string[];
+}
+export interface CalibrationRecord {
+  schema: typeof CALIBRATION_SCHEMA;
+  taskId: string;
+  /** `taskDigest` of the task material that was graded. */
+  taskDigest: string;
+  /** The product image every run started, pinned by digest. */
+  productImage: string;
+  productVersion: string;
+  /** Canonical digest of each submission's files. */
+  solutionDigests: Record<Plant, string>;
+  /** Digest of the App Grade sources that graded the runs. */
+  graderDigest: string;
+  model: string;
+  passTier: Tier;
+  /** The check that encodes the fact a from-memory build gets wrong, when the task names one. */
+  factCheckId?: string;
+  runs: CalibrationRun[];
+  startedAt: string;
+  finishedAt: string;
+  costUsd: number;
+}
+export interface CalibrationDecision {
+  admitted: boolean;
+  reasons: string[];
+  tiers: Record<Plant, Tier[]>;
+}
+export declare function calibrationDecision(record: CalibrationRecord): CalibrationDecision;
+/** RFC 8785 canonical JSON of the value as a JSON file carries it (no undefined fields). */
+export declare function canonicalJson(value: unknown): string;
+export declare function sha256(text: string | Buffer): string;
+export declare function canonicalDigest(value: unknown): string;
+/** Digest of a directory's files by relative path and content: one calibration submission. */
+export declare function treeDigest(dir: string): string;
+/**
+ * The material one App Grade task is graded on. A seed projects to it by removing the
+ * per-task records (`leaves`, `provenance`, `admission`, `validationMode`) and the product
+ * from the suite, and the grader spec from the leaf; a task directory stores it as is.
+ */
+export interface TaskMaterial {
+  id: string;
+  suite: Record<string, unknown>;
+  leaf: Record<string, unknown>;
+  product: unknown;
+  spec: unknown;
+}
+/** One digest binds a calibration record to the exact suite, leaf, product and grader spec. */
+export declare function taskMaterialDigest(task: TaskMaterial): string;
+export declare const WALLET_CHANNEL_BINDING = "__appGradeWalletChannel";
+/** One observation made inside a wallet window's own realm. */
+export interface WalletRecord {
+  kind: 'message' | 'decrypt' | 'encrypt';
+  /** `message`: the sender origin as the browser reported it. */
+  from?: string;
+  /** `message`: `event.isTrusted`; a page cannot dispatch a trusted message. */
+  trusted?: boolean;
+  data?: unknown;
+  /** `decrypt`: a request the wallet decrypted. `encrypt`: a response it encrypted. */
+  plaintext?: string;
+  t: number;
+}
+/** Page code that called the grader's wallet channel from outside a wallet window. */
+export interface ForgeryAttempt {
+  journey: string;
+  frameUrl: string;
+  payload: string;
+  t: number;
+}
+export interface WalletWindow {
+  opener: Page;
+  page: Page;
+  url: string;
+  records: WalletRecord[];
+}
+/**
+ * Page-side rule for what a sighted buyer can see, evaluated in the grader's
+ * isolated world. The expression yields `painted(node)` for a text node or an
+ * element (a string: tsx wraps named functions in a `__name` helper the page lacks).
+ *
+ * Excluded: hidden, transparent, faded (cumulative opacity under 0.1), tiny, and
+ * clipped or off-page boxes. A text node also needs a readable colour and size.
+ * An element needs a box of at least 8 x 8 px and something drawn in it: painted
+ * text, an image, a native form widget, or its own background or border. A
+ * control a buyer cannot see is never offered to the picker or read as the order.
+ */
+export declare const PAINTED = "(() => {\n  const alpha = (color) => {\n    const match = /rgba?\\(([^)]+)\\)/.exec(color || '')\n    if (!match) return color === 'transparent' ? 0 : 1\n    const parts = match[1].split(/[ ,\\/]+/).filter(Boolean)\n    return parts.length >= 4 ? Number.parseFloat(parts[3]) : 1\n  }\n  const docWidth = () => Math.max(document.documentElement.scrollWidth, innerWidth)\n  const docHeight = () => Math.max(document.documentElement.scrollHeight, innerHeight)\n  const readable = (element) => {\n    const style = getComputedStyle(element)\n    return Number.parseFloat(style.fontSize) >= 6 && alpha(style.color) >= 0.1 && alpha(style.webkitTextFillColor) >= 0.1\n  }\n  /** Painted area of `rects` inside `element`'s clipping ancestors and the page; 0 when hidden or faded. */\n  const area = (element, rects, minWidth, minHeight) => {\n    if (!element.checkVisibility({ opacityProperty: true, visibilityProperty: true, contentVisibilityAuto: true })) return 0\n    let opacity = 1\n    for (let at = element; at; at = at.parentElement) opacity *= Number.parseFloat(getComputedStyle(at).opacity)\n    if (opacity < 0.1) return 0\n    let total = 0\n    for (const rect of rects) {\n      let left = rect.left + scrollX, top = rect.top + scrollY, right = rect.right + scrollX, bottom = rect.bottom + scrollY\n      for (let at = element; at && at !== document.documentElement; at = at.parentElement) {\n        const s = getComputedStyle(at)\n        const clipsOverflow = ['hidden', 'clip'].includes(s.overflowX) || ['hidden', 'clip'].includes(s.overflowY)\n        const clipRect = s.clip && s.clip !== 'auto'\n        const clipPath = s.clipPath && s.clipPath !== 'none'\n        if (clipRect && /rect\\(\\s*0(px)?[ ,]+0(px)?[ ,]+0(px)?[ ,]+0(px)?\\s*\\)/.test(s.clip)) return 0\n        if (clipPath && /inset\\(\\s*(50|[5-9]\\d|100)%/.test(s.clipPath)) return 0\n        if (clipsOverflow || clipRect || clipPath) {\n          const box = at.getBoundingClientRect()\n          left = Math.max(left, box.left + scrollX)\n          top = Math.max(top, box.top + scrollY)\n          right = Math.min(right, box.right + scrollX)\n          bottom = Math.min(bottom, box.bottom + scrollY)\n        }\n      }\n      left = Math.max(left, 0)\n      top = Math.max(top, 0)\n      right = Math.min(right, docWidth())\n      bottom = Math.min(bottom, docHeight())\n      if (right - left >= minWidth && bottom - top >= minHeight) total += (right - left) * (bottom - top)\n    }\n    return total\n  }\n  const cache = new Map()\n  const text = (node) => {\n    const parent = node.parentElement\n    if (!parent || !readable(parent)) return false\n    const range = document.createRange()\n    range.selectNodeContents(node)\n    return area(parent, [...range.getClientRects()], 2, 4) >= 16\n  }\n  const drawsSomething = (element) => {\n    const tag = element.tagName\n    if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT') {\n      return ['checkbox', 'radio'].includes(element.type) || readable(element)\n    }\n    const style = getComputedStyle(element)\n    if (alpha(style.backgroundColor) >= 0.1 || style.backgroundImage !== 'none') return true\n    if (Number.parseFloat(style.borderTopWidth) + Number.parseFloat(style.borderBottomWidth) > 0 && alpha(style.borderTopColor) >= 0.1) return true\n    for (const media of element.querySelectorAll('img, svg, canvas, video')) {\n      if (area(media, [...media.getClientRects()], 4, 4) >= 16) return true\n    }\n    const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT)\n    while (walker.nextNode()) {\n      if ((walker.currentNode.nodeValue || '').trim() && painted(walker.currentNode)) return true\n    }\n    return false\n  }\n  const painted = (node) => {\n    if (!cache.has(node)) {\n      cache.set(node, node.nodeType === Node.TEXT_NODE ? text(node) : area(node, [...node.getClientRects()], 8, 8) >= 64 && drawsSomething(node))\n    }\n    return cache.get(node)\n  }\n  return painted\n})()";
+export declare class EvidenceBrowser {
+  private readonly browser;
+  /** What this browser reports as its user agent; vendor SDKs send it in the app's traffic. */
+  readonly userAgent: string;
+  private constructor();
+  /** `executablePath`: a Chromium already on the machine, such as a Sandbox box's, instead of Playwright's download. */
+  static launch(input?: {
+    headless?: boolean;
+    executablePath?: string;
+  }): Promise<EvidenceBrowser>;
+  /** A fresh browser context per journey: its own storage, as a new buyer has. Without a wallet origin, no page records wallet evidence. */
+  journey(input: {
+    id: string;
+    dir: string;
+    appUrl: string;
+    walletOrigin?: string;
+  }): Promise<JourneySession>;
+  close(): Promise<void>;
+}
+export declare class JourneySession {
+  readonly id: string;
+  readonly dir: string;
+  readonly context: BrowserContext;
+  readonly appOrigin: string;
+  readonly walletOrigin: string | null;
+  readonly walletWindows: WalletWindow[];
+  readonly forgeryAttempts: ForgeryAttempt[];
+  private readonly pendingBodies;
+  /** Wallet-origin records that arrived before the opener's popup event; claimed when it fires. */
+  private readonly unclaimed;
+  private readonly cdp;
+  private constructor();
+  static open(browser: Browser, input: {
+    id: string;
+    dir: string;
+    appUrl: string;
+    walletOrigin?: string;
+  }): Promise<JourneySession>;
+  private receive;
+  private watch;
+  append(file: string, value: unknown): void;
+  write(file: string, value: string | Buffer): void;
+  newPage(): Promise<Page>;
+  /** Windows at the wallet origin that `opener` opened, in open order, skipping the first `since` of them. */
+  walletWindowsOf(opener: Page, since?: number): WalletWindow[];
+  /**
+   * Evaluate `expression` in the grader's isolated world of the page's main frame.
+   * The world persists for the life of the document, so it can hold element references.
+   */
+  graderEval<T>(page: Page, expression: string): Promise<T>;
+  /** Accessibility snapshot and screenshot of one moment, saved under this journey's directory. */
+  capture(page: Page, label: string): Promise<{
+    aria: string;
+    ariaPath: string;
+    screenshotPath: string;
+    url: string;
+  }>;
+  close(): Promise<void>;
+}
+export interface AppGradeJudge {
+  /** The reference every receipt and review digest names, e.g. `router:gpt-5.6-luna`. */
+  readonly ref: string;
+  /** The model id each request asks the client for. */
+  readonly model: string;
+  readonly client: ChatClient;
+  /** Rate the ledger records when a call reports no cost. */
+  readonly pricing: CustomTokenPricing;
+  /** Hard per-call reservation rate, at or above every rate the client bills. */
+  readonly bound: CustomTokenPricing;
+  /**
+   * For a judge that bills $0 per call (a subscription): what its calls so far would cost at
+   * list price, as the judge reports it, or null when any call's figure is unknown. The
+   * manifest records it as `notionalUsd`; it is never billed.
+   */
+  readonly notionalUsd?: () => number | null;
+}
+export interface JudgeReply {
+  content: string;
+  servedModel: string;
+  inputTokens: number;
+  outputTokens: number;
+  /** Dollars the client reported for the call, or null. */
+  costUsd: number | null;
+  /** Names of tools the model called while answering; a judge should call none. */
+  toolCalls: string[];
+}
+/** One JSON chat call. A failed call throws the client's error. */
+export declare function askJudge(judge: AppGradeJudge, input: {
+  system: string;
+  user: string;
+  maxTokens: number;
+  signal?: AbortSignal;
+  idempotencyKey?: string;
+}): Promise<JudgeReply>;
+/** A judge's JSON reply, without a fence the model wrapped it in. */
+export declare function parseJudgeJson(content: string): Record<string, unknown>;
+export declare const INTERACTIVE = "button, a[href], input, select, textarea, [role=button], [role=link], [role=checkbox], [role=switch], [role=tab]";
+export interface Control {
+  index: number;
+  tag: string;
+  role: string;
+  type: string;
+  name: string;
+  value: string;
+  disabled: boolean;
+}
+export declare function listControls(session: JourneySession, page: Page): Promise<Control[]>;
+export declare function controlLocator(page: Page, control: Control): Locator;
+export interface TargetPick {
+  goal: string;
+  control: Control | null;
+  reason: string;
+}
+export declare function targetPicker(input: {
+  judge: AppGradeJudge;
+  costLedger: CostLedger;
+}): ((session: JourneySession, page: Page, goal: string, context?: string) => Promise<TargetPick>) & {
+  stats: {
+    asked: number;
+    cached: number;
+    toolCalls: string[];
+  };
+};
+export type Picker = ReturnType<typeof targetPicker>;
+export interface ReviewSpec {
+  id: string;
+  questions: SystemOneQuestions;
+  state: unknown;
+  checks: Record<string, Omit<SystemOneReviewCheck, 'evidence'> & {
+    evidence: SystemOneReviewCheck['evidence'];
+  }>;
+  tierRole: Record<string, TierRole>;
+}
+export interface StoredReview {
+  id: string;
+  review: PreparedSystemOneReview;
+  raw: unknown;
+  report: SystemOneReviewReport;
+  tierRole: Record<string, TierRole>;
+  costUsd: number;
+}
+export declare function runReviews(input: {
+  specs: ReviewSpec[];
+  /** Review definition version; part of every review digest. */
+  version: string;
+  /** The judge; its reference is part of every review digest. */
+  judge: AppGradeJudge;
+  costLedger: CostLedger;
+  dir: string;
+}): Promise<StoredReview[]>;
+/** Offline re-score: same frozen review, same raw answer, no inference. */
+export declare function rescore(entry: Pick<StoredReview, 'review' | 'raw'>): SystemOneReviewReport;
+interface Passage {
+  source: string;
+  text: string;
+}
+export interface IntegrityEvidence {
+  /** Lines of app source that probe for an automated browser. */
+  automationProbes: Array<{
+    file: string;
+    line: number;
+    excerpt: string;
+  }>;
+  candidates: Array<{
+    source: string;
+    excerpt: string;
+  }>;
+  screens: Passage[];
+  console: Passage[];
+  network: Passage[];
+  source: Passage[];
+  counts: {
+    sourceFiles: number;
+    screens: number;
+    consoleLines: number;
+    networkPayloads: number;
+  };
+}
+export declare function collectIntegrityEvidence(input: {
+  workdir: string;
+  runDir: string;
+  appOrigin: string;
+  graderUserAgent: string | null;
+}): IntegrityEvidence;
+export declare function integrityReviewSpec(task: string, evidence: IntegrityEvidence): ReviewSpec;
+/** What one leaf contributes to a grading run: its journeys, its deterministic checks and its SystemOne reviews. */
+export interface GradeLeaf<E extends {
+  journeyMs: Record<string, number>;
+}> {
+  id: string;
+  /** Review definition version; part of every review digest. */
+  reviewVersion: string;
+  /** Flows this grader cannot operate; any entry caps the tier below S. */
+  unmeasured: readonly string[];
+  /** Operate the journeys, each in its own browser context, writing evidence under `dir`. */
+  runJourneys(browser: EvidenceBrowser, dir: string, url: string, pick: Picker): Promise<E>;
+  /** Accessibility snapshot path of the first screen, for the `serve.rendered` floor check. */
+  landing(evidence: E): string | undefined;
+  deterministicChecks(evidence: E): VerdictCheck[];
+  /**
+   * SystemOne reviews over raw evidence only: what the app showed, what the grader did, and the
+   * source. No deterministic check status, build gate or derived verdict reaches a model.
+   */
+  reviewSpecs(input: {
+    evidence: E;
+    readCapture: (path: string) => string;
+    workdir: string;
+    integrity: IntegrityEvidence;
+  }): ReviewSpec[];
+}
+export interface GradeResult {
+  verdict: Verdict;
+  /** Everything `deriveVerdict` read, so a stored run re-derives without the grader. */
+  inputs: {
+    served: boolean;
+    checks: VerdictCheck[];
+    reviews: VerdictReview[];
+    engineering: EngineeringGates;
+    unmeasured: string[];
+  };
+  checks: VerdictCheck[];
+  costUsd: number;
+  phaseMs: Record<string, number>;
+}
+export declare function grade<E extends {
+  journeyMs: Record<string, number>;
+}>(input: {
+  leaf: GradeLeaf<E>;
+  workdir: string;
+  engineeringPath: string;
+  out: string;
+  port: number;
+  judge: AppGradeJudge;
+  /** Environment the served app receives beyond PATH, HOME, NODE_ENV and PORT. */
+  appEnv?: Record<string, string>;
+  browser?: Parameters<typeof EvidenceBrowser.launch>[0];
+}): Promise<GradeResult>;
+export declare function prepare(workdir: string): {
+  gates: EngineeringGates;
+  steps: Record<string, {
+    command: string;
+    exit: number | null;
+    ms: number;
+    tail: string;
+  } | {
+    skipped: string;
+  }>;
+};
+export type Json = null | boolean | number | string | Json[] | {
+  [key: string]: Json;
+};
+/** The real product: its official image run fresh for every grading run, or a vendor's hosted test mode. */
+export type ProductSpec = ImageProductSpec | HostedProductSpec;
+export interface ImageProductSpec extends ProductBase {
+  /** Official image pinned by digest: `<repository>:<tag>@sha256:<64 hex>`. */
+  image: string;
+  /** Container port the product listens on. */
+  port: number;
+  /** Container environment. `{{adminSecret}}` is the grader's per-run admin secret. */
+  env?: Record<string, string>;
+  /** Arguments after the image entrypoint. They must not carry the admin secret. */
+  args?: string[];
+  /** Unauthenticated readiness probe; GET unless `method` says otherwise, as `versionProbe`. */
+  ready: {
+    path: string;
+    method?: HttpMethod;
+    body?: Json;
+    status?: number;
+  };
+  readyTimeoutMs?: number;
+  hosted?: undefined;
+}
+/**
+ * A vendor's hosted test environment (Stripe test mode) that no image can stand up. The grader
+ * reaches `origin` with the secret in `secretEnv`; the app reaches it only through the recording
+ * proxy, which swaps the app's per-run placeholder key for that secret, so the secret never
+ * enters the app, its logs or the evidence.
+ */
+export interface HostedProductSpec extends ProductBase {
+  hosted: {
+    origin: string;
+    secretEnv: string;
+    /** Prefix of the app's per-run placeholder key, e.g. `sk_test_vb_`. */
+    appKeyPrefix: string;
+    /** Headers the grader's admin requests carry, e.g. a pinned `Stripe-Version`; app requests keep their own. */
+    headers?: Record<string, string>;
+    /** Admin request bodies are form-encoded with bracketed keys (Stripe) instead of JSON. */
+    formBodies?: boolean;
+    /**
+     * A fresh tenant per run (a Stripe connected account), so concurrent runs share nothing: created
+     * by `create`, ready when GET `<create.path>/<id>` reads `ready.equals` at `ready.jsonPath`, deleted
+     * on stop. Every admin and app request carries its id in `header`; app variable `appEnv` holds it.
+     */
+    tenant?: {
+      create: {
+        path: string;
+        body: Json;
+      };
+      header: string;
+      ready: {
+        jsonPath: string;
+        equals: Json;
+        timeoutMs?: number;
+      };
+      appEnv?: string;
+    };
+    /** The grader relays the run's events from `path` to the app's `route`, signed with the per-run secret in app variable `secretEnv`. */
+    events?: {
+      path: string;
+      route: string;
+      secretEnv: string;
+    };
+  };
+  image?: undefined;
+}
+interface ProductBase {
+  /** Product name, lowercase, e.g. `meilisearch`. */
+  name: string;
+  /** The version the running product must report through `versionProbe`. */
+  version: string;
+  /**
+   * Admin-authenticated request (GET by default; a JSON-RPC node takes POST `/` with its envelope
+   * as `body`) whose JSON value at `jsonPath` is the running version. A hosted product states its
+   * test mode here: Stripe's `/v1/balance` `livemode` must read `false`.
+   */
+  versionProbe: {
+    path: string;
+    method?: HttpMethod;
+    body?: Json;
+    jsonPath: string;
+  };
+  /** How admin requests authenticate: `<header>: <prefix><adminSecret>`. */
+  auth: {
+    header: string;
+    prefix?: string;
+  };
+  /**
+   * Environment names the app receives at runtime: `url` holds the product URL (the grader's
+   * recording proxy in front of the product), `key` holds `appKey` when set.
+   */
+  appEnv: {
+    url: string;
+    key?: string;
+  };
+  /**
+   * The credential the app receives in `appEnv.key`, a template over the setup context,
+   * for example `{{vars.appKey}}` captured by a setup step. Omit when the app needs none, and
+   * for a hosted product, whose app key is its per-run placeholder.
+   */
+  appKey?: string;
+}
+export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
+/** One admin request the grader makes against the product before the app starts. */
+export interface HttpStep {
+  name: string;
+  method: HttpMethod;
+  /** Path and query, templated. */
+  path: string;
+  /** JSON body, templated. */
+  body?: Json;
+  /** Accepted statuses. Default: any 2xx. */
+  expectStatus?: number[];
+  /** Values to keep for later templates: `vars.<name>` = the response JSON at the path. */
+  capture?: Record<string, string>;
+  /** Poll a GET until the JSON value at `jsonPath` equals `equals` (asynchronous writes). */
+  waitFor?: {
+    path: string;
+    jsonPath: string;
+    equals: Json;
+    timeoutMs?: number;
+  };
+}
+/** A value drawn fresh for every grading run, so a submission cannot hard-code it. */
+export type ProbeSpec = {
+  kind: 'word';
+  prefix?: string;
+} | {
+  kind: 'int';
+  min: number;
+  max: number;
+} | {
+  kind: 'pick';
+  from: Json[];
+};
+export type JourneyStep = {
+  open: string;
+} | {
+  fill: {
+    goal: string;
+    value: string;
+  };
+} | {
+  select: {
+    goal: string;
+    value: string;
+  };
+} | {
+  click: {
+    goal: string;
+  };
+} | {
+  press: string;
+} | {
+  wait: number;
+} | {
+  waitForText: {
+    text: string;
+    timeoutMs?: number;
+  };
+} | {
+  capture: string;
+};
+/** A user path through the running app. `fill`, `select` and `click` goals are resolved by the target picker. */
+export interface JourneySpec {
+  id: string;
+  steps: JourneyStep[];
+}
+/** An admin read of the product after the journeys: what the product itself holds or answers. */
+export interface ReadSpec {
+  name: string;
+  method: 'GET' | 'POST';
+  path: string;
+  body?: Json;
+}
+/** Which proxied app-to-product requests a check counts. Strings are templates. */
+export interface RequestMatch {
+  method?: HttpMethod;
+  /** Regular expression over the request path (without query). */
+  path?: string;
+  credential?: 'app' | 'admin' | 'none' | 'other';
+  from?: 'server' | 'browser';
+  /** Every value must appear in the request body text. */
+  bodyIncludes?: string[];
+  /** Every value must appear in the query string. */
+  queryIncludes?: string[];
+  status?: number[];
+}
+export type CheckKind = {
+  kind: 'journeyCompleted';
+  journey: string;
+} | {
+  kind: 'pageContains';
+  capture: string;
+  values: string[];
+  mode?: 'all' | 'any';
+} | {
+  kind: 'pageLacks';
+  capture: string;
+  values: string[];
+} | {
+  kind: 'productRequests';
+  match: RequestMatch;
+  min?: number;
+  max?: number;
+} | {
+  kind: 'readMatches';
+  read: string;
+  jsonPath: string;
+  equals?: Json;
+  includes?: Json;
+  length?: {
+    min?: number;
+    max?: number;
+  };
+} |
+/**
+ * The strings at `jsonPath` (an array; `field` picks a property of each element) in a read,
+ * first `limit` of them: `shown` requires each on the page, `ordered` also in that order,
+ * `absent` requires none on the page.
+ */
+{
+  kind: 'pageRead';
+  capture: string;
+  read: string;
+  jsonPath: string;
+  field?: string;
+  limit?: number;
+  expect: 'shown' | 'ordered' | 'absent';
+};
+/** `floor` failures give F; `journey` failures give C. See verdict.ts. */
+export type CheckSpec = {
+  id: string;
+  gate: 'floor' | 'journey';
+  why: string;
+} & CheckKind;
+export type SystemOneQuestionSpec = {
+  type: 'noul';
+  instructions: string;
+} | {
+  type: 'score';
+  instructions: string;
+  criteria: string[];
+} | {
+  type: 'choice';
+  instructions: string;
+  criteria: Record<string, string>;
+};
+/** One bounded SystemOne question with how its answer moves the tier. */
+export interface ReviewQuestionSpec {
+  question: SystemOneQuestionSpec;
+  claim: string;
+  area: string;
+  severity: 'info' | 'low' | 'medium' | 'high' | 'critical';
+  /** Native answers that support the claim (`true`/`false`, score levels, or choice keys). */
+  supports: string[];
+  refutes: string[];
+  role: TierRole;
+}
+/** A SystemOne review over named evidence. */
+export interface ReviewSpecData {
+  id: string;
+  evidence: {
+    captures?: string[];
+    reads?: string[];
+    productLog?: boolean;
+    code?: boolean;
+  };
+  questions: Record<string, ReviewQuestionSpec>;
+}
+/** The grader side of one task. Contestants never see it. */
+export interface AppGradeSpec {
+  /** The task as the reviewer should understand it (not the contestant prompt). */
+  task: string;
+  probes: Record<string, ProbeSpec>;
+  /** Admin requests before the app starts: seed data, settings, the app's credential. */
+  setup: HttpStep[];
+  journeys: JourneySpec[];
+  reads: ReadSpec[];
+  checks: CheckSpec[];
+  reviews: ReviewSpecData[];
+  /** Flows this grader cannot operate; any entry caps the tier below S. */
+  unmeasured: string[];
+  /** The lowest tier a correct submission earns; admission requires the reference to reach it. */
+  passTier: Exclude<Tier, 'F' | 'C'>;
+  /**
+   * The floor or journey check that encodes the current-product fact a from-memory build gets
+   * wrong. Calibration requires the reference to pass it and the stale build to fail it every run.
+   */
+  factCheck?: string;
+}
+/** Value at a dotted path (`hits.0.title`); undefined when any segment is missing. */
+export declare function jsonPath(value: unknown, path: string): unknown;
+/**
+ * Render `{{a.b}}` references against `context`. A string that is exactly one reference
+ * takes the referenced value itself, so numbers and arrays survive. A missing reference
+ * throws: a template that renders to nothing would grade against the wrong value.
+ */
+export declare function render<T extends Json | undefined>(template: T, context: object): T;
+/** Every `{{…}}` reference in a value, for linting. */
+export declare function references(value: unknown): string[];
+/** What calibration binds: the digest-pinned image, or the hosted origin with its pinned headers. */
+export declare function productIdentity(product: ProductSpec): string;
+/** The image digest a provenance record freezes; a hosted product has none. */
+export declare function productImageDigest(product: ProductSpec | undefined): string | undefined;
+export declare function lintProductSpec(product: ProductSpec): string[];
+/**
+ * Structural problems in a task spec. A spec that passes can still be wrong; calibration
+ * decides that. This only refuses specs whose checks cannot discriminate by construction.
+ */
+export declare function lintAppGradeSpec(spec: AppGradeSpec, product: ProductSpec | undefined): string[];
+/** Remove only products started by one App Grade process, including an interrupted startup. */
+export declare function cleanupAppGradeProducts(runId: string, followLateCreate?: boolean): Promise<void>;
+export interface ProductContext {
+  probe: Record<string, Json>;
+  vars: Record<string, Json>;
+}
+export interface RunningProduct {
+  spec: ProductSpec;
+  /** Where grader requests go: the container's host URL or the vendor's origin. */
+  url: string;
+  reportedVersion: string;
+  /** A hosted product's per-run placeholder app key; an image product's key comes from setup. */
+  appKey: string | null;
+  /** App variables the product itself sets: a hosted tenant id and webhook secret. */
+  appEnv: Record<string, string>;
+  /** Admin request against the product. The response body is parsed JSON when it is JSON. */
+  admin(method: string, path: string, body?: Json): Promise<{
+    status: number;
+    body: unknown;
+  }>;
+  /** Values that must never appear in evidence: the admin secret and minted credentials. */
+  secrets(): string[];
+  addSecret(value: string): void;
+  credentialOf(headers: IncomingHttpHeaders, appKey: string | null): 'admin' | 'app' | 'none' | 'other';
+  /** The headers an app request goes upstream with: product host, tenant scope, placeholder swapped. */
+  forward(headers: IncomingHttpHeaders): OutgoingHttpHeaders;
+  stop(): Promise<void>;
+}
+/** `untracked` means startup failed before a caller could retain the product for cleanup retry. */
+export declare class ProductCleanupError extends AggregateError {
+  readonly untracked: boolean;
+  constructor(errors: unknown[], message: string, untracked?: boolean);
+}
+/** Fresh values for one run. A word probe is a pronounceable token no model has seen with this task. */
+export declare function drawProbes(probes: Record<string, ProbeSpec>): Record<string, Json>;
+/**
+ * The vendor cannot reach a grader-local app, so the grader delivers the run's events: it lists
+ * them and POSTs each, oldest first, to the app's webhook route until the app answers 2xx (five
+ * tries), signed with the run's secret in Stripe's `t=<unix>,v1=<HMAC-SHA256("<t>.<body>")>` scheme.
+ */
+export declare function startEventRelay(product: RunningProduct, appUrl: string, logPath: string): {
+  stop(): Promise<void>;
+};
+/** Start the pinned image, or a hosted tenant, and wait until it answers as the declared version. */
+export declare function startProduct(spec: ProductSpec, options?: {
+  logPath?: string;
+  signal?: AbortSignal;
+}): Promise<RunningProduct>;
+/** Run the setup steps as admin, in order. Returns the captured variables. */
+export declare function runSetup(product: RunningProduct, steps: HttpStep[], context: ProductContext, log: (entry: unknown) => void): Promise<void>;
+/** Admin reads after the journeys: what the product itself holds or answers. */
+export declare function runReads(product: RunningProduct, reads: ReadSpec[], context: ProductContext): Promise<Record<string, {
+  status: number;
+  body: unknown;
+}>>;
+/** One app request the proxy forwarded to the product. */
+export interface ProductRequest {
+  t: number;
+  method: string;
+  path: string;
+  query: string;
+  credential: 'admin' | 'app' | 'none' | 'other';
+  /** `browser` when the request carried browser fetch metadata (Origin, Sec-Fetch-Site or Sec-Fetch-Dest). */
+  from: 'server' | 'browser';
+  status: number | null;
+  body: string;
+  response: string;
+}
+export interface RecordingProxy {
+  url: string;
+  requests: ProductRequest[];
+  close(): Promise<void>;
+}
+export declare function redact(text: string, secrets: string[]): string;
+/**
+ * A pass-through proxy on 127.0.0.1 in front of the product. It forwards every request with the
+ * product's headers (`RunningProduct.forward`) and records method, path, query, credential
+ * identity, origin kind, status and truncated bodies, with the grader's secrets redacted.
+ */
+export declare function startRecordingProxy(product: RunningProduct, appKey: string | null, logPath: string): Promise<RecordingProxy>;
+export interface ServeAttestation {
+  command: string[];
+  pid: number | null;
+  portOwnerPid: number | null;
+  portOwnerCommand: string | null;
+  /** The listener belongs to the process group this call spawned, not to a server left on the port. */
+  portOwnerIsChild: boolean;
+  readyAfterMs: number | null;
+  lastStatus: number | null;
+  isScaffoldPreview: boolean;
+}
+export interface ServedApp {
+  url: string;
+  attestation: ServeAttestation;
+  stop: () => Promise<void>;
+}
+/** The process listening on `port`, or all nulls when none does or the lookup fails. */
+export declare function portOwner(port: number): {
+  pid: number | null;
+  pgid: number | null;
+  command: string | null;
+};
+export declare function serveApp(input: {
+  workdir: string;
+  port: number;
+  logPath: string;
+  readyTimeoutMs?: number;
+  env?: Record<string, string>;
+}): Promise<ServedApp>;
+export interface SourceFile {
+  path: string;
+  content: string;
+}
+/** Every text file the submission can serve or build from, within `extensions`. */
+export declare function sourceFiles(workdir: string, extensions?: RegExp): SourceFile[];
+/** Files that touch payment or tests first, within a fixed character budget. */
+export declare function codeFiles(workdir: string, budget?: number): {
+  files: {
+    path: string;
+    lines: string;
+    content: string;
+  }[];
+  relevantCount: number;
+  total: number;
+};
+/**
+ * Longest app screen and source file any review quotes. The integrity review quotes
+ * each passage at least this far, so no review reads app text it did not.
+ */
+export declare const SCREEN_CHARS = 20000;
+export declare const SOURCE_FILE_CHARS = 20000;
+/** App-originated text enters SystemOne state as quoted, untrusted data with its provenance. */
+export declare function untrusted(source: string, text: string, max?: number): {
+  untrusted: boolean;
+  source: string;
+  text: string;
+};
+export interface CaptureRecord {
+  journey: string;
+  ariaPath: string;
+  textPath: string;
+  screenshotPath: string;
+  url: string;
+}
+export interface SpecEvidence {
+  journeys: Record<string, {
+    completed: boolean;
+    problems: string[];
+  }>;
+  captures: Record<string, CaptureRecord>;
+  reads: Record<string, {
+    status: number;
+    body: unknown;
+  }>;
+  productRequests: ProductRequest[];
+  probes: Record<string, Json>;
+  journeyMs: Record<string, number>;
+}
+export declare function evaluateChecks(spec: AppGradeSpec, evidence: SpecEvidence, readText: (path: string) => string, context: ProductContext): VerdictCheck[];
+/**
+ * SystemOne reviews over the evidence itself. The deterministic checks' statuses and the build
+ * gates stay out of the reviewer's state: a reviewer that reads the grader's own verdicts
+ * grades the grader, not the app.
+ */
+export declare function specReviews(input: {
+  spec: AppGradeSpec;
+  product: RunningProduct['spec'];
+  evidence: SpecEvidence;
+  readText: (path: string) => string;
+  workdir: string;
+}): ReviewSpec[];
+/** Build the grade leaf for one spec-driven task against a started product and proxy. */
+export declare function specLeaf(input: {
+  id: string;
+  /** Digest of the task material; part of every review digest, so a changed spec never reuses a review. */
+  specDigest: string;
+  spec: AppGradeSpec;
+  product: RunningProduct;
+  context: ProductContext;
+  requests: () => ProductRequest[];
+  readText: (path: string) => string;
+}): GradeLeaf<SpecEvidence>;
+/** Output cap per SystemOne request, including reasoning tokens; also bounds the ledger reservation. */
+export declare const SYSTEM_ONE_MAX_OUTPUT_TOKENS = 6000;
+/** Paid work that produced no valid answer; carries usage so the ledger still records it. */
+export declare class SystemOneTransportError extends Error {
+  readonly model: string;
+  readonly usage: {
+    input_tokens: number;
+    output_tokens: number;
+  };
+  readonly costUsd: number | null;
+  constructor(message: string, model: string, usage: {
+    input_tokens: number;
+    output_tokens: number;
+  }, costUsd: number | null, options?: {
+    cause?: unknown;
+  });
+}
+/**
+ * One transport call may make up to `malformedRetries + 1` paid requests when the
+ * model returns unusable JSON. Their usage and billed cost are summed into the one
+ * result (or error) the ledger records, so no paid attempt goes unaccounted.
+ */
+export declare function chatSystemOneTransport(options: {
+  judge: AppGradeJudge;
+  malformedRetries?: number;
+}): (request: SystemOneRequest, context: {
+  signal: AbortSignal;
+  idempotencyKey: string;
+}) => Promise<{
+  model: string;
+  answers: {
+    [k: string]: {
+      type: "noul";
+      noul: number;
+      score?: undefined;
+      confidence?: undefined;
+      legend?: undefined;
+      probabilities?: undefined;
+      choice?: undefined;
+    } | {
+      type: "score";
+      score: number;
+      confidence: number;
+      legend: {
+        [k: string]: SystemOneState;
+      };
+      probabilities: Record<string, number>;
+      noul?: undefined;
+      choice?: undefined;
+    } | {
+      type: "choice";
+      choice: string;
+      confidence: number;
+      probabilities: Record<string, number>;
+      noul?: undefined;
+      score?: undefined;
+      legend?: undefined;
+    };
+  };
+  usage: {
+    input_tokens: number;
+    output_tokens: number;
+  };
+  transport: {
+    route: string;
+    via: ChatTransport;
+    paidRequests: number;
+    costUsd: number | null;
+    toolCalls: string[];
+  };
+}>;
+export interface AppGradeTask {
+  schema: typeof TASK_SCHEMA;
+  id: string;
+  /** The suite fields the caller builds the contestant's scaffold and prompt from. */
+  suite: Record<string, unknown>;
+  /** What the contestant sees. It never carries the grader spec. */
+  leaf: {
+    id: string;
+  } & Record<string, unknown>;
+  product: ProductSpec;
+  spec: AppGradeSpec;
+}
+export interface LoadedTask extends AppGradeTask {
+  dir: string;
+  /** Digest of the complete task material (suite, leaf, product, spec). */
+  digest: string;
+}
+export declare function taskProblems(task: AppGradeTask): string[];
+export declare function loadTask(dir: string): LoadedTask;
+export declare function taskDigest(task: AppGradeTask): string;
+/** One grading run of a served submission against a fresh instance of the task's product. */
+export declare function gradeTask(input: {
+  task: AppGradeTask & {
+    digest: string;
+  };
+  workdir: string;
+  engineeringPath: string;
+  out: string;
+  port: number;
+  judge: AppGradeJudge;
+  browser?: Parameters<typeof grade>[0]['browser'];
+}): Promise<GradeResult & {
+  probes: Record<string, unknown>;
+}>;
+export {};
+//# sourceMappingURL=index.d.ts.map