@ggui-ai/ui-gen 0.24.0 → 0.26.0

package/src/evaluation/types-public.ts

@@ -20,6 +20,7 @@
 
 export type Priority = "P0" | "P1" | "P2";
 
+import type { VisionFinishReason } from '../harness/llm-router.js';
 import type { GenerationRuntimeProbeOutcome, GenerationRuntimeProbeStatus } from "@ggui-ai/mcp-server-core";
 import type { CanvasClass, DesignMode } from "../design-mode.js";
 
@@ -61,6 +62,11 @@ export interface EvalIssue {
   description: string;
   fix: string;
   line?: number;
+  /**
+   * The visual leg's provenance (ggui#1545): `judge` (the model's finding) or `instrument` (a deterministic
+   * measurement). Absent on every other leg's issues. `visualEvaluation.actOn` acts on `judge` only.
+   */
+  origin?: "judge" | "instrument";
 }
 
 /**
@@ -100,6 +106,23 @@ export interface ProbeHostLoad {
   readonly cores: number;
 }
 
+/**
+ * ggui#1398 — whether the controls the probe clicked showed they were
+ * working once their action was pending. Report-only: it is read by the
+ * benchmark and never counted into the probe's warnings or fed back to
+ * the coding loop.
+ */
+export interface ProbePendingAffordance {
+  /** Controls the pending phase marked pending: those whose click dispatched their action (click or submit wiring) and that were still in the page. `visible + missing.length === dispatched`. */
+  readonly dispatched: number;
+  /** Of those, how many visibly changed while pending (disabled, busy, text, or replaced). */
+  readonly visible: number;
+  /** The actions whose control did not visibly change, including one whose card threw while pending. */
+  readonly missing: readonly string[];
+  /** Actions whose control was no longer in the page when its turn came (the card replaced it, or blanked after a throw): never marked, not counted as dispatched. */
+  readonly gone: readonly string[];
+}
+
 /**
  * The engine's own fields on a probe's meta, beside the outcome a
  * generation's metadata reports: why a probe did not finish, its render
@@ -116,6 +139,11 @@ export interface RuntimeProbeMetaDetail {
   readonly hostLoad?: ProbeHostLoad;
   /** Time the check waited for a probe slot before it started, ms; present only when > 0 (ggui#1380 C2b). */
   readonly queuedMs?: number;
+  /**
+   * `ran` only: the pending-affordance walk (ggui#1398). Absent when no
+   * action walk ran, so "no data" and "0 of 0" read differently.
+   */
+  readonly pendingAffordance?: ProbePendingAffordance;
 }
 
 /**
@@ -143,9 +171,12 @@ export interface RuntimeProbeMetaDetail {
  * The verdict and the repair trigger are two different questions. The
  * verdict is `fail` on any failing check. The one repair turn a probe-only
  * round buys fires on a `render-no-throw` fail the engine recognises
- * (`isRecoverableRenderCrash`) and on nothing else: a `fail` verdict whose
- * checks are `prop-sensitivity`, `action-wiring` or any other kind is
- * recorded here and never fed back.
+ * (`isRecoverableRenderCrash`) and on nothing else: on the PROBE-ONLY round a
+ * `fail` verdict whose checks are `prop-sensitivity`, `action-wiring` or any
+ * other kind is recorded here and never fed back. The same meta is stamped on
+ * the EVALUATION lane, where a `prop-sensitivity` fail buys that lane's own
+ * contract-feedback round (ggui#1261, `FEEDBACK_PROBE_CHECKS`) — a different
+ * turn under a different rule, not the repair turn.
  */
 export type RuntimeProbeMeta = GenerationRuntimeProbeOutcome & RuntimeProbeMetaDetail;
 
@@ -331,8 +362,173 @@ export interface CanvasJudgeRecord {
   readonly sigma: number;
   /** The judge's per-canvas critique per sample — the words beside a swing. */
   readonly notes: string[];
+  /**
+   * ggui#1127 — the cause of each scoring sample whose answer failed to parse and was SALVAGED: its four dimensions
+   * were recovered from the closed prefix, its issues kept only as far as they closed, its critique lost. One entry
+   * per salvaged sample; absent when every sample parsed. The salvaged sample's score still counts.
+   */
+  readonly salvaged?: readonly JudgeSalvageCause[];
+  /**
+   * ggui#1127 — the criteria call's answer was salvaged. A salvage keeps no criteria, so that frame's judge rows read
+   * "not answered" with this cause rather than a clean `n/a`. Absent when the call parsed or was not asked.
+   */
+  readonly criteriaSalvaged?: JudgeSalvageCause;
+  /**
+   * ggui#1687 — the model asked for, and every model the provider said answered this canvas's calls (scoring and
+   * criteria), deduplicated. `served` is empty when no provider named one: unknown, never "the requested model".
+   */
+  readonly models?: { readonly requested: string; readonly served: readonly string[] };
+}
+
+/**
+ * ggui#1687 — what the criteria call's answer held, before resolution. A row the judge left unanswered reads `n/a`
+ * either way; this says whether the call came back empty, cut, or with entries the parser had to drop.
+ */
+export interface CriteriaAnswerCounts {
+  /** The answer carried a `criteria` array. `false`: absent or not an array, so no row was read from it. */
+  readonly array: boolean;
+  /** Entries in the array. */
+  readonly received: number;
+  /** Entries kept: an object with a string id and a verdict of pass, fail or n/a. */
+  readonly kept: number;
+  /** Entries dropped, by why. */
+  readonly dropped: { readonly notObject: number; readonly badId: number; readonly badVerdict: number };
+}
+
+/**
+ * ggui#1127 — why a judge answer that failed to parse had to be salvaged. `output-cap`: the provider reported it
+ * stopped at the output cap. `malformed`: the provider reported a normal stop, so the answer ended but did not
+ * parse. `stopped`: the provider stopped it for another reason it named (a safety stop, or one it reports as
+ * other), so it is cut, not malformed. `unknown`: the provider or route reported no stop reason. Never inferred
+ * from the answer's length.
+ */
+export type JudgeSalvageCause = "output-cap" | "malformed" | "stopped" | "unknown";
+
+/** ggui#1127 — salvaged judge answers across a canvas run, per leg, as N of M. */
+export interface JudgeSalvageCounts {
+  /** Scoring answers: `answers` = samples that parsed or were salvaged, `salvaged` = those salvaged. */
+  readonly scoring: { readonly salvaged: number; readonly answers: number };
+  /** Criteria calls: `answered` = calls that parsed or were salvaged, `salvaged` = those salvaged (criteria lost). */
+  readonly criteria: { readonly salvaged: number; readonly answered: number };
+}
+
+// ─── Visual criteria (ggui#1436) ──────────────────────────────────────────
+// The judge's typed criteria block per canvas — report-only beside the score
+// until a later word flips a gate to it. Every field below is optional on the
+// stamps so yesterday's `eval.json` and an old judge answer still parse (N−1).
+import type { FetchShape, LayoutShape, RenderShape, RiskTier, StateShape, WriteShape } from "../classifier/axes.js";
+
+export type CriteriaSeverity = "must" | "should";
+export type CriteriaVerdict = "pass" | "fail" | "n/a";
+/** The method that produces a verdict: an instrument's measurement, the vision judge off the frame, or a person's eye. */
+export type CriteriaMethod = "instrument" | "judge" | "human";
+/** The method's readiness, as the bank's author wrote it — the only source of an `n/a` that is not the judge's own. */
+export type CriteriaStatus = "live" | "reference" | "planned" | "manual";
+/**
+ * How a criterion entered the selection: bank-level (`static`, an empty scope) or by the selector's
+ * rules for this card (`context`). A model PROPOSING a criterion at judge time is RESERVED and not a
+ * member until a consumer exists — it would be `should`-only, recorded verbatim, never a gate.
+ */
+export type CriteriaSource = "static" | "context";
+export type CriteriaChroma = "achromatic" | "chromatic" | "unknown";
+/** The classifier's shape axes, as hashed into the set id. */
+export interface CriteriaAxes {
+  render: RenderShape;
+  state: StateShape;
+  writes: WriteShape;
+  fetch: FetchShape;
+  layout: LayoutShape;
+}
+/** The selector's inputs, stored with the verdict so a reader recomputes `criteriaSetId` from the record alone. */
+export interface CriteriaContext {
+  canvas: CanvasClass;
+  hasActions: boolean;
+  riskTier: RiskTier;
+  axes: CriteriaAxes;
+  chroma: CriteriaChroma;
+  profilePresent: boolean;
+  /** The harness's shell as named; `unknown` when the round carries none. */
+  shell: string;
+  /** The caller's item kind (a bootstrap mint's `hello`); hashed when present. */
+  kind?: string;
+  /** The styling preset in force (the profile's aesthetic id); a recipe is selected only under its preset. */
+  preset?: string;
+}
+/** What was asked and why — recorded BEFORE the judge runs. Omission is the selector's decision. */
+export interface CriteriaSelection {
+  id: string;
+  source: CriteriaSource;
+  reason: string;
+}
+/** One criterion's verdict; `n/a` is the judge's answer ("cannot read this off the frame"), an unanswered id, a tie, or a method whose `status` is not live. */
+export interface CriterionVerdict {
+  id: string;
+  severity: CriteriaSeverity;
+  method: CriteriaMethod;
+  status: CriteriaStatus;
+  source: CriteriaSource;
+  verdict: CriteriaVerdict;
+  evidence: string;
+  /**
+   * ggui#1663 — a measured reading beside a judge row, REPORT-ONLY: the row's `verdict` stays the judge's, so this
+   * changes no verdict, score or bar. Present only on rows an instrument answers beside the judge (`space.edge`).
+   */
+  instrument?: { verdict: CriteriaVerdict; evidence: string };
+}
+/** ggui#1687 — the criteria call as it came back: its answer's counts, ids it answered outside the bank, its stop. */
+export interface CriteriaCallRecord {
+  readonly answer: CriteriaAnswerCounts;
+  /** Kept entries whose id names no row of the bank. */
+  readonly unknownIds: number;
+  /** The provider's normalized stop reason (`length` = the output cap); absent when it reported none. */
+  readonly finishReason?: VisionFinishReason;
+}
+export interface CriteriaBlock {
+  /** sha256(bank version | selector version | canonical context), 16 hex — the block's own digest. */
+  criteriaSetId: string;
+  bankVersion: string;
+  selectorVersion: string;
+  context: CriteriaContext;
+  selection: CriteriaSelection[];
+  verdicts: CriterionVerdict[];
+  /** ggui#1687 — the call that answered this block, when one did. Absent for a block resolved with no call. */
+  call?: CriteriaCallRecord;
+}
+/** The roll-up across canvases; a `must` at `n/a` is listed, never counted as a pass. */
+export interface CriteriaRollup {
+  mustFailed: string[];
+  shouldFailed: string[];
+  mustNa: string[];
 }
 
+/** ggui#1492 — a host"s frame round the inline card, as data. Colours are `#rrggbb`; the ring"s `alpha` is 0..1. */
+export interface HostInlineFrame {
+  /** The frame"s surface — what shows behind a transparent card, and under the floor"s residual. */
+  readonly surface: string;
+  /** The room the frame sits in (the page ground round it). */
+  readonly ground: string;
+  readonly ring: { readonly widthPx: number; readonly color: string; readonly alpha: number };
+  readonly radiusPx: number;
+  /** The host"s minimum frame height: a card shorter than it sits in a frame this tall, and the visitor sees the rest as void. */
+  readonly minHeightPx?: number;
+}
+/** ggui#1492 — how a host presents one canvas: the label the judge is told, the page ground, and (the inline card only) its frame. */
+export interface CanvasHostPresentation {
+  readonly label: string;
+  /** The page ground under a fill canvas"s panel; the inline card"s ground lives on its {@link HostInlineFrame}. */
+  readonly ground?: string;
+  readonly frame?: HostInlineFrame;
+}
+/** ggui#1492 — why the judge did not draw a canvas's host presentation. The codes are the lane reader's own. */
+export type HostPresentationIgnoredReason = "label_missing" | "color_not_hex" | "frame_off_inline" | "frame_malformed" | "ground_on_inline";
+/**
+ * ggui#1492 — what the judge DID with a canvas's host presentation, read from the judge itself: `applied` carries
+ * the presentation it drew, `ignored` the reason it drew none. Absent when the caller supplied no presentation.
+ */
+export type CanvasPresentationOutcome =
+  | { readonly status: "applied"; readonly applied: CanvasHostPresentation }
+  | { readonly status: "ignored"; readonly reason: HostPresentationIgnoredReason };
+
 export interface CanvasVisualSummary {
   canvas: CanvasClass;
   viewport: { width: number; height: number };
@@ -344,8 +540,19 @@ export interface CanvasVisualSummary {
   contentHeight: number | null;
   /** `contentHeight > viewport.height` — measured on every canvas, judged per `canvasFitPolicy`. */
   overflow: boolean;
+  /**
+   * ggui#1120 — how far down the capture anything was painted: the last inked row over the measured
+   * region's height (inside a drawn host panel's chrome on md/lg/xl), read from the screenshot's
+   * pixels against their dominant colour. `0` is the blank the judge fails (`canvas-blank`); every
+   * other value is reported, never scored. `null` when the capture could not be read.
+   */
+  inkRatio: number | null;
   /** How `score` was reached — always present (ggui#1072). */
   judge: CanvasJudgeRecord;
+  /** ggui#1436 — the typed criteria block, when a bank was configured; report-only. */
+  criteria?: CriteriaBlock;
+  /** ggui#1492 — what the judge did with this canvas's host presentation, when one was supplied. */
+  presentation?: CanvasPresentationOutcome;
   /**
    * How the judge composed the mount (ggui#1100): `'fill'` on every
    * fullscreen canvas — the served runtime's fit, the root stretched to
@@ -382,6 +589,8 @@ export interface VisualEvalSummary {
   design?: { readonly src: string; readonly srcSha256: string };
   /** The mode the judge's tokens were composed in, when the caller said (ggui#1076). */
   themeMode?: 'light' | 'dark';
+  /** ggui#1436 — the criteria roll-up across canvases, when any canvas carries a block; report-only. */
+  criteria?: CriteriaRollup;
 }
 
 // ─── Quality mode ──────────────────────────────────────────────────────────
@@ -407,6 +616,7 @@ export const DEFAULT_QUALITY_CONFIG: QualityConfig = {
 // `./axis-checks/`.
 
 import type { DataContract } from "@ggui-ai/protocol";
+import type { JsonObject } from "@ggui-ai/protocol";
 import type { AxisVector, Classification } from "../classifier/axes.js";
 import type { AxisKey } from "../fragments/index.js";
 
@@ -433,6 +643,14 @@ export interface AxisCheckInput {
    * default (a silent `lg` would be the same guess with a nicer face).
    */
   canvas?: CanvasClass;
+  /**
+   * The props the probe and the judge RENDERED the source with (ggui#1285) — a bootstrap order's
+   * `sampleProps`, the bench commit's fixture. They are judging data, not the request: a literal in
+   * the source that equals one of these values, and that the request never stated, is baked copy a
+   * visitor sees whenever the agent leaves the prop out. Populated on the served loop and at
+   * auto-commit; ABSENT where the caller has none — the check stands down, it never guesses.
+   */
+  fixtureProps?: JsonObject;
 }
 
 /**

package/src/evaluation/types.ts

@@ -1,3 +1,4 @@
+import type { CriteriaAnswerCounts } from './types-public.js';
 // packages/ui-gen/src/evaluation/types.ts
 
 /**
@@ -28,11 +29,25 @@ export interface EvaluationIssue {
   severity: 'critical' | 'major' | 'minor';
   /** Suggested fix for the agent to apply */
   fix: string;
+  /**
+   * Who produced it, on the visual leg (ggui#1545): `judge` for the model's own findings, `instrument` for a
+   * deterministic measurement (fit, overflow, blank). Absent on issues from other evaluators. The loop's
+   * `visualEvaluation.actOn` acts on `judge` findings only, so a new instrument check never joins the treatment
+   * by accident: it has to say it is a judge finding, and absence reads as "not acted on".
+   */
+  origin?: 'judge' | 'instrument';
 }
 
 /**
  * Result of a single evaluation round.
  */
+/** One judge answer to a criterion (ggui#1436), as parsed — before the K-majority. */
+export interface CriteriaAnswer {
+  id: string;
+  verdict: 'pass' | 'fail' | 'n/a';
+  evidence: string;
+}
+
 export interface EvaluationResult {
   /** Whether the component passed the quality threshold */
   passed: boolean;
@@ -48,6 +63,10 @@ export interface EvaluationResult {
   inputTokens?: number;
   /** Output tokens consumed by the evaluation LLM call */
   outputTokens?: number;
+  /** ggui#1436 — the judge's criteria answers, when the prompt asked for them; never part of the score. */
+  criteriaAnswers?: CriteriaAnswer[];
+  /** ggui#1687 — what the answer's criteria array held and what the parser dropped; set whenever the answer parsed. */
+  criteriaCounts?: CriteriaAnswerCounts;
 }
 
 /**