@aglyn/plugins-ai 1.0.0-beta.143

package/src/lib/runtime/ai-eval.d.ts

@@ -0,0 +1,441 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { HostTheme } from '@aglyn/shared-data-types';
+import { type HostAction } from '@aglyn/aglyn/app-utils/actions';
+import type { HostThemeSource } from '@aglyn/aglyn/app-utils/marketplace-theme';
+import { type AiAutomationRecords } from '../model/ai-automation-draft';
+import type { ConsoleImportColumn } from '@aglyn/aglyn/plugin-manager/record-zone-props';
+import { type AiBuildPlanCreateKind } from '../model/ai-build-plan';
+import { type AiPlanCapabilities } from '../model/ai-plan-capabilities';
+import { type AiExperimentArm, type AiExperimentTarget, type AiExperimentTask } from '../model/ai-experiment';
+import type { AiSiteInventory } from '../model/ai-site-inventory';
+import type { AiAutomationCapabilities } from '../model/ai-workflow-job';
+import type { AiStepKind } from '../providers/catalog';
+import type { AiEffort, AiUsage } from '../providers/contract';
+import { type AiProductCategory, type AiProductFacts, type AiProductPhotoRead } from '../model/ai-products';
+import type { AiRunRecord } from '../model/ai-automation-outline';
+import type { AiCrmRecordKind, AiCrmTask } from '../model/ai-crm';
+import type { AiInsightTable } from '../model/ai-insight';
+import { type AiDeviceAudit } from './ai-device-audit';
+import { type AiValidatedTree } from './ai-doctrine';
+import { type AiAssetFacts, type AiCopyFraming } from './ai-doctrine-validators';
+import { type AiOutputKind } from './ai-palette';
+/**
+ * THE EVAL HARNESS (AGL-2937): the measure that gates every token lever.
+ *
+ * A token saved on a page the customer then has to fix costs more than the
+ * token. So no lever ships on its token figure alone: each golden brief is
+ * held to the same checks the doctrine holds a live answer to, and a kind's
+ * pass rate and mean score must stay at or above its floor.
+ *
+ * A case is a golden brief for one output kind — the site it is built for,
+ * the plan shape a good answer has, reference answers, and controls that
+ * must fail. Every answer is scored on four checks:
+ *
+ *  - **readable** — it reads as its kind: `validateAiNodeTree` admits the
+ *    tree, the plan parses, the fields and the copy are there;
+ *  - **rules** — no doctrine rule is broken (`ai-doctrine-validators.ts`),
+ *    nor any rule of the kind's own (a link to a page the answer was not
+ *    given, a title line on a blog body);
+ *  - **budget** — within the kind's measured budget: rule 17 for a document,
+ *    the length ceiling for copy;
+ *  - **rubric** — a grade against the brief for structure, copy fit and
+ *    reuse decisions, recorded from a stronger model on a live run.
+ *
+ * A document of a kind a site renders is also held to **responsive** (AGL-3020):
+ * rendered at every device of the besigner's switcher, nothing runs past the
+ * screen and no band keeps its desktop columns on a phone. It is measured in a
+ * browser by `tools/scripts/record-ai-page-axe.mts` and read back from that
+ * recording, so it gates a pass without moving a score: a page that only works
+ * on desktop does not pass, whatever its checks and grade average to.
+ *
+ * A planned kind's answer also carries the plan, held to the plan rules and
+ * to the case's expected shape.
+ *
+ * Offline is the default and the only mode CI runs: answers and grades are
+ * read from the fixtures under `tools/ai-eval/cases`, so a run costs nothing
+ * and asserts the same verdicts every time. An answer marked `authored` was
+ * written by hand as a reference and is graded by hand; one marked
+ * `recorded` came from a live run. A live run spends real money and is
+ * refused unless `AI_EVAL_LIVE=1` names it.
+ */
+/** The output kinds the harness holds golden briefs for. */
+export type AiEvalKind = 'page' | 'template' | 'component' | 'layout' | 'form' | 'email' | 'section' | 'seo' | 'product' | 'catalog' | 'categories' | 'theme' | 'element' | 'blog' | 'text' | 'chat' | 'workflow' | 'insight' | 'crm' | 'experiment';
+export declare const AI_EVAL_KINDS: readonly AiEvalKind[];
+/** The document kind a tree kind is held to; a section rewrite is one reusable block. */
+export declare const AI_EVAL_TREE_OUTPUT: Partial<Record<AiEvalKind, AiOutputKind>>;
+/**
+ * Whether a kind's answer is a document a site renders, and so is held to
+ * every device width (AGL-3020). An email is not: its column is fixed at 600px
+ * by the medium, which rule 12 exempts it for.
+ */
+export declare function aiEvalRendersAtDeviceWidths(kind: AiEvalKind): boolean;
+/**
+ * The recorded device audit of an answer, or `null` when none was recorded
+ * from it. The spec that scores the cases reads the recordings and answers
+ * this; a scorer given none holds nothing to its widths.
+ */
+export type AiEvalAudits = (evalCase: AiEvalCase, answer: unknown) => AiDeviceAudit | null;
+/** The checks an answer is scored on, beside its rubric. */
+export type AiEvalCheck = 'readable' | 'rules' | 'budget' | 'plan' | 'responsive' | 'rubric';
+/** A grade against the brief, each criterion from 1 (fails) to 5 (excellent). */
+export interface AiEvalRubric {
+    /** Organized the way the brief needs, nothing missing, nothing extra. */
+    structure: number;
+    /** The words fit the brief, the site and its audience, with no filler. */
+    copy: number;
+    /** Reuses what the site lists before creating; `null` for a kind that places nothing. */
+    reuse: number | null;
+    /** The model that graded it, or `authored` for a grade given by hand. */
+    grader: string;
+    notes?: string;
+}
+/** A plan's expected shape: what a good plan for the brief has. */
+export interface AiEvalPlanShape {
+    /** Screens planned, as `[min, max]`. */
+    screens: [number, number];
+    /** Inventory ids the plan must reuse. */
+    reuse?: string[];
+    /** The layout every planned screen renders inside. */
+    layout?: string;
+    /** The most creations a good plan needs. */
+    maxCreate?: number;
+    /** The kinds a creation may be. */
+    createKinds?: AiBuildPlanCreateKind[];
+}
+export interface AiEvalCandidate {
+    /** `authored` for a reference written by hand; `recorded` for one a live run captured. */
+    source: 'authored' | 'recorded';
+    /**
+     * `full` for an answer to the whole brief, the default; `plan` for the plan
+     * alone, which a live run records for a planned kind whose generator has
+     * not landed. A plan answer is held to the plan checks and its rubric, and
+     * counts toward the plan step's score rather than the kind's floor.
+     */
+    scope?: 'full' | 'plan';
+    /** The routing step and model a recorded answer came from; `null` for an authored one. */
+    step: AiStepKind | null;
+    model: string | null;
+    effort: AiEffort | null;
+    /** The plan the plan step answered with, for a planned kind. */
+    plan: unknown;
+    /**
+     * The output as the kind's door answers: a tree tool's input (or the node
+     * map itself), the copy as text, `{ title, description }` for search
+     * fields, or the theme tool's input.
+     */
+    answer: unknown;
+    /** What a recorded answer spent; `null` for an authored one. */
+    usage: AiUsage | null;
+    /**
+     * A recorded answer's spend, one entry per metered exchange in the order
+     * the job ran them — the plan, then every pass of its generation — with
+     * the credits each came to as the machine meters a step (AGL-3030).
+     * Absent on an authored answer, and on a recording that keeps its tokens
+     * alone.
+     */
+    steps?: AiEvalRecordedStep[];
+    /**
+     * What a recorded product copy answer was written with of its product's
+     * photo, as the products step reports it (AGL-3074): `read` when the
+     * picture rode the request.
+     */
+    photo?: AiProductPhotoRead;
+    /**
+     * What a recorded page was built as beside its tree (AGL-3073): its
+     * screen's address and listing, and the layout it renders inside. Absent
+     * on an authored answer, and on a recording made before it was kept.
+     */
+    screen?: AiEvalRecordedScreen;
+    /**
+     * The grade, or `null` where the grader itself failed and there is none
+     * (AGL-3143). An authored candidate always carries one.
+     *
+     * A brief's answer is bought before it is graded, so a grader that throws
+     * would otherwise take the answer down with it and the run would pay
+     * again for what it already had. The answer is kept and the grade is
+     * marked missing instead of being invented: a fabricated 1/1 would read
+     * as "the model answered badly" and pull the kind's floor down for
+     * something the model never did, and the floors are the whole reason the
+     * harness exists. An ungraded recording passes nothing — `rubric` is what
+     * a score is taken from — so a missing grade can only read low, never
+     * high, and the run that wrote it exits non-zero naming the throw.
+     */
+    rubric: AiEvalRubric | null;
+    note?: string;
+    /**
+     * What the provider wrote when the answer ran out of ceiling (AGL-3143):
+     * the bytes `answer` is only as much of as still parsed. Absent on every
+     * recording but one whose answer was cut off. It is here because the
+     * recording is the only place a live run's output outlives the run, and
+     * it is what says whether the ceiling went on a runaway string or on a
+     * decoding stall that wrote nothing usable. Never authored by hand.
+     */
+    rawOutput?: string;
+}
+/** One metered exchange of a recorded job. */
+export interface AiEvalRecordedStep {
+    step: AiStepKind;
+    model: string;
+    usage: AiUsage;
+    estCostUsd: number;
+    /** What the machine meters it as: the billed cost in credits, rounded up per exchange. */
+    credits: number;
+}
+/**
+ * A recorded page's screen, as the page step stored its draft. None of it is
+ * a node of the page's tree: the address and the listing are fields of the
+ * screen, the navigation entry is a proposal beside it, and the header, the
+ * footer and the main landmark belong to the layout it renders inside.
+ */
+export interface AiEvalRecordedScreen {
+    /** The draft's one-segment address, as the draft writer stored it. */
+    slug: string;
+    /** The search title and description the draft holds; `null` where it holds none. */
+    seoTitle: string | null;
+    seoDescription: string | null;
+    /** Whether a navigation entry for the page travels with it as a proposal. */
+    nav: boolean;
+    /** The layout the draft renders inside; `null` for a page that renders inside none. */
+    layout: AiEvalRecordedLayout | null;
+}
+/** The layout a recorded page renders inside. */
+export interface AiEvalRecordedLayout {
+    id: string;
+    name: string;
+    /** `true` for a layout the job built before the page; `false` for one the site already had. */
+    built: boolean;
+    /** Its tree as stored; `null` for a layout the case's site lists by name alone. */
+    tree: {
+        rootId: string;
+        nodes: Record<string, unknown>;
+    } | null;
+}
+/** An answer that must fail, and the checks it must fail. */
+export interface AiEvalControl {
+    why: string;
+    answer: unknown;
+    plan?: unknown;
+    fails: AiEvalCheck[];
+}
+export interface AiEvalCase {
+    id: string;
+    kind: AiEvalKind;
+    brief: string;
+    framing: AiCopyFraming;
+    /** The site the answer is built for; `null` for a brief that builds from none. */
+    inventory: AiSiteInventory | null;
+    /**
+     * The site's name as its visitors read it, where the case gives one
+     * (AGL-3077): a recorded page's listing names the site by it. A case that
+     * gives none is recorded on an untitled site.
+     */
+    siteName?: string;
+    /** Media library facts the image budget reads, by media id. */
+    assets?: Record<string, AiAssetFacts>;
+    /** The documentation URLs a chat answer was grounded in; it may link only these. */
+    docsUrls?: string[];
+    /** Copy ceilings the brief sets, beside the kind's own. */
+    maxChars?: number;
+    maxWords?: number;
+    /**
+     * For an insight brief (AGL-2915): the tables the readers returned, which
+     * every insight an answer writes is traced against.
+     */
+    tables?: AiInsightTable[];
+    /**
+     * For a product copy brief (AGL-2916): the product as its editor handed it
+     * over, and the site's categories the request lists.
+     */
+    product?: {
+        facts: AiProductFacts;
+        categories: AiProductCategory[];
+    };
+    /** For a categories brief: the names of the categories the store already has. */
+    existingCategoryNames?: string[];
+    /** For a theme brief: the site's current theme and where it comes from. */
+    siteTheme?: HostTheme;
+    themeSource?: HostThemeSource;
+    /**
+     * For an automation brief (AGL-2919): what the workspace can run, and the
+     * site's records the answer's words are looked up among. Absent, the
+     * workspace runs everything and the site holds nothing.
+     */
+    automationCapabilities?: AiAutomationCapabilities;
+    automationRecords?: AiAutomationRecords;
+    /**
+     * For an automation brief about a saved automation (AGL-3074): the action
+     * as the site stores it, and for a run's diagnosis, what the run history
+     * recorded of the failed run. Absent, the brief asks for a draft.
+     */
+    automation?: AiEvalAutomation;
+    /**
+     * The media library assets of the case's site a recording reads
+     * (AGL-3074), by media id: each the path of the file under
+     * `tools/ai-eval/fixtures` that holds its bytes. A product's photo is
+     * `media:<site id>/<media id>`, where the site is the inventory's, or
+     * `AI_EVAL_SITE_ID` for a case that describes none.
+     */
+    media?: Record<string, string>;
+    /**
+     * For a CRM brief (AGL-2917): what the job was asked, about which kind of
+     * record, the facts the CRM reported for it, and a mapping's columns.
+     */
+    crm?: AiEvalCrmContext;
+    /**
+     * For an A/B test brief (AGL-2914): which half of the step the brief asks
+     * for, what the test varies, the copy under test, and — for a result — the
+     * test's arms as the results reader published them.
+     */
+    experiment?: AiEvalExperiment;
+    /**
+     * What the workspace may create on the case's site (AGL-3030): a brief
+     * for a workspace that keeps no reusable components is held to the inline
+     * doctrine, and its plan to what the workspace may create. Absent, the
+     * doctrine applies whole.
+     */
+    capabilities?: AiPlanCapabilities;
+    expected?: {
+        plan?: AiEvalPlanShape;
+    };
+    candidates: AiEvalCandidate[];
+    controls: AiEvalControl[];
+}
+/** The site a case's job runs on when the case describes no inventory. */
+export declare const AI_EVAL_SITE_ID = "eval-site";
+/** What an automation brief's workspace can run when the case does not say: everything. */
+export declare const AI_EVAL_AUTOMATION_CAPABILITIES: AiAutomationCapabilities;
+/** A saved automation an automation brief asks about, as the workflow step reads one. */
+export interface AiEvalAutomation {
+    /** The action as `hosts/{hostId}/actions/{id}` holds it. */
+    action: HostAction;
+    /** A failed run of it; given, the brief asks why that run failed. */
+    run?: AiRunRecord;
+}
+/** An A/B test brief, as the experiment step is given one. */
+export interface AiEvalExperiment {
+    task: AiExperimentTask;
+    /** What the test varies; a variants brief always names one. */
+    target?: AiExperimentTarget;
+    /** The copy under test, as the surface handed it over. */
+    subject?: string;
+    /** A result brief's test, and its arms as the results reader published them. */
+    test?: string;
+    arms?: AiExperimentArm[];
+}
+/** What a CRM brief's answer is checked against, as the `crm` step checks it. */
+export interface AiEvalCrmContext {
+    task: AiCrmTask;
+    /** The record a `record` or `email` brief is about. */
+    record?: AiCrmRecordKind;
+    /** The facts the CRM's reader reported: a record's, or an import's field catalog. */
+    facts?: Record<string, unknown>;
+    /** A mapping brief's columns: each header and its shape. */
+    columns?: ConsoleImportColumn[];
+}
+/** A kind's floor: the pass rate and the mean score its candidates may not fall below. */
+export interface AiEvalFloor {
+    passRate: number;
+    meanScore: number;
+}
+/**
+ * The floors, per kind. Every reference answer must pass, because each was
+ * written to; the mean floor leaves room for a rubric grade of four on an
+ * answer that passes every mechanical check. Recorded answers are held to
+ * the same floors, and a lever that lowers a kind below its floor does not
+ * ship.
+ */
+export declare const AI_EVAL_FLOORS: Readonly<Record<AiEvalKind, AiEvalFloor>>;
+/** A rubric passes at this mean, with no criterion below three. */
+export declare const AI_EVAL_RUBRIC_PASS_MEAN = 3.5;
+/** How long a blog body may run: the blog mode's output ceiling, at four characters a token. */
+export declare const AI_EVAL_BLOG_MAX_CHARS = 8000;
+/** How long a job's copy may run when the brief implies no length: the text step's own rule. */
+export declare const AI_EVAL_TEXT_MAX_WORDS = 150;
+/** How long a chat answer may run: the chat door's output ceiling, at four characters a token. */
+export declare const AI_EVAL_CHAT_MAX_CHARS = 4000;
+export interface AiEvalScore {
+    caseId: string;
+    kind: AiEvalKind;
+    source: AiEvalCandidate['source'];
+    scope: NonNullable<AiEvalCandidate['scope']>;
+    model: string | null;
+    /** A check that does not apply to the answer — the plan of an unplanned kind, the document of a plan answer — is `null`. */
+    checks: Record<'rubric', boolean> & Record<Exclude<AiEvalCheck, 'rubric'>, boolean | null>;
+    /** The rubric's mean, from 0 (every criterion 1) to 1 (every criterion 5). */
+    rubricScore: number;
+    /** The checks and the rubric, averaged: from 0 to 1. */
+    score: number;
+    pass: boolean;
+    /** Why a check failed: violation codes and the kind's own findings. */
+    findings: string[];
+}
+/**
+ * The tree an answer of a tree kind is checked as — a page's repeated items
+ * drawn into their copies — or `null` when it cannot be read as one. The
+ * device audit renders exactly this tree (AGL-3020).
+ */
+export declare function aiEvalAnswerTree(evalCase: AiEvalCase, answer: unknown): AiValidatedTree | null;
+/** The plan held to the plan rules, then to the shape the case expects. */
+export declare function checkAiEvalPlan(evalCase: AiEvalCase, plan: unknown): {
+    pass: boolean;
+    findings: string[];
+};
+/** A rubric's mean and whether it passes: the mean at `AI_EVAL_RUBRIC_PASS_MEAN`, nothing under three. */
+export declare function aiEvalRubricVerdict(rubric: AiEvalRubric | null | undefined): {
+    mean: number;
+    pass: boolean;
+};
+/** One answer to one golden brief, scored. */
+export declare function scoreAiEvalCandidate(evalCase: AiEvalCase, candidate: AiEvalCandidate, audits?: AiEvalAudits): AiEvalScore;
+/**
+ * The checks a control failed; the harness requires every one it names. A
+ * control whose widths were never recorded fails none of them.
+ */
+export declare function scoreAiEvalControl(evalCase: AiEvalCase, control: AiEvalControl, audits?: AiEvalAudits): AiEvalCheck[];
+export interface AiEvalKindSummary {
+    kind: AiEvalKind;
+    candidates: number;
+    passed: number;
+    /** Passed over candidates; `null` with none, which is below every floor. */
+    passRate: number | null;
+    meanScore: number | null;
+    floor: AiEvalFloor;
+    belowFloor: boolean;
+}
+/** Every kind's pass rate and mean score against its floor; a kind with no candidate is below it. */
+export declare function summarizeAiEval(scores: readonly AiEvalScore[]): AiEvalKindSummary[];
+/** How the plans a planned kind's answers carry score, across every brief. */
+export interface AiEvalPlanSummary {
+    plans: number;
+    passed: number;
+    passRate: number | null;
+}
+/**
+ * The plan step's own score: every answer that carried a plan checked
+ * against its brief's expected shape, whole answers and plan answers alike.
+ */
+export declare function summarizeAiEvalPlans(scores: readonly AiEvalScore[]): AiEvalPlanSummary;
+/** A recording a live run wrote: the brief it answers, and the answer. */
+export interface AiEvalRecording {
+    caseId: string;
+    candidate: AiEvalCandidate;
+}
+/** A recording file read and checked for shape; throws naming the field at fault. */
+export declare function readAiEvalRecording(raw: unknown, file: string): AiEvalRecording;
+/** A case file read and checked for shape; throws naming the field at fault. */
+export declare function readAiEvalCase(raw: unknown, file: string): AiEvalCase;