@aglyn/plugins-ai 1.0.0-beta.143

package/src/lib/runtime/ai-eval.js

@@ -0,0 +1,1151 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ import { _ as _extends } from "@swc/helpers/_/_extends";
+import { validateHostAction } from "@aglyn/aglyn/app-utils/actions";
+import { aiAutomationDraft, emptyAiAutomationRecords } from "../model/ai-automation-draft.js";
+import { AI_BUILD_PLAN_CREATE_KINDS } from "../model/ai-build-plan.js";
+import { aiUnrestrictedPlanCapabilities } from "../model/ai-plan-capabilities.js";
+import { AI_EXPERIMENT_BODY_MAX_CHARS, AI_EXPERIMENT_HEADLINE_MAX_CHARS, AI_EXPERIMENT_LINE_MAX_CHARS, AI_EXPERIMENT_MAX_POINTS, AI_EXPERIMENT_NAME_MAX_CHARS, AI_EXPERIMENT_NEXT_MAX_CHARS, AI_EXPERIMENT_POINT_MAX_CHARS, AI_EXPERIMENT_RATIONALE_MAX_CHARS, aiExperimentVariantFields, checkAiExperimentExplanation, checkAiExperimentVariants, readAiExperimentResult } from "../model/ai-experiment.js";
+import { parseAiExperimentExplanation, parseAiExperimentVariants } from "../tools/ai-experiment-tool.js";
+import { aiComponentCheck } from "../jobs/ai-job-component-checks.js";
+import { aiProductMerchantWords } from "../model/ai-products.js";
+import { checkAiCatalog, checkAiCategories, checkAiProductCopy } from "../tools/ai-products-tool.js";
+import { aiCrmEmailMergeFields, aiCrmImportFields, aiCrmRecordCheckContext, checkAiCrmEmail, checkAiCrmMapping, checkAiCrmRecord } from "../tools/ai-crm-tool.js";
+import { parseAiThemeToolInput } from "../tools/ai-theme-tool.js";
+import { AI_AUTOMATION_OVERSIZE_CODES, AI_AUTOMATION_UNREADABLE_CODES, readAiAutomationAnswer, readAiWorkflowExplanation } from "../tools/ai-workflow-tool.js";
+import { parseAiInsightAnswer } from "../tools/ai-insight-tool.js";
+import { checkAiInsightAnswer } from "./ai-insight-check.js";
+import { aiDeviceAuditFindings } from "./ai-device-audit.js";
+import { aiAnswerTree, aiDoctrinePlanCheck, aiDoctrineTreeCheck, aiDoctrineTreeContext } from "./ai-doctrine.js";
+import { AI_SEO_DESCRIPTION_MAX, AI_SEO_TITLE_MAX, detectOffVoiceCopy, detectPublishIntent } from "./ai-doctrine-validators.js";
+import { AI_TEXT_LIMITS } from "./ai-palette.js";
+import { expandAiRepeatedItems } from "./ai-repeated-items.js";
+export const AI_EVAL_KINDS = [
+    'page',
+    'template',
+    'component',
+    'layout',
+    'form',
+    'email',
+    'section',
+    'seo',
+    'product',
+    'catalog',
+    'categories',
+    'theme',
+    'element',
+    'blog',
+    'text',
+    'chat',
+    'workflow',
+    'insight',
+    'crm',
+    'experiment'
+];
+/** The document kind a tree kind is held to; a section rewrite is one reusable block. */ export const AI_EVAL_TREE_OUTPUT = {
+    page: 'page',
+    template: 'template',
+    component: 'component',
+    layout: 'layout',
+    form: 'form',
+    email: 'email',
+    section: 'component'
+};
+/**
+ * Whether a kind's answer is a document a site renders, and so is held to
+ * every device width (AGL-3020). An email is not: its column is fixed at 600px
+ * by the medium, which rule 12 exempts it for.
+ */ export function aiEvalRendersAtDeviceWidths(kind) {
+    const output = AI_EVAL_TREE_OUTPUT[kind];
+    return output !== undefined && output !== 'email';
+}
+/** The site a case's job runs on when the case describes no inventory. */ export const AI_EVAL_SITE_ID = 'eval-site';
+/** What an automation brief's workspace can run when the case does not say: everything. */ export const AI_EVAL_AUTOMATION_CAPABILITIES = {
+    crm: true,
+    webhooks: true,
+    bookings: true
+};
+/** A fixture file a case's media names: a relative path to a picture, never out of its folder. */ const AI_EVAL_MEDIA_FILE = /^(?!.*(?:^|\/)\.\.?(?:\/|$))[A-Za-z0-9][A-Za-z0-9._/-]*\.(?:jpe?g|png|webp|gif|avif)$/;
+/**
+ * The floors, per kind. Every reference answer must pass, because each was
+ * written to; the mean floor leaves room for a rubric grade of four on an
+ * answer that passes every mechanical check. Recorded answers are held to
+ * the same floors, and a lever that lowers a kind below its floor does not
+ * ship.
+ */ export const AI_EVAL_FLOORS = {
+    page: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    template: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    component: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    layout: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    form: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    email: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    section: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    seo: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    product: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    catalog: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    categories: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    theme: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    element: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    blog: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    text: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    chat: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    workflow: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    insight: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    crm: {
+        passRate: 1,
+        meanScore: 0.9
+    },
+    experiment: {
+        passRate: 1,
+        meanScore: 0.9
+    }
+};
+/** A rubric passes at this mean, with no criterion below three. */ export const AI_EVAL_RUBRIC_PASS_MEAN = 3.5;
+/** How long a blog body may run: the blog mode's output ceiling, at four characters a token. */ export const AI_EVAL_BLOG_MAX_CHARS = 8000;
+/** How long a job's copy may run when the brief implies no length: the text step's own rule. */ export const AI_EVAL_TEXT_MAX_WORDS = 150;
+/** How long a chat answer may run: the chat door's output ceiling, at four characters a token. */ export const AI_EVAL_CHAT_MAX_CHARS = 4000;
+const isRecord = (value)=>typeof value === 'object' && value !== null && !Array.isArray(value);
+const codes = (violations)=>violations.map((violation)=>violation.code);
+/**
+ * The sections a page answer's plan lays out, by name, which a link on the
+ * page may take a visitor to as the page step lets it (AGL-3097); empty for an
+ * answer of another kind, or with no plan to read them from.
+ */ function planSections(evalCase, plan) {
+    const screens = evalCase.kind === 'page' && isRecord(plan) && Array.isArray(plan['screens']) ? plan['screens'] : [];
+    const sections = isRecord(screens[0]) && Array.isArray(screens[0]['sections']) ? screens[0]['sections'] : [];
+    return sections.flatMap((section)=>isRecord(section) && typeof section['name'] === 'string' ? [
+            section['name']
+        ] : []);
+}
+function readTree(evalCase, outputKind, answer, plan) {
+    var _evalCase_capabilities;
+    const inline = ((_evalCase_capabilities = evalCase.capabilities) == null ? void 0 : _evalCase_capabilities.reusableComponents) === false;
+    let input = isRecord(answer) ? answer : {
+        tree: answer
+    };
+    // A component answer that declares its properties binds them in its tree,
+    // and is held to the component step's own checks as that step holds it
+    // (AGL-3054).
+    const declaresProps = evalCase.kind === 'component' && Array.isArray(input['props']);
+    const sections = planSections(evalCase, plan);
+    const check = aiDoctrineTreeCheck(outputKind, aiDoctrineTreeContext(evalCase.inventory, _extends({}, evalCase.assets ? {
+        assets: evalCase.assets
+    } : {}, {
+        framing: evalCase.framing
+    }, inline ? {
+        reusableComponents: false
+    } : {}, declaresProps ? {
+        definesComponent: true
+    } : {}, sections.length ? {
+        pageSections: sections
+    } : {})));
+    // A page's repeated item written once is drawn into its copies first, as the
+    // page step draws it, and refused as the page step refuses it (AGL-3053).
+    if (evalCase.kind === 'page') {
+        const drawn = expandAiRepeatedItems(aiAnswerTree(input), {
+            inline,
+            noun: 'page'
+        });
+        if (drawn.ok === false) return {
+            input,
+            declaresProps,
+            undrawn: drawn.violations,
+            result: null
+        };
+        if (drawn.items) input = {
+            tree: drawn.tree
+        };
+    }
+    return {
+        input,
+        declaresProps,
+        undrawn: null,
+        result: check(input)
+    };
+}
+/**
+ * The tree an answer of a tree kind is checked as — a page's repeated items
+ * drawn into their copies — or `null` when it cannot be read as one. The
+ * device audit renders exactly this tree (AGL-3020).
+ */ export function aiEvalAnswerTree(evalCase, answer) {
+    var _ref;
+    var _readTree_result;
+    const outputKind = AI_EVAL_TREE_OUTPUT[evalCase.kind];
+    return outputKind ? (_ref = (_readTree_result = readTree(evalCase, outputKind, answer).result) == null ? void 0 : _readTree_result.value) != null ? _ref : null : null;
+}
+function checkTree(evalCase, outputKind, answer, plan) {
+    const { input, declaresProps, undrawn, result } = readTree(evalCase, outputKind, answer, plan);
+    if (undrawn || !result) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: codes(undrawn != null ? undrawn : [])
+        };
+    }
+    const violations = [
+        ...result.violations,
+        ...declaresProps && result.value ? aiComponentCheck({
+            inventory: evalCase.inventory,
+            plan: null,
+            onProps: ()=>undefined
+        })(result.value, input) : []
+    ];
+    const budget = violations.filter((violation)=>violation.rule === 17);
+    const rules = violations.filter((violation)=>violation.rule !== 17);
+    return {
+        readable: result.value !== null,
+        rules: result.value !== null && rules.length === 0,
+        budget: result.value !== null && budget.length === 0,
+        findings: codes(violations)
+    };
+}
+const FENCE = /```/;
+const PREAMBLE = /^\s*(?:sure|certainly|of course|here(?:'s| is| are))\b/i;
+const QUOTED = RegExp("^\\s*[\"“'].*[\"”']\\s*$", "s");
+const MARKUP = /<\/?[a-z][^>]*>/i;
+function words(text) {
+    return text.trim().split(/\s+/).filter(Boolean).length;
+}
+/** Copy answered as text: there, unwrapped, in the site's voice, within its ceiling. */ function checkCopy(evalCase, answer, limits, own = ()=>[]) {
+    var _evalCase_maxChars, _evalCase_maxWords;
+    if (typeof answer !== 'string' || !answer.trim()) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'copy-missing'
+            ]
+        };
+    }
+    const text = answer.trim();
+    const unreadable = [
+        ...FENCE.test(text) ? [
+            'copy-fenced'
+        ] : [],
+        ...PREAMBLE.test(text) ? [
+            'copy-preamble'
+        ] : [],
+        ...QUOTED.test(text) ? [
+            'copy-quoted'
+        ] : []
+    ];
+    const broken = [
+        ...codes(detectOffVoiceCopy([
+            {
+                at: 'answer',
+                text
+            }
+        ], evalCase.framing)),
+        ...MARKUP.test(text) ? [
+            'copy-markup'
+        ] : [],
+        ...own(text)
+    ];
+    const maxChars = (_evalCase_maxChars = evalCase.maxChars) != null ? _evalCase_maxChars : limits.maxChars;
+    const maxWords = (_evalCase_maxWords = evalCase.maxWords) != null ? _evalCase_maxWords : limits.maxWords;
+    const over = [
+        ...maxChars !== undefined && text.length > maxChars ? [
+            'copy-over-chars'
+        ] : [],
+        ...maxWords !== undefined && words(text) > maxWords ? [
+            'copy-over-words'
+        ] : []
+    ];
+    return {
+        readable: unreadable.length === 0,
+        rules: broken.length === 0,
+        budget: over.length === 0,
+        findings: [
+            ...unreadable,
+            ...broken,
+            ...over
+        ]
+    };
+}
+const MARKDOWN_LINK = /\[[^\]]*\]\(([^)\s]+)\)/g;
+/** A chat answer links only the documentation it was given and console paths. */ function chatLinks(evalCase) {
+    var _evalCase_docsUrls;
+    const allowed = new Set((_evalCase_docsUrls = evalCase.docsUrls) != null ? _evalCase_docsUrls : []);
+    return (text)=>[
+            ...text.matchAll(MARKDOWN_LINK)
+        ].map((match)=>match[1]).filter((url)=>!url.startsWith('/') && !allowed.has(url)).map((url)=>`chat-link-not-given:${url}`);
+}
+function checkBlog(evalCase, answer) {
+    const checked = checkCopy(evalCase, answer, {
+        maxChars: AI_EVAL_BLOG_MAX_CHARS
+    });
+    if (typeof answer !== 'string') return checked;
+    const head = answer.trimStart();
+    const extra = [
+        ...head.startsWith('---') ? [
+            'blog-front-matter'
+        ] : [],
+        .../^#\s/.test(head) ? [
+            'blog-title-line'
+        ] : []
+    ];
+    return extra.length ? _extends({}, checked, {
+        readable: false,
+        findings: [
+            ...checked.findings,
+            ...extra
+        ]
+    }) : checked;
+}
+function checkSeo(evalCase, answer) {
+    const title = isRecord(answer) && typeof answer['title'] === 'string' ? answer['title'].trim() : '';
+    const description = isRecord(answer) && typeof answer['description'] === 'string' ? answer['description'].trim() : '';
+    if (!title || !description) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'seo-field-missing'
+            ]
+        };
+    }
+    const broken = [
+        ...codes(detectPublishIntent(answer)),
+        ...codes(detectOffVoiceCopy([
+            {
+                at: 'title',
+                text: title
+            },
+            {
+                at: 'description',
+                text: description
+            }
+        ], evalCase.framing)),
+        ...MARKUP.test(title) || MARKUP.test(description) ? [
+            'seo-markup'
+        ] : []
+    ];
+    const over = [
+        ...title.length > AI_SEO_TITLE_MAX ? [
+            'seo-title-over'
+        ] : [],
+        ...description.length > AI_SEO_DESCRIPTION_MAX ? [
+            'seo-description-over'
+        ] : []
+    ];
+    return {
+        readable: true,
+        rules: broken.length === 0,
+        budget: over.length === 0,
+        findings: [
+            ...broken,
+            ...over
+        ]
+    };
+}
+/** What a products check refuses an answer for, by the check it fails. */ const PRODUCTS_UNREADABLE = new Set([
+    'type',
+    'missing',
+    'option-count'
+]);
+const PRODUCTS_OVER_BUDGET = new Set([
+    'too-long',
+    'too-many',
+    'product-count'
+]);
+/**
+ * A products answer (AGL-2916), held by the check its generation is held by:
+ * a product's copy, a catalog, or categories with discounts. A finding about
+ * the answer's shape makes it unreadable, one about a length or a count is
+ * over its budget, and every other one — a claim, a price, markup, a category
+ * the request did not list — breaks a rule.
+ */ function checkProducts(evalCase, answer) {
+    var _ref, _ref1, _evalCase_existingCategoryNames;
+    var _evalCase_product, _evalCase_product1;
+    if (!isRecord(answer)) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'products-not-a-call'
+            ]
+        };
+    }
+    const result = evalCase.kind === 'product' ? checkAiProductCopy(answer, {
+        categoryIds: ((_ref = (_evalCase_product = evalCase.product) == null ? void 0 : _evalCase_product.categories) != null ? _ref : []).map((category)=>category.id),
+        optionCount: (_ref1 = (_evalCase_product1 = evalCase.product) == null ? void 0 : _evalCase_product1.facts.options.length) != null ? _ref1 : 0,
+        merchantWords: evalCase.product ? aiProductMerchantWords(evalCase.product.facts) : evalCase.brief
+    }) : evalCase.kind === 'catalog' ? checkAiCatalog(answer, {
+        merchantWords: evalCase.brief
+    }) : checkAiCategories(answer, {
+        existingCategoryNames: (_evalCase_existingCategoryNames = evalCase.existingCategoryNames) != null ? _evalCase_existingCategoryNames : [],
+        merchantWords: evalCase.brief
+    });
+    const found = [
+        ...codes(result.violations),
+        ...codes(detectPublishIntent(answer))
+    ];
+    return {
+        readable: !found.some((code)=>PRODUCTS_UNREADABLE.has(code)),
+        rules: !found.some((code)=>!PRODUCTS_UNREADABLE.has(code) && !PRODUCTS_OVER_BUDGET.has(code)),
+        budget: !found.some((code)=>PRODUCTS_OVER_BUDGET.has(code)),
+        findings: found
+    };
+}
+function checkTheme(evalCase, answer) {
+    if (!isRecord(answer)) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'theme-not-a-call'
+            ]
+        };
+    }
+    const parse = parseAiThemeToolInput(answer);
+    const changes = parse.changes.length + parse.components.length + (parse.resetComponents ? 1 : 0);
+    const broken = [
+        ...codes(detectPublishIntent(answer)),
+        ...codes(detectOffVoiceCopy([
+            {
+                at: 'summary',
+                text: parse.summary
+            }
+        ], evalCase.framing))
+    ];
+    return {
+        readable: changes > 0 && parse.dropped.length === 0,
+        rules: broken.length === 0,
+        budget: true,
+        findings: [
+            ...parse.dropped.map((reason)=>`theme-dropped:${reason}`),
+            ...broken
+        ]
+    };
+}
+/**
+ * An automation (AGL-2919), or an explanation of one: read through the tool's
+ * own reader, and an automation also made into the document the workflows
+ * plugin stores and held to the Actions editor's validator.
+ */ function checkWorkflow(evalCase, answer) {
+    var _evalCase_automationCapabilities, _evalCase_automationRecords;
+    if (!isRecord(answer)) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'workflow-not-a-call'
+            ]
+        };
+    }
+    const publish = codes(detectPublishIntent(answer));
+    if ('summary' in answer) {
+        var _answer_summary;
+        const read = readAiWorkflowExplanation(answer);
+        const lines = [
+            String((_answer_summary = answer['summary']) != null ? _answer_summary : ''),
+            ...Array.isArray(answer['points']) ? answer['points'] : [],
+            ...Array.isArray(answer['suggestions']) ? answer['suggestions'] : []
+        ].map(String);
+        const offVoice = codes(detectOffVoiceCopy(lines.map((text, index)=>({
+                at: `line[${index}]`,
+                text
+            })), evalCase.framing));
+        const found = codes(read.violations);
+        return {
+            readable: !found.includes('explanation-shape'),
+            rules: read.value !== null && publish.length === 0 && offVoice.length === 0,
+            budget: true,
+            findings: [
+                ...found,
+                ...publish,
+                ...offVoice
+            ]
+        };
+    }
+    const capabilities = (_evalCase_automationCapabilities = evalCase.automationCapabilities) != null ? _evalCase_automationCapabilities : AI_EVAL_AUTOMATION_CAPABILITIES;
+    const read = readAiAutomationAnswer(answer, capabilities);
+    const found = codes(read.violations);
+    const unreadable = found.filter((code)=>AI_AUTOMATION_UNREADABLE_CODES.includes(code));
+    const oversize = found.filter((code)=>AI_AUTOMATION_OVERSIZE_CODES.includes(code));
+    const broken = found.filter((code)=>!unreadable.includes(code) && !oversize.includes(code));
+    const invalid = read.value && !read.value.unsupported ? validateHostAction(aiAutomationDraft(read.value, (_evalCase_automationRecords = evalCase.automationRecords) != null ? _evalCase_automationRecords : emptyAiAutomationRecords()).action) : null;
+    return {
+        readable: unreadable.length === 0,
+        rules: broken.length === 0 && publish.length === 0 && invalid === null,
+        budget: oversize.length === 0,
+        findings: [
+            ...found,
+            ...publish,
+            ...invalid ? [
+                `workflow-invalid:${invalid}`
+            ] : []
+        ]
+    };
+}
+/**
+ * An insight answer held to the trace (AGL-2915): readable when it is a
+ * `submit_insights` call that keeps an insight, within the rules when no
+ * insight is left out for its numbers, its citations or its words, and within
+ * the budget when no insight runs long and there are no more than five.
+ */ function checkInsight(evalCase, answer) {
+    var _evalCase_tables;
+    const parsed = parseAiInsightAnswer(answer);
+    if (!parsed || !parsed.insights.length) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'insight-no-answer'
+            ]
+        };
+    }
+    const check = checkAiInsightAnswer(parsed, (_evalCase_tables = evalCase.tables) != null ? _evalCase_tables : []);
+    const findings = check.findings.map((finding)=>finding.split(':')[0]);
+    const over = findings.filter((code)=>code === 'insight-too-long' || code === 'insight-too-many');
+    return {
+        readable: check.kept.length > 0,
+        rules: findings.length === over.length,
+        budget: over.length === 0,
+        findings: check.findings
+    };
+}
+/** The CRM findings that mean the answer could not be read as one, and the ones over a length. */ const CRM_UNREADABLE = new Set([
+    'type',
+    'missing'
+]);
+const CRM_OVER = new Set([
+    'too-long'
+]);
+/**
+ * A CRM answer, held by the checks the `crm` step holds it to (AGL-2917),
+ * with rule 13 and the site's voice on everything a person reads.
+ */ function checkCrm(evalCase, answer) {
+    var _context_facts, _context_record, _context_columns;
+    const context = evalCase.crm;
+    if (!context || !isRecord(answer)) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'crm-not-a-call'
+            ]
+        };
+    }
+    const facts = (_context_facts = context.facts) != null ? _context_facts : {};
+    const kind = (_context_record = context.record) != null ? _context_record : 'contact';
+    const result = context.task === 'mapping' ? checkAiCrmMapping(answer, {
+        columns: (_context_columns = context.columns) != null ? _context_columns : [],
+        fields: aiCrmImportFields(facts)
+    }) : context.task === 'email' ? checkAiCrmEmail(answer, {
+        mergeFields: aiCrmEmailMergeFields(kind)
+    }) : checkAiCrmRecord(answer, aiCrmRecordCheckContext(kind, facts));
+    const own = codes(result.violations);
+    const read = (path)=>{
+        let value = answer;
+        for (const key of path)value = isRecord(value) ? value[key] : undefined;
+        return typeof value === 'string' ? value : '';
+    };
+    const samples = [
+        [
+            'summary'
+        ],
+        [
+            'standing'
+        ],
+        [
+            'nextStep',
+            'title'
+        ],
+        [
+            'nextStep',
+            'reason'
+        ],
+        [
+            'stage',
+            'reason'
+        ],
+        [
+            'subject'
+        ],
+        [
+            'body'
+        ]
+    ].map((path)=>({
+            at: path.join('.'),
+            text: read(path)
+        })).filter((sample)=>sample.text);
+    const broken = [
+        ...codes(detectPublishIntent(answer)),
+        ...codes(detectOffVoiceCopy(samples, evalCase.framing)),
+        ...own.filter((code)=>!CRM_UNREADABLE.has(code) && !CRM_OVER.has(code))
+    ];
+    return {
+        readable: !own.some((code)=>CRM_UNREADABLE.has(code)),
+        rules: broken.length === 0,
+        budget: !own.some((code)=>CRM_OVER.has(code)),
+        findings: [
+            ...new Set([
+                ...own,
+                ...broken
+            ])
+        ]
+    };
+}
+/** What an experiment answer is refused for, by the check it fails. */ const EXPERIMENT_RULE_FINDINGS = new Set([
+    'variant-carries-markup',
+    'variant-fills-another-targets-fields',
+    'variant-repeats-another',
+    'variant-has-no-copy'
+]);
+/**
+ * An A/B test answer (AGL-2914), held by the checks its step holds it by.
+ *
+ * A variants answer is readable when at least two variants survive, breaks a
+ * rule when one carried markup, repeated another or answered a different
+ * target, and is over budget when a field ran past the length the card
+ * stores. A result answer is readable when an explanation survives its
+ * reading, and breaks a rule when the reading had to take a winner out of it
+ * — which is the whole point of the kind: an explanation that claims more
+ * than the figures support does not pass here.
+ */ function checkExperiment(evalCase, answer) {
+    var _brief_test, _brief_arms;
+    const brief = evalCase.experiment;
+    if (!brief) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'experiment-brief-missing'
+            ]
+        };
+    }
+    if (brief.task === 'variants') {
+        var _brief_target;
+        const target = (_brief_target = brief.target) != null ? _brief_target : 'screen';
+        const parsed = parseAiExperimentVariants(answer);
+        const checked = checkAiExperimentVariants(parsed, target);
+        if (!parsed || !checked) {
+            return {
+                readable: false,
+                rules: false,
+                budget: false,
+                findings: [
+                    'experiment-no-variants'
+                ]
+            };
+        }
+        const fields = aiExperimentVariantFields(target);
+        const over = parsed.variants.flatMap((variant)=>[
+                ...variant.name.length > AI_EXPERIMENT_NAME_MAX_CHARS ? [
+                    'experiment-name-over'
+                ] : [],
+                ...variant.rationale.length > AI_EXPERIMENT_RATIONALE_MAX_CHARS ? [
+                    'experiment-rationale-over'
+                ] : [],
+                ...fields.flatMap((field)=>{
+                    var _variant_field;
+                    return String((_variant_field = variant[field]) != null ? _variant_field : '').length > (field === 'body' ? AI_EXPERIMENT_BODY_MAX_CHARS : AI_EXPERIMENT_LINE_MAX_CHARS) ? [
+                        `experiment-${field}-over`
+                    ] : [];
+                })
+            ]);
+        const samples = checked.proposal.variants.flatMap((variant)=>fields.map((field)=>{
+                var _variant_field;
+                return {
+                    at: `${variant.name}.${field}`,
+                    text: String((_variant_field = variant[field]) != null ? _variant_field : '')
+                };
+            }));
+        const broken = [
+            ...checked.findings.filter((finding)=>EXPERIMENT_RULE_FINDINGS.has(finding)),
+            ...codes(detectPublishIntent(answer)),
+            ...codes(detectOffVoiceCopy(samples.filter((sample)=>sample.text), evalCase.framing))
+        ];
+        return {
+            readable: true,
+            rules: broken.length === 0,
+            budget: over.length === 0,
+            findings: [
+                ...new Set([
+                    ...broken,
+                    ...over
+                ])
+            ]
+        };
+    }
+    const reading = readAiExperimentResult((_brief_test = brief.test) != null ? _brief_test : '', (_brief_arms = brief.arms) != null ? _brief_arms : []);
+    const parsed = parseAiExperimentExplanation(answer);
+    const explained = checkAiExperimentExplanation(parsed, reading);
+    if (!parsed || !explained) {
+        return {
+            readable: false,
+            rules: false,
+            budget: false,
+            findings: [
+                'experiment-no-explanation'
+            ]
+        };
+    }
+    const over = [
+        ...parsed.headline.length > AI_EXPERIMENT_HEADLINE_MAX_CHARS ? [
+            'experiment-headline-over'
+        ] : [],
+        ...parsed.points.length > AI_EXPERIMENT_MAX_POINTS ? [
+            'experiment-points-over'
+        ] : [],
+        ...parsed.points.flatMap((point)=>point.length > AI_EXPERIMENT_POINT_MAX_CHARS ? [
+                'experiment-point-over'
+            ] : []),
+        ...parsed.next.length > AI_EXPERIMENT_NEXT_MAX_CHARS ? [
+            'experiment-next-over'
+        ] : []
+    ];
+    const broken = [
+        ...explained.findings,
+        ...codes(detectPublishIntent(answer)),
+        ...codes(detectOffVoiceCopy([
+            {
+                at: 'headline',
+                text: explained.headline
+            },
+            ...explained.points.map((point, index)=>({
+                    at: `points.${index}`,
+                    text: point
+                }))
+        ], evalCase.framing))
+    ];
+    return {
+        readable: true,
+        rules: broken.length === 0,
+        budget: over.length === 0,
+        findings: [
+            ...new Set([
+                ...broken,
+                ...over
+            ])
+        ]
+    };
+}
+/** An answer held to its kind's checks; a page's links to the sections of the plan it was built from. */ function checkAnswer(evalCase, answer, plan) {
+    const outputKind = AI_EVAL_TREE_OUTPUT[evalCase.kind];
+    if (outputKind) return checkTree(evalCase, outputKind, answer, plan);
+    switch(evalCase.kind){
+        case 'seo':
+            return checkSeo(evalCase, answer);
+        case 'product':
+        case 'catalog':
+        case 'categories':
+            return checkProducts(evalCase, answer);
+        case 'theme':
+            return checkTheme(evalCase, answer);
+        case 'element':
+            return checkCopy(evalCase, answer, {
+                maxChars: AI_TEXT_LIMITS.body
+            });
+        case 'blog':
+            return checkBlog(evalCase, answer);
+        case 'text':
+            return checkCopy(evalCase, answer, {
+                maxWords: AI_EVAL_TEXT_MAX_WORDS
+            });
+        case 'chat':
+            return checkCopy(evalCase, answer, {
+                maxChars: AI_EVAL_CHAT_MAX_CHARS
+            }, chatLinks(evalCase));
+        case 'workflow':
+            return checkWorkflow(evalCase, answer);
+        case 'insight':
+            return checkInsight(evalCase, answer);
+        case 'crm':
+            return checkCrm(evalCase, answer);
+        case 'experiment':
+            return checkExperiment(evalCase, answer);
+        default:
+            return {
+                readable: false,
+                rules: false,
+                budget: false,
+                findings: [
+                    'kind-unknown'
+                ]
+            };
+    }
+}
+/** The plan held to the plan rules, then to the shape the case expects. */ export function checkAiEvalPlan(evalCase, plan) {
+    var _evalCase_capabilities, _shape_reuse;
+    var _evalCase_expected;
+    const shape = (_evalCase_expected = evalCase.expected) == null ? void 0 : _evalCase_expected.plan;
+    if (!shape) return {
+        pass: true,
+        findings: []
+    };
+    const result = aiDoctrinePlanCheck(evalCase.inventory, evalCase.framing, (_evalCase_capabilities = evalCase.capabilities) != null ? _evalCase_capabilities : null)(isRecord(plan) ? plan : {});
+    if (!result.value) return {
+        pass: false,
+        findings: [
+            'plan-unreadable',
+            ...codes(result.violations)
+        ]
+    };
+    const value = result.value;
+    const shapeFindings = [
+        ...value.screens.length < shape.screens[0] || value.screens.length > shape.screens[1] ? [
+            `plan-screens:${value.screens.length}`
+        ] : [],
+        ...((_shape_reuse = shape.reuse) != null ? _shape_reuse : []).filter((id)=>!value.reuse.some((entry)=>entry.id === id)).map((id)=>`plan-reuse-missing:${id}`),
+        ...shape.layout ? value.screens.filter((screen)=>screen.layout !== shape.layout).map((screen)=>`plan-layout:${screen.slug}`) : [],
+        ...shape.maxCreate !== undefined && value.create.length > shape.maxCreate ? [
+            `plan-create:${value.create.length}`
+        ] : [],
+        ...shape.createKinds ? value.create.filter((entry)=>{
+            var _shape_createKinds;
+            return !((_shape_createKinds = shape.createKinds) == null ? void 0 : _shape_createKinds.includes(entry.kind));
+        }).map((entry)=>`plan-create-kind:${entry.kind}`) : []
+    ];
+    const findings = [
+        ...codes(result.violations),
+        ...shapeFindings
+    ];
+    return {
+        pass: findings.length === 0,
+        findings
+    };
+}
+const clampGrade = (value)=>{
+    if (value === null || value === undefined) return null;
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? Math.min(5, Math.max(1, parsed)) : null;
+};
+/** A rubric's mean and whether it passes: the mean at `AI_EVAL_RUBRIC_PASS_MEAN`, nothing under three. */ export function aiEvalRubricVerdict(rubric) {
+    const grades = [
+        rubric == null ? void 0 : rubric.structure,
+        rubric == null ? void 0 : rubric.copy,
+        rubric == null ? void 0 : rubric.reuse
+    ].map(clampGrade).filter((grade)=>grade !== null);
+    if (!grades.length) return {
+        mean: 1,
+        pass: false
+    };
+    const mean = grades.reduce((sum, grade)=>sum + grade, 0) / grades.length;
+    return {
+        mean,
+        pass: mean >= AI_EVAL_RUBRIC_PASS_MEAN && Math.min(...grades) >= 3
+    };
+}
+/**
+ * The widths a readable document of a rendered kind is held to, from its
+ * recorded device audit: `null` when the answer is not such a document or no
+ * audits were given; failing, with `widths-unrecorded`, when none was recorded
+ * from this answer.
+ */ function checkWidths(evalCase, answer, checked, audits) {
+    if (!audits || !(checked == null ? void 0 : checked.readable) || !aiEvalRendersAtDeviceWidths(evalCase.kind)) return null;
+    const audit = audits(evalCase, answer);
+    if (!audit) return {
+        pass: false,
+        findings: [
+            'widths-unrecorded'
+        ]
+    };
+    const findings = aiDeviceAuditFindings(audit);
+    return {
+        pass: findings.length === 0,
+        findings
+    };
+}
+/** One answer to one golden brief, scored. */ export function scoreAiEvalCandidate(evalCase, candidate, audits) {
+    var _candidate_scope, _ref, _ref1, _ref2, _ref3, _ref4;
+    var _evalCase_expected;
+    const scope = (_candidate_scope = candidate.scope) != null ? _candidate_scope : 'full';
+    const checked = scope === 'full' ? checkAnswer(evalCase, candidate.answer, candidate.plan) : null;
+    const widths = checkWidths(evalCase, candidate.answer, checked, audits);
+    const plan = ((_evalCase_expected = evalCase.expected) == null ? void 0 : _evalCase_expected.plan) ? checkAiEvalPlan(evalCase, candidate.plan) : null;
+    const rubric = aiEvalRubricVerdict(candidate.rubric);
+    const rubricScore = (rubric.mean - 1) / 4;
+    const parts = [
+        ...checked ? [
+            checked.readable ? 1 : 0,
+            checked.rules ? 1 : 0,
+            checked.budget ? 1 : 0
+        ] : [],
+        ...plan ? [
+            plan.pass ? 1 : 0
+        ] : [],
+        rubricScore
+    ];
+    const score = Math.round(parts.reduce((sum, part)=>sum + part, 0) / parts.length * 10000) / 10000;
+    const documentPasses = checked ? checked.readable && checked.rules && checked.budget : true;
+    return {
+        caseId: evalCase.id,
+        kind: evalCase.kind,
+        source: candidate.source,
+        scope,
+        model: candidate.model,
+        checks: {
+            readable: checked ? checked.readable : null,
+            rules: checked ? checked.rules : null,
+            budget: checked ? checked.budget : null,
+            plan: plan ? plan.pass : null,
+            responsive: widths ? widths.pass : null,
+            rubric: rubric.pass
+        },
+        rubricScore: Math.round(rubricScore * 10000) / 10000,
+        score,
+        // A plan answer to a brief with no expected plan has nothing to pass.
+        pass: documentPasses && ((_ref = widths == null ? void 0 : widths.pass) != null ? _ref : true) && ((_ref1 = plan == null ? void 0 : plan.pass) != null ? _ref1 : scope === 'full') && rubric.pass,
+        findings: [
+            ...(_ref2 = checked == null ? void 0 : checked.findings) != null ? _ref2 : [],
+            ...(_ref3 = widths == null ? void 0 : widths.findings) != null ? _ref3 : [],
+            ...(_ref4 = plan == null ? void 0 : plan.findings) != null ? _ref4 : []
+        ]
+    };
+}
+/**
+ * The checks a control failed; the harness requires every one it names. A
+ * control whose widths were never recorded fails none of them.
+ */ export function scoreAiEvalControl(evalCase, control, audits) {
+    var _control_plan;
+    var _evalCase_candidates_, _evalCase_expected;
+    // A control is the case's first answer with one thing wrong, built from its plan unless it carries its own.
+    const checked = checkAnswer(evalCase, control.answer, (_control_plan = control.plan) != null ? _control_plan : (_evalCase_candidates_ = evalCase.candidates[0]) == null ? void 0 : _evalCase_candidates_.plan);
+    const widths = (audits == null ? void 0 : audits(evalCase, control.answer)) ? checkWidths(evalCase, control.answer, checked, audits) : null;
+    const plan = control.plan !== undefined && ((_evalCase_expected = evalCase.expected) == null ? void 0 : _evalCase_expected.plan) ? checkAiEvalPlan(evalCase, control.plan) : null;
+    return [
+        ...checked.readable ? [] : [
+            'readable'
+        ],
+        ...checked.rules ? [] : [
+            'rules'
+        ],
+        ...checked.budget ? [] : [
+            'budget'
+        ],
+        ...plan && !plan.pass ? [
+            'plan'
+        ] : [],
+        ...widths && !widths.pass ? [
+            'responsive'
+        ] : []
+    ];
+}
+/** Every kind's pass rate and mean score against its floor; a kind with no candidate is below it. */ export function summarizeAiEval(scores) {
+    return AI_EVAL_KINDS.map((kind)=>{
+        const own = scores.filter((score)=>score.kind === kind && score.scope === 'full');
+        const passed = own.filter((score)=>score.pass).length;
+        const passRate = own.length ? passed / own.length : null;
+        const meanScore = own.length ? Math.round(own.reduce((sum, score)=>sum + score.score, 0) / own.length * 10000) / 10000 : null;
+        const floor = AI_EVAL_FLOORS[kind];
+        return {
+            kind,
+            candidates: own.length,
+            passed,
+            passRate,
+            meanScore,
+            floor,
+            belowFloor: passRate === null || meanScore === null || passRate < floor.passRate || meanScore < floor.meanScore
+        };
+    });
+}
+/**
+ * The plan step's own score: every answer that carried a plan checked
+ * against its brief's expected shape, whole answers and plan answers alike.
+ */ export function summarizeAiEvalPlans(scores) {
+    const plans = scores.filter((score)=>score.checks.plan !== null);
+    const passed = plans.filter((score)=>score.checks.plan && score.checks.rubric).length;
+    return {
+        plans: plans.length,
+        passed,
+        passRate: plans.length ? passed / plans.length : null
+    };
+}
+/** A recording file read and checked for shape; throws naming the field at fault. */ export function readAiEvalRecording(raw, file) {
+    if (!isRecord(raw) || typeof raw['caseId'] !== 'string' || !isRecord(raw['candidate'])) {
+        throw new Error(`${file}: a recording holds a caseId and a candidate`);
+    }
+    const candidate = raw['candidate'];
+    // `null` says the grader failed and this answer was kept ungraded
+    // (AGL-3143); a missing key is a recording written by something that does
+    // not know what a grade is.
+    if (candidate['source'] !== 'recorded' || !(isRecord(candidate['rubric']) || candidate['rubric'] === null)) {
+        throw new Error(`${file}: a recording's candidate is recorded, and graded or explicitly ungraded`);
+    }
+    return raw;
+}
+// ── Reading a case file ───────────────────────────────────────────────────
+/** A case file read and checked for shape; throws naming the field at fault. */ export function readAiEvalCase(raw, file) {
+    const fail = (why)=>{
+        throw new Error(`${file}: ${why}`);
+    };
+    if (!isRecord(raw)) return fail('not an object');
+    const kind = raw['kind'];
+    if (typeof raw['id'] !== 'string' || !raw['id']) fail('no id');
+    if (typeof kind !== 'string' || !AI_EVAL_KINDS.includes(kind)) {
+        fail(`kind "${String(kind)}" is not one of ${AI_EVAL_KINDS.join(', ')}`);
+    }
+    if (typeof raw['brief'] !== 'string' || !raw['brief'].trim()) fail('no brief');
+    if (raw['siteName'] !== undefined && (typeof raw['siteName'] !== 'string' || !raw['siteName'].trim())) {
+        fail('siteName, where given, is the site’s name');
+    }
+    const automation = raw['automation'];
+    if (automation !== undefined) {
+        const action = isRecord(automation) ? automation['action'] : undefined;
+        const problem = isRecord(action) ? validateHostAction(action) : 'it holds no action';
+        if (problem) fail(`automation.action is not an action the Actions editor would save: ${problem}`);
+        if (isRecord(automation) && automation['run'] !== undefined && !isRecord(automation['run'])) {
+            fail('automation.run, where given, is what the run history recorded');
+        }
+    }
+    const media = raw['media'];
+    if (media !== undefined) {
+        if (!isRecord(media) || !Object.keys(media).length) fail('media, where given, names at least one asset');
+        for (const [mediaId, file] of Object.entries(media)){
+            if (typeof file !== 'string' || !AI_EVAL_MEDIA_FILE.test(file)) {
+                fail(`media.${mediaId} is not a picture's path under tools/ai-eval/fixtures`);
+            }
+        }
+    }
+    const candidates = raw['candidates'];
+    const controls = raw['controls'];
+    if (!Array.isArray(candidates) || !candidates.length) fail('no candidates');
+    if (!Array.isArray(controls) || !controls.length) fail('no controls: every case proves it can fail');
+    for (const [index, candidate] of candidates.entries()){
+        if (!isRecord(candidate)) fail(`candidates[${index}] is not an object`);
+        const source = candidate['source'];
+        if (source !== 'authored' && source !== 'recorded') {
+            fail(`candidates[${index}].source is neither authored nor recorded`);
+        }
+        if (!isRecord(candidate['rubric'])) {
+            fail(`candidates[${index}] has no rubric`);
+        }
+    }
+    for (const [index, control] of controls.entries()){
+        const fails = isRecord(control) ? control['fails'] : undefined;
+        if (!Array.isArray(fails) || !fails.length) fail(`controls[${index}] names no check it fails`);
+    }
+    const capabilities = raw['capabilities'] === undefined ? undefined : readAiEvalCapabilities(raw['capabilities'], fail);
+    return _extends({
+        framing: null,
+        inventory: null
+    }, raw, capabilities ? {
+        capabilities
+    } : {});
+}
+/**
+ * A case's capabilities read and checked for shape. A creation kind the case
+ * leaves out may be created, so a case names only what its workspace lacks;
+ * a Free workspace's case says so with `freeTaste`, which holds its plan to
+ * the sections the Free wall pays for (AGL-3070).
+ */ function readAiEvalCapabilities(raw, fail) {
+    if (!isRecord(raw) || typeof raw['reusableComponents'] !== 'boolean') {
+        return fail('capabilities.reusableComponents is not true or false');
+    }
+    const create = isRecord(raw['create']) ? raw['create'] : {};
+    const unrestricted = aiUnrestrictedPlanCapabilities();
+    const entries = AI_BUILD_PLAN_CREATE_KINDS.map((kind)=>{
+        const entry = create[kind];
+        if (entry === undefined) return [
+            kind,
+            unrestricted.create[kind]
+        ];
+        if (!isRecord(entry) || typeof entry['allowed'] !== 'boolean') {
+            return fail(`capabilities.create.${kind}.allowed is not true or false`);
+        }
+        const left = entry['left'];
+        const reason = entry['reason'];
+        return [
+            kind,
+            {
+                allowed: entry['allowed'],
+                left: typeof left === 'number' ? left : null,
+                reason: typeof reason === 'string' ? reason : null
+            }
+        ];
+    });
+    for (const kind of Object.keys(create)){
+        if (!AI_BUILD_PLAN_CREATE_KINDS.includes(kind)) {
+            fail(`capabilities.create.${kind} is not a creation kind`);
+        }
+    }
+    if (raw['freeTaste'] !== undefined && typeof raw['freeTaste'] !== 'boolean') {
+        return fail('capabilities.freeTaste is not true or false');
+    }
+    return _extends({
+        reusableComponents: raw['reusableComponents'],
+        create: Object.fromEntries(entries)
+    }, raw['freeTaste'] === true ? {
+        freeTaste: true
+    } : {});
+}
+
+//# sourceMappingURL=ai-eval.js.map