@texuli/tools 0.0.0-stage → 0.1.0-next.0

package/dist/golden/report.d.ts

@@ -0,0 +1,28 @@
+import { type GoldenScoreOptions } from './score.js';
+import { type GoldenCheckId, type GoldenScore, type GoldenTask } from './types.js';
+/**
+ * Model runs are report-only: a run is scored and summarized, and nothing
+ * here throws or fails a build on a model's output. The reference trees are
+ * the gate (`packages/tools/tests/golden.test.ts`).
+ */
+export interface GoldenRunReport {
+    /** One score per golden task, in task order (a missing output scores as empty). */
+    scores: readonly GoldenScore[];
+    /** Task ids the run had no output for. */
+    missing: readonly string[];
+    /** Output keys that name no golden task (ignored). */
+    unknown: readonly string[];
+    /** Per check: how many tasks passed it, and how many ran it. */
+    checks: Readonly<Record<GoldenCheckId, {
+        passed: number;
+        ran: number;
+    }>>;
+    /** Tasks that passed every check that ran. */
+    passed: number;
+    total: number;
+}
+export declare function scoreGoldenRun(outputs: unknown, options?: GoldenScoreOptions & {
+    tasks?: readonly GoldenTask[];
+}): Promise<GoldenRunReport>;
+/** A Markdown summary of a run: totals, per-check pass counts, then failures. */
+export declare function formatGoldenReport(report: GoldenRunReport): string;

package/dist/golden/report.js

@@ -0,0 +1,64 @@
+import { scoreGoldenTree } from "./score.js";
+import { GOLDEN_TASKS } from "./tasks.js";
+import { GOLDEN_CHECKS } from "./types.js";
+export async function scoreGoldenRun(outputs, options = {}) {
+    const tasks = options.tasks ?? GOLDEN_TASKS;
+    const record = outputs && typeof outputs === 'object' && !Array.isArray(outputs)
+        ? outputs
+        : {};
+    const ids = new Set(tasks.map((task) => task.id));
+    const scores = [];
+    for (const task of tasks)
+        scores.push(await scoreGoldenTree(task, record[task.id], options));
+    const checks = Object.fromEntries(GOLDEN_CHECKS.map((id) => {
+        const rows = scores.map((score) => score.checks.find((check) => check.check === id));
+        return [
+            id,
+            {
+                passed: rows.filter((row) => row?.status === 'pass').length,
+                ran: rows.filter((row) => row && row.status !== 'skipped').length,
+            },
+        ];
+    }));
+    return {
+        scores,
+        missing: tasks.filter((task) => !(task.id in record)).map((task) => task.id),
+        unknown: Object.keys(record).filter((key) => !ids.has(key)),
+        checks,
+        passed: scores.filter((score) => score.passed).length,
+        total: tasks.length,
+    };
+}
+/** A Markdown summary of a run: totals, per-check pass counts, then failures. */
+export function formatGoldenReport(report) {
+    const lines = [
+        '# Golden set run (report only)',
+        '',
+        `**${report.passed} of ${report.total}** tasks passed every check that ran.`,
+        '',
+        '| Check | Passed | Ran |',
+        '| --- | ---: | ---: |',
+        ...GOLDEN_CHECKS.map((id) => `| ${id} | ${report.checks[id].passed} | ${report.checks[id].ran} |`),
+    ];
+    if (report.missing.length)
+        lines.push('', `Missing outputs: ${report.missing.join(', ')}`);
+    if (report.unknown.length)
+        lines.push('', `Ignored keys: ${report.unknown.join(', ')}`);
+    const missing = new Set(report.missing);
+    const failed = report.scores.filter((score) => !score.passed && !missing.has(score.taskId));
+    if (failed.length) {
+        lines.push('', '## Failures');
+        for (const score of failed) {
+            lines.push('', `### ${score.taskId}`);
+            for (const check of score.checks) {
+                if (check.status !== 'fail')
+                    continue;
+                for (const issue of check.issues.slice(0, 8))
+                    lines.push(`- ${check.check}: ${issue}`);
+                if (check.issues.length > 8)
+                    lines.push(`- ${check.check}: … ${check.issues.length - 8} more`);
+            }
+        }
+    }
+    return `${lines.join('\n')}\n`;
+}

package/dist/golden/score.d.ts

@@ -0,0 +1,16 @@
+import type { RenderCatalog } from '@texuli/catalog/render/renderer';
+import type { ElementComponentLoader } from '../elements/content.js';
+import { type GoldenAxeRunner, type GoldenScore, type GoldenTask } from './types.js';
+export interface GoldenScoreOptions {
+    /** Defaults to the golden render catalog (the agent surface). */
+    catalog?: RenderCatalog;
+    /**
+     * Runs axe on the painted tree. Without it (or without `document`), the axe
+     * check is `skipped`: offline scoring needs a DOM and an axe-core instance.
+     */
+    axe?: GoldenAxeRunner;
+    document?: Document;
+    loaders?: Readonly<Record<string, ElementComponentLoader>>;
+}
+/** Scores one tree against one golden task. Never throws on a bad tree. */
+export declare function scoreGoldenTree(task: GoldenTask, tree: unknown, options?: GoldenScoreOptions): Promise<GoldenScore>;

package/dist/golden/score.js

@@ -0,0 +1,116 @@
+import { safeParseBlock } from '@texuli/blocks';
+import { validateDraft } from '@texuli/catalog/validate';
+import { GOLDEN_FRAME_IDS, goldenRenderCatalog } from "./catalog.js";
+import { checkGuidance } from "./guidance.js";
+import { paintGoldenTree } from "./paint.js";
+import { checkSinks } from "./sinks.js";
+import { GOLDEN_CHECKS, } from "./types.js";
+function result(check, issues) {
+    return { check, status: issues.length === 0 ? 'pass' : 'fail', issues };
+}
+function isTreeInput(value) {
+    return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+function componentIds(roots, out = new Set()) {
+    for (const node of roots) {
+        if (node.kind !== 'component')
+            continue;
+        out.add(node.id);
+        componentIds(node.children, out);
+    }
+    return out;
+}
+function line(d) {
+    return `${d.path} ${d.id ?? ''}: ${d.code} (${d.message})`;
+}
+/**
+ * Schema and grammar verdicts from the shared `validateDraft`: input,
+ * structure, limits, the sink-free renderer, deep block schemas and the
+ * frame root are `schema`; the composition grammar is `grammar`. The raw
+ * sink scan is its own golden check (`checkSinks`, the same scan).
+ */
+function checkDraft(tree, task, catalog) {
+    if (!isTreeInput(tree)) {
+        return { issues: ['the output is not a catalog tree object'], grammar: [], roots: [] };
+    }
+    const validation = validateDraft(tree, {
+        catalog,
+        tier: task.tier,
+        frameRoots: GOLDEN_FRAME_IDS,
+        checkBlock: (block) => safeParseBlock(block),
+    });
+    const of = (check) => validation.diagnostics.filter((d) => d.check === check).map(line);
+    return {
+        issues: [
+            ...of('input'),
+            ...of('structure'),
+            ...of('limits'),
+            ...of('render'),
+            ...of('root'),
+            ...of('blocks'),
+        ],
+        grammar: of('grammar'),
+        roots: validation.roots,
+    };
+}
+/** Scores one tree against one golden task. Never throws on a bad tree. */
+export async function scoreGoldenTree(task, tree, options = {}) {
+    const catalog = options.catalog ?? goldenRenderCatalog();
+    const schema = checkDraft(tree, task, catalog);
+    const checks = [result('schema', schema.issues)];
+    const skipped = (check, why) => ({
+        check,
+        status: 'skipped',
+        issues: [why],
+    });
+    if (!isTreeInput(tree)) {
+        // Nothing to inspect: only the schema (and coverage) verdicts mean anything.
+        checks.push(skipped('dontUseWhen', 'not a tree'), skipped('sinks', 'not a tree'));
+    }
+    else {
+        checks.push(result('dontUseWhen', checkGuidance(tree)), result('sinks', checkSinks(tree)));
+    }
+    if (schema.roots.length === 0) {
+        checks.push(skipped('grammar', 'nothing rendered'));
+    }
+    else {
+        checks.push(result('grammar', schema.grammar));
+    }
+    if (!options.axe || !options.document) {
+        checks.push({ check: 'axe', status: 'skipped', issues: ['no DOM axe runner given'] });
+    }
+    else if (schema.issues.length > 0) {
+        checks.push({
+            check: 'axe',
+            status: 'skipped',
+            issues: ['not painted: the schema check failed'],
+        });
+    }
+    else {
+        const paint = await paintGoldenTree(schema.roots, options.document, {
+            loaders: options.loaders,
+            title: task.title,
+        });
+        try {
+            const violations = await options.axe(options.document);
+            checks.push(result('axe', [
+                ...paint.issues,
+                ...violations.map((v) => `${v.id} (${v.impact ?? 'n/a'}, ${v.nodes} node${v.nodes === 1 ? '' : 's'}): ${v.sample ?? ''}`),
+            ]));
+        }
+        finally {
+            paint.destroy();
+        }
+    }
+    const present = componentIds(schema.roots);
+    checks.push(result('coverage', task.mustInclude.filter((id) => !present.has(id)).map((id) => `missing a ${id} component`)));
+    const ordered = GOLDEN_CHECKS.map((id) => checks.find((check) => check.check === id));
+    const ran = ordered.filter((check) => check.status !== 'skipped');
+    const passedCount = ran.filter((check) => check.status === 'pass').length;
+    return {
+        taskId: task.id,
+        passed: ran.every((check) => check.status === 'pass'),
+        score: ran.length === 0 ? 0 : passedCount / ran.length,
+        checks: ordered,
+    };
+}

package/dist/golden/sinks.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Sink checks on the raw model output, before the renderer cleans it: a tree
+ * that tries an HTML string, a script URL or a style is penalized even though
+ * the sink-free renderer would have made it inert. The scan is the shared
+ * `scanSinks` that `validateDraft` runs.
+ */
+/** Every sink-like value in an untrusted tree (any shape), with its path. */
+export declare function checkSinks(tree: unknown): string[];

package/dist/golden/sinks.js

@@ -0,0 +1,11 @@
+import { scanSinks } from '@texuli/catalog/validate';
+/**
+ * Sink checks on the raw model output, before the renderer cleans it: a tree
+ * that tries an HTML string, a script URL or a style is penalized even though
+ * the sink-free renderer would have made it inert. The scan is the shared
+ * `scanSinks` that `validateDraft` runs.
+ */
+/** Every sink-like value in an untrusted tree (any shape), with its path. */
+export function checkSinks(tree) {
+    return scanSinks(tree).map((finding) => `${finding.path}: ${finding.message}`);
+}

package/dist/golden/tasks.d.ts

@@ -0,0 +1,4 @@
+import type { GoldenTask } from './types.js';
+export declare const GOLDEN_TASKS: readonly GoldenTask[];
+/** The task with `id`, or `undefined`. */
+export declare function goldenTask(id: string): GoldenTask | undefined;