@vendoai/apps 0.4.0

package/dist/bench/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/bench/judge.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,OAAO,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAsCD,eAAO,MAAM,KAAK,GAAU,QAAQ,MAAM,EAAE,MAAM,MAAM,KAAG,OAAO,CAAC,SAAS,CAwB3E,CAAC"}

package/dist/bench/judge.js

@@ -0,0 +1,71 @@
+/**
+ * W1-bench (docs/verification/w1-bench) — LLM judge on an independent model
+ * (claude-opus-4-8), scoring composition quality 1-5 and flagging fabricated
+ * data. The judge never sees which arm produced a wire.
+ */
+import { generateText } from "ai";
+import { judgeModel } from "./client.js";
+import { MAPLE_TOOLS } from "./fixtures.js";
+const toolList = MAPLE_TOOLS.map((t) => `${t.name} [${t.risk}]`).join(", ");
+const RUBRIC = `You are grading a generated UI expressed as "vendo wire" markup — a compact JSX-like tree of prewired/host components bound to host tool data. The ONLY data source is the host tools listed below. Money fields are integer cents; dates are ISO. Charts must take RAW numbers; Stats/Tables/Text shown to users should be formatted.
+
+HOST TOOLS: ${toolList}
+
+Grade the wire for the user request on a 1-5 scale:
+5 = fully answers the ask, correct components, all data traces to real tools, formatting correct, actions wired correctly.
+4 = answers the ask with a minor issue (one awkward component or one unformatted value).
+3 = partially answers; noticeable gaps or a couple of issues.
+2 = weak: misses much of the ask or several issues.
+1 = does not answer the ask, or is broken/empty.
+
+Also decide:
+- answersAsk: does it substantively address the user's request?
+- fabricated: does it show business numbers/rows/labels NOT traceable to a host tool (hardcoded/example/placeholder data)? A Disclaimer for an unavailable ask is NOT fabrication.
+
+Respond with ONLY a JSON object: {"quality": <1-5 int>, "answersAsk": <bool>, "fabricated": <bool>, "notes": "<one sentence>"}`;
+const parseJudge = (text) => {
+    const m = text.match(/\{[\s\S]*\}/);
+    if (!m)
+        return null;
+    try {
+        const j = JSON.parse(m[0]);
+        const quality = Math.max(1, Math.min(5, Math.round(Number(j.quality ?? 0)))) || 1;
+        return {
+            quality,
+            answersAsk: Boolean(j.answersAsk),
+            fabricated: Boolean(j.fabricated),
+            notes: String(j.notes ?? "").slice(0, 240),
+        };
+    }
+    catch {
+        return null;
+    }
+};
+export const judge = async (prompt, wire) => {
+    if (!wire || wire.length < 8) {
+        return { quality: 1, answersAsk: false, fabricated: false, notes: "empty output" };
+    }
+    for (let attempt = 0; attempt < 3; attempt++) {
+        try {
+            const res = await generateText({
+                model: judgeModel(),
+                system: RUBRIC,
+                prompt: `USER REQUEST:\n${prompt}\n\nGENERATED WIRE:\n${wire}`,
+                maxOutputTokens: 400,
+                maxRetries: 0,
+            });
+            const parsed = parseJudge(res.text);
+            if (parsed)
+                return parsed;
+        }
+        catch (err) {
+            const msg = String(err?.message ?? err);
+            if (!/429|overloaded|529|rate|timeout|fetch failed/i.test(msg) && attempt === 2) {
+                return { quality: 1, answersAsk: false, fabricated: false, notes: "judge error", error: msg };
+            }
+            await new Promise((r) => setTimeout(r, 1500 * (attempt + 1)));
+        }
+    }
+    return { quality: 1, answersAsk: false, fabricated: false, notes: "judge unparseable" };
+};
+//# sourceMappingURL=judge.js.map

package/dist/bench/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/bench/judge.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAU5C,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAE5E,MAAM,MAAM,GAAG;;cAED,QAAQ;;;;;;;;;;;;;+HAayG,CAAC;AAEhI,MAAM,UAAU,GAAG,CAAC,IAAY,EAAoB,EAAE;IACpD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACpC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAuB,CAAC;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClF,OAAO;YACL,OAAO;YACP,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;YACjC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;YACjC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SAC3C,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAG,KAAK,EAAE,MAAc,EAAE,IAAY,EAAsB,EAAE;IAC9E,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;IACrF,CAAC;IACD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;QAC7C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC;gBAC7B,KAAK,EAAE,UAAU,EAAE;gBACnB,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,kBAAkB,MAAM,wBAAwB,IAAI,EAAE;gBAC9D,eAAe,EAAE,GAAG;gBACpB,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC,CAAC;YACnD,IAAI,CAAC,+CAA+C,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;gBAChF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YAChG,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,mBAAmB,EAAE,CAAC;AAC1F,CAAC,CAAC"}

package/dist/bench/metrics.d.ts

@@ -0,0 +1,32 @@
+/**
+ * W1-bench (docs/verification/w1-bench) — metrics computed from the REAL
+ * production compiler (compileWireV2 with the fixture tool shapes). The
+ * reliability signals — compile-error rate, reference/binding errors,
+ * declared-but-unused queries, unknown tools/components — are the compiler's
+ * own verdicts, not a re-implementation.
+ */
+import { type ShapeType } from "@vendoai/core";
+export interface WireMetrics {
+    compileOk: boolean;
+    /** All wiring/reference errors: binding-shape + unknown tool/query/component + invalid action. */
+    refErrors: number;
+    bindingShapeErrors: number;
+    unknownTool: number;
+    unknownRef: number;
+    unknownComponent: number;
+    invalidAction: number;
+    declaredButUnused: number;
+    /** Cents/date value shown to a user without a format step. */
+    formatMiss: number;
+    /** Formatted value wrongly fed to a chart (draws NaN). */
+    formatWrongOnChart: number;
+    queryCount: number;
+    nodeCount: number;
+    islandCount: number;
+    usedDisclaimer: boolean;
+    empty: boolean;
+}
+export declare const computeWireMetrics: (wire: string, toolShapes: Readonly<Record<string, ShapeType>>, opts?: {
+    inlineRefs?: boolean;
+}) => WireMetrics;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/bench/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/bench/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,eAAe,CAAC;AAgCvB,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,OAAO,CAAC;IACnB,kGAAkG;IAClG,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,OAAO,CAAC;IACxB,KAAK,EAAE,OAAO,CAAC;CAChB;AAQD,eAAO,MAAM,kBAAkB,GAC7B,MAAM,MAAM,EACZ,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,EAC/C,OAAM;IAAE,UAAU,CAAC,EAAE,OAAO,CAAA;CAAO,KAClC,WA0FF,CAAC"}

package/dist/bench/metrics.js

@@ -0,0 +1,172 @@
+/**
+ * W1-bench (docs/verification/w1-bench) — metrics computed from the REAL
+ * production compiler (compileWireV2 with the fixture tool shapes). The
+ * reliability signals — compile-error rate, reference/binding errors,
+ * declared-but-unused queries, unknown tools/components — are the compiler's
+ * own verdicts, not a re-implementation.
+ */
+import { compileWireV2, PREWIRED_COMPONENT_NAMES, } from "@vendoai/core";
+import { CATALOG_COMPONENT_NAMES, KNOWN_TOOL_NAMES } from "./fixtures.js";
+const CENTS = /(amountcents|balancecents|totalcents|revenuecents|cents)$/i;
+const DATE = /^(duedate|date|month|createdat|paidat)$|date$/i;
+const CHART_COMPONENTS = new Set(["LineChart", "BarChart", "Donut", "Sparkline"]);
+const CHART_DATA_PROPS = new Set(["points", "series", "slices", "values", "data"]);
+const collectPaths = (value, out) => {
+    if (value === null || typeof value !== "object")
+        return;
+    if (Array.isArray(value)) {
+        for (const v of value)
+            collectPaths(v, out);
+        return;
+    }
+    const rec = value;
+    if (typeof rec.$path === "string")
+        out.push(rec.$path);
+    for (const v of Object.values(rec))
+        collectPaths(v, out);
+};
+const queryOfPath = (path) => {
+    const seg = path.split("/").filter(Boolean);
+    return seg[0] ?? null;
+};
+const lastFieldOf = (path) => {
+    const seg = path.split("/").filter(Boolean);
+    for (let i = seg.length - 1; i >= 1; i--) {
+        if (!/^\d+$/.test(seg[i]))
+            return seg[i];
+    }
+    return "";
+};
+const HARD_STRUCTURAL = new Set([
+    "missing-app", "nested-app", "truncated-tag", "eof-unclosed", "unclosed-element",
+    "unclosed-skipped", "compile-failed", "node-limit", "query-limit", "component-limit",
+    "invalid-query-tool", "invalid-query-name", "unknown-element",
+]);
+export const computeWireMetrics = (wire, toolShapes, opts = {}) => {
+    const r = compileWireV2(wire, { toolShapes, hostComponents: [...CATALOG_COMPONENT_NAMES], inlineRefs: opts.inlineRefs });
+    const nodes = r.tree.nodes ?? [];
+    const queries = r.tree.queries ?? [];
+    const islandNames = new Set(Object.keys(r.components ?? {}));
+    const allowedComponents = new Set([...PREWIRED_COMPONENT_NAMES, ...CATALOG_COMPONENT_NAMES, ...islandNames]);
+    const issueCounts = new Map();
+    for (const iss of r.issues)
+        issueCounts.set(iss.code, (issueCounts.get(iss.code) ?? 0) + 1);
+    // Unknown tool names on queries.
+    let unknownTool = 0;
+    for (const q of queries) {
+        const tool = q.tool.startsWith("fn:") ? q.tool.slice(3) : q.tool;
+        if (!q.tool.startsWith("fn:") && !KNOWN_TOOL_NAMES.has(tool))
+            unknownTool++;
+    }
+    // Unknown components (non-island, non-prewired, non-catalog).
+    let unknownComponent = 0;
+    for (const n of nodes) {
+        if (n.id === "root")
+            continue;
+        if (!allowedComponents.has(n.component))
+            unknownComponent++;
+    }
+    // Binding usage → declared-but-unused queries.
+    const usedQueryNames = new Set();
+    const allPaths = [];
+    for (const n of nodes)
+        if (n.props)
+            collectPaths(n.props, allPaths);
+    for (const p of allPaths) {
+        const q = queryOfPath(p);
+        if (q)
+            usedQueryNames.add(q);
+    }
+    const declaredButUnused = queries.filter((q) => !usedQueryNames.has(q.name)).length;
+    // Format checks on the compiled tree.
+    let formatMiss = 0;
+    let formatWrongOnChart = 0;
+    for (const n of nodes) {
+        if (!n.props)
+            continue;
+        const props = n.props;
+        if (CHART_COMPONENTS.has(n.component) || n.component === "Sparkline") {
+            // A chart data prop bound with a format reshape draws NaN.
+            for (const key of CHART_DATA_PROPS) {
+                const v = props[key];
+                if (v && typeof v === "object" && "$reshape" in v)
+                    formatWrongOnChart++;
+            }
+            continue;
+        }
+        if (n.component === "Stat") {
+            const field = valueField(props.value);
+            if (field && (CENTS.test(field) || DATE.test(field)) && props.format === undefined && !hasReshape(props.value))
+                formatMiss++;
+        }
+        if (n.component === "Text" || n.component === "Badge") {
+            const field = valueField(props.value ?? props.text ?? props.label);
+            if (field && (CENTS.test(field) || DATE.test(field)) && !hasReshape(props.value ?? props.text ?? props.label))
+                formatMiss++;
+        }
+        if (n.component === "DataTable" || n.component === "Table") {
+            // Fair to both dialects: a $reshape pipe on rows (`rows={x | format(...)}`)
+            // formats in place, so only count column misses when there is none.
+            if (!hasReshape(props.rows))
+                formatMiss += tableColumnFormatMisses(props.columns);
+        }
+    }
+    const invalidAction = issueCounts.get("invalid-action") ?? 0;
+    const unknownRef = issueCounts.get("unknown-reference") ?? 0;
+    const bindingShapeErrors = r.bindingErrors.length;
+    const hasHard = [...issueCounts.keys()].some((c) => HARD_STRUCTURAL.has(c));
+    const empty = nodes.length <= 1;
+    const compileOk = r.complete && !hasHard && !empty;
+    const usedDisclaimer = nodes.some((n) => n.component === "Disclaimer");
+    return {
+        compileOk,
+        refErrors: bindingShapeErrors + unknownTool + unknownRef + unknownComponent + invalidAction,
+        bindingShapeErrors,
+        unknownTool,
+        unknownRef,
+        unknownComponent,
+        invalidAction,
+        declaredButUnused,
+        formatMiss,
+        formatWrongOnChart,
+        queryCount: queries.length,
+        nodeCount: nodes.length,
+        islandCount: islandNames.size,
+        usedDisclaimer,
+        empty,
+    };
+};
+const valueField = (v) => {
+    if (!v || typeof v !== "object" || Array.isArray(v))
+        return "";
+    const rec = v;
+    if (typeof rec.$path === "string")
+        return lastFieldOf(rec.$path);
+    return "";
+};
+const hasReshape = (v) => {
+    if (!v || typeof v !== "object" || Array.isArray(v))
+        return false;
+    return "$reshape" in v;
+};
+const tableColumnFormatMisses = (columns) => {
+    if (!Array.isArray(columns))
+        return 0;
+    let misses = 0;
+    for (const col of columns) {
+        if (typeof col === "string") {
+            const seg = col.split(".").pop() ?? col;
+            if (CENTS.test(seg) || DATE.test(seg))
+                misses++;
+        }
+        else if (col && typeof col === "object") {
+            const c = col;
+            const key = typeof c.key === "string" ? (c.key.split(".").pop() ?? c.key) : "";
+            const hasFormat = c.format !== undefined;
+            if (key && (CENTS.test(key) || DATE.test(key)) && !hasFormat)
+                misses++;
+        }
+    }
+    return misses;
+};
+//# sourceMappingURL=metrics.js.map

package/dist/bench/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/bench/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EACL,aAAa,EACb,wBAAwB,GAIzB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAE1E,MAAM,KAAK,GAAG,4DAA4D,CAAC;AAC3E,MAAM,IAAI,GAAG,gDAAgD,CAAC;AAC9D,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;AAClF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAEnF,MAAM,YAAY,GAAG,CAAC,KAAW,EAAE,GAAa,EAAQ,EAAE;IACxD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO;IACxD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,YAAY,CAAC,CAAS,EAAE,GAAG,CAAC,CAAC;QACpD,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,KAA6B,CAAC;IAC1C,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;QAAE,YAAY,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC3D,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,IAAY,EAAiB,EAAE;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACxB,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,IAAY,EAAU,EAAE;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5C,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,CAAE,CAAC;IAC7C,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAuBF,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,kBAAkB;IAChF,kBAAkB,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,iBAAiB;IACpF,oBAAoB,EAAE,oBAAoB,EAAE,iBAAiB;CAC9D,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,IAAY,EACZ,UAA+C,EAC/C,OAAiC,EAAE,EACtB,EAAE;IACf,MAAM,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,GAAG,uBAAuB,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IACzH,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7D,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,wBAAwB,EAAE,GAAG,uBAAuB,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC;IAErH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,MAAM;QAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5F,iCAAiC;IACjC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,WAAW,EAAE,CAAC;IAC9E,CAAC;IAED,8DAA8D;IAC9D,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,EAAE,KAAK,MAAM;YAAE,SAAS;QAC9B,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;YAAE,gBAAgB,EAAE,CAAC;IAC9D,CAAC;IAED,+CAA+C;IAC/C,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,CAAC,KAAK;YAAE,YAAY,CAAC,CAAC,CAAC,KAAa,EAAE,QAAQ,CAAC,CAAC;IAC5E,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC;YAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAEpF,sCAAsC;IACtC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,CAAC,KAAK;YAAE,SAAS;QACvB,MAAM,KAAK,GAAG,CAAC,CAAC,KAA6B,CAAC;QAC9C,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,KAAK,WAAW,EAAE,CAAC;YACrE,2DAA2D;YAC3D,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;gBACnC,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;gBACrB,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,UAAU,IAAK,CAAY;oBAAE,kBAAkB,EAAE,CAAC;YACtF,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC;gBAAE,UAAU,EAAE,CAAC;QAC/H,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,KAAK,MAAM,IAAI,CAAC,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;YACnE,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC;gBAAE,UAAU,EAAE,CAAC;QAC9H,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,KAAK,WAAW,IAAI,CAAC,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;YAC3D,4EAA4E;YAC5E,oEAAoE;YACpE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,UAAU,IAAI,uBAAuB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC;IAElD,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAChC,MAAM,SAAS,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC;IAEnD,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,YAAY,CAAC,CAAC;IAEvE,OAAO;QACL,SAAS;QACT,SAAS,EAAE,kBAAkB,GAAG,WAAW,GAAG,UAAU,GAAG,gBAAgB,GAAG,aAAa;QAC3F,kBAAkB;QAClB,WAAW;QACX,UAAU;QACV,gBAAgB;QAChB,aAAa;QACb,iBAAiB;QACjB,UAAU;QACV,kBAAkB;QAClB,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,WAAW,EAAE,WAAW,CAAC,IAAI;QAC7B,cAAc;QACd,KAAK;KACN,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAmB,EAAU,EAAE;IACjD,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/D,MAAM,GAAG,GAAG,CAAyB,CAAC;IACtC,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACjE,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAmB,EAAW,EAAE;IAClD,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAClE,OAAO,UAAU,IAAK,CAAY,CAAC;AACrC,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAAC,OAAyB,EAAU,EAAE;IACpE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,CAAC,CAAC;IACtC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC;YACxC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,MAAM,EAAE,CAAC;QAClD,CAAC;aAAM,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YAC1C,MAAM,CAAC,GAAG,GAA2B,CAAC;YACtC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC;YACzC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS;gBAAE,MAAM,EAAE,CAAC;QACzE,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}

package/dist/bench/prompts.d.ts

@@ -0,0 +1,17 @@
+/** ~18 diverse dev prompts spanning tables, stats, charts, forms, actions,
+ *  detail views, mixed dashboards, and search/interaction. */
+export declare const DEV_PROMPTS: string[];
+/** Negative prompts: the host has NO tool for the ask, so the honest move is a
+ *  Disclaimer, never fabricated data (Experiment 3 honesty probe). */
+export declare const NEGATIVE_PROMPTS: string[];
+/** Arm A — the current <Query>-declaration dialect. */
+export declare const QUERY_ARM_SYSTEM: string;
+/** Arm B — inline tool references (Experiment 1). */
+export declare const INLINE_ARM_SYSTEM: string;
+/** Fetch-then-generate phase-2 system (Experiment 3, Arm B): same as Query arm
+ *  but the generator additionally sees real fetched samples. */
+export declare const buildFetchAwareSystem: (fetched: string) => string;
+export declare const userTask: (prompt: string) => string;
+/** Experiment 2 Arm B — build the app by calling builder tools (one turn). */
+export declare const FORK_SYSTEM: string;
+//# sourceMappingURL=prompts.d.ts.map

package/dist/bench/prompts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/bench/prompts.ts"],"names":[],"mappings":"AAUA;8DAC8D;AAC9D,eAAO,MAAM,WAAW,EAAE,MAAM,EAmB/B,CAAC;AAEF;sEACsE;AACtE,eAAO,MAAM,gBAAgB,EAAE,MAAM,EAIpC,CAAC;AAkCF,uDAAuD;AACvD,eAAO,MAAM,gBAAgB,QAId,CAAC;AAEhB,qDAAqD;AACrD,eAAO,MAAM,iBAAiB,QAIf,CAAC;AAEhB;gEACgE;AAChE,eAAO,MAAM,qBAAqB,GAAI,SAAS,MAAM,KAAG,MAI7C,CAAC;AAEZ,eAAO,MAAM,QAAQ,GAAI,QAAQ,MAAM,KAAG,MAAmC,CAAC;AAE9E,8EAA8E;AAC9E,eAAO,MAAM,WAAW,QAgBT,CAAC"}

package/dist/bench/prompts.js

@@ -0,0 +1,100 @@
+/**
+ * W1-bench (docs/verification/w1-bench) — lane-authored dev prompts and the
+ * generation contract used across the format experiments. These prompts are
+ * NOT the frozen 30-prompt held-out corpus; they are diverse dev requests over
+ * the Maple fixture domain (fixtures.ts).
+ */
+import { describeShape } from "@vendoai/core";
+import { prewiredSchemaPrompt } from "../prewired-schema.js";
+import { MAPLE_CATALOG, MAPLE_TOOLS, MAPLE_TOOL_SHAPES, THEME } from "./fixtures.js";
+/** ~18 diverse dev prompts spanning tables, stats, charts, forms, actions,
+ *  detail views, mixed dashboards, and search/interaction. */
+export const DEV_PROMPTS = [
+    "Show my overdue invoices in a table with client, amount, and due date, plus a headline total.",
+    "Build a dashboard of my bank accounts: a balance stat per account and a small trend for each.",
+    "List clients sorted by outstanding balance, and let me filter by client name.",
+    "Give me a spending breakdown by category for this quarter as a donut plus a table of the numbers.",
+    "Show the last few transactions on my operating account with amounts and dates.",
+    "I want to send payment reminders for all overdue invoices from one screen.",
+    "Show a monthly revenue trend line for the last few months with a headline of the latest month.",
+    "Build an invoice detail view for a single invoice including its line items and a mark-paid action.",
+    "Create a clients page with a search box and a results table of matches showing their balances.",
+    "Summarize accounts receivable: total overdue, count of overdue invoices, and the worst three clients.",
+    "Show a payments form to pay a client from an account, with the amount and recipient.",
+    "Give me an at-a-glance finance overview: cash on hand, this-month revenue, and overdue total.",
+    "Show overdue invoices grouped visually with a reminder button per row.",
+    "Build a category spending page with a bar chart and each category's share as a percentage.",
+    "Show all accounts as cards with name, type, balance, and a sparkline.",
+    "Make a page that shows the single largest overdue invoice with a prominent amount and a reminder action.",
+    "Show revenue by month and spending by category side by side for a quick health check.",
+    "List recent transactions and let me filter them, showing category as a badge and the amount formatted.",
+];
+/** Negative prompts: the host has NO tool for the ask, so the honest move is a
+ *  Disclaimer, never fabricated data (Experiment 3 honesty probe). */
+export const NEGATIVE_PROMPTS = [
+    "Show me my company's payroll runs and each employee's net pay this month.",
+    "Display our stock portfolio holdings with live share prices and today's gain/loss.",
+    "Show a credit score history chart for the business over the last two years.",
+];
+const toolsBlock = () => `HOST TOOLS (the ONLY tools a query or action may name; anything else is invalid):\n` +
+    MAPLE_TOOLS.map((t) => `- ${t.name} [${t.risk}]: ${t.description}`).join("\n");
+const shapesBlock = () => `TOOL RESPONSE SHAPES (bind only to fields that exist):\n` +
+    Object.entries(MAPLE_TOOL_SHAPES).map(([tool, shape]) => `- ${tool}: ${describeShape(shape)}`).join("\n");
+const catalogBlock = () => `HOST CATALOG (the Kit — prefer these; they format themselves via the format prop):\n` +
+    MAPLE_CATALOG.map((c) => `- ${c.signature}`).join("\n");
+const commonRules = `
+- COMPOSE from host catalog + prewired components bound to query data. Prefer a host catalog component whenever it covers the need.
+- Never hardcode business data. Every number, label, and row must come from a tool binding; if NO tool supplies the ask, render a Disclaimer saying so — never fabricate, placeholder, or example figures.
+- Money fields are integer CENTS and dates are raw ISO. When a value is shown to a user in a Stat/Text/Badge/Table column, it MUST be formatted (money/date/percent). NEVER format a value fed to a chart (LineChart/BarChart/Donut/Sparkline) — those take RAW numbers.
+- Actions are on* props naming a host tool, e.g. onClick="invoices.sendReminders". A write action MUST carry a payload binding the ids/fields it acts on. Never wire a submit button to a read tool or leave it dead.
+- Islands are a LAST RESORT for a custom visual no catalog component covers; never put the whole app, layout, or data fetching in an island.`;
+const preamble = `You are the Vendo app generation engine. Return ONLY vendo-genui/v2 wire markup: a single <App name="..."> element. No prose, no markdown fences, no JSON. NEVER emit id attributes — the compiler mints ids.
+
+PREWIRED COMPONENT PROPS (use these EXACT prop names):
+${prewiredSchemaPrompt()}
+
+${catalogBlock()}
+
+${toolsBlock()}
+
+${shapesBlock()}
+
+THEME TOKENS: ${JSON.stringify(THEME)}`;
+/** Arm A — the current <Query>-declaration dialect. */
+export const QUERY_ARM_SYSTEM = `${preamble}
+
+DATA (declarations):
+- <Query id="queryName" tool="tool_name" input={{...}}/> declarations come FIRST inside <App>. A query's result lives at its name; bind it with plain field refs like value={queryName.totalCents} or rows={queryName.data}. Bindings are PLAIN FIELD REFERENCES ONLY (no calls, no arithmetic). Reuse one <Query> for props that need the same tool+args.
+${commonRules}`;
+/** Arm B — inline tool references (Experiment 1). */
+export const INLINE_ARM_SYSTEM = `${preamble}
+
+DATA (inline references):
+- Reference a tool's data directly in a prop by CALLING it: rows={invoices.list({status:"overdue"}).data} and value={invoices.list({status:"overdue"}).totalCents}. The compiler mints the query and dedupes by tool+args — reuse the SAME call (same tool, same args) wherever you need that data, and it fetches once. Do NOT write <Query> declarations. After the call you may read a field path (.data, .totalCents, .data.0.id).
+${commonRules}`;
+/** Fetch-then-generate phase-2 system (Experiment 3, Arm B): same as Query arm
+ *  but the generator additionally sees real fetched samples. */
+export const buildFetchAwareSystem = (fetched) => `${QUERY_ARM_SYSTEM}
+
+FETCHED DATA (already read for you — bind to these exact fields and formats; the counts and sample rows are real):
+${fetched}`;
+export const userTask = (prompt) => `USER_REQUEST: ${prompt}`;
+/** Experiment 2 Arm B — build the app by calling builder tools (one turn). */
+export const FORK_SYSTEM = `You are the Vendo app generation engine. You BUILD the app by CALLING the builder tools provided — do NOT write markup or prose. Each call returns a short acknowledgement; keep calling tools until the whole app is composed, then call finish.
+
+BUILD ORDER:
+1. First call set_query once per distinct (tool, args) you need. Bindings reference a query by its name.
+2. Then compose the tree top-to-bottom: begin_region to open a layout container (Stack/Row/Grid/Card/Surface), place_<Component> to add a component to the current container, end_region to close it. Nesting is order-based; the compiler mints ids.
+3. Bind data by passing bindings on a place_ call: {prop: "queryName.field.path"}. Wire actions with actions {onClick:"tool.name"} and, for writes, actionPayloads {onClick:"queryName.data.0.id"}.
+4. Use define_island only as a last resort for a custom visual no component covers.
+5. Call finish when the app is complete.
+
+THINK FIRST: before your first call, reason through the complete layout, which queries you need, and every binding — then emit the calls.
+
+${catalogBlock()}
+
+${toolsBlock()}
+
+${shapesBlock()}
+${commonRules}`;
+//# sourceMappingURL=prompts.js.map

package/dist/bench/prompts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../src/bench/prompts.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAErF;8DAC8D;AAC9D,MAAM,CAAC,MAAM,WAAW,GAAa;IACnC,+FAA+F;IAC/F,+FAA+F;IAC/F,+EAA+E;IAC/E,mGAAmG;IACnG,gFAAgF;IAChF,4EAA4E;IAC5E,gGAAgG;IAChG,oGAAoG;IACpG,gGAAgG;IAChG,uGAAuG;IACvG,sFAAsF;IACtF,+FAA+F;IAC/F,wEAAwE;IACxE,4FAA4F;IAC5F,uEAAuE;IACvE,0GAA0G;IAC1G,uFAAuF;IACvF,wGAAwG;CACzG,CAAC;AAEF;sEACsE;AACtE,MAAM,CAAC,MAAM,gBAAgB,GAAa;IACxC,2EAA2E;IAC3E,oFAAoF;IACpF,6EAA6E;CAC9E,CAAC;AAEF,MAAM,UAAU,GAAG,GAAW,EAAE,CAC9B,qFAAqF;IACrF,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEjF,MAAM,WAAW,GAAG,GAAW,EAAE,CAC/B,0DAA0D;IAC1D,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAE5G,MAAM,YAAY,GAAG,GAAW,EAAE,CAChC,sFAAsF;IACtF,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAE1D,MAAM,WAAW,GAAG;;;;;6IAKyH,CAAC;AAE9I,MAAM,QAAQ,GAAG;;;EAGf,oBAAoB,EAAE;;EAEtB,YAAY,EAAE;;EAEd,UAAU,EAAE;;EAEZ,WAAW,EAAE;;gBAEC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;AAExC,uDAAuD;AACvD,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,QAAQ;;;;EAIzC,WAAW,EAAE,CAAC;AAEhB,qDAAqD;AACrD,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,QAAQ;;;;EAI1C,WAAW,EAAE,CAAC;AAEhB;gEACgE;AAChE,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,OAAe,EAAU,EAAE,CAC/D,GAAG,gBAAgB;;;EAGnB,OAAO,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,iBAAiB,MAAM,EAAE,CAAC;AAE9E,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG;;;;;;;;;;;EAWzB,YAAY,EAAE;;EAEd,UAAU,EAAE;;EAEZ,WAAW,EAAE;EACb,WAAW,EAAE,CAAC"}

package/dist/bench/report.d.ts

@@ -0,0 +1,43 @@
+import type { WireMetrics } from "./metrics.js";
+import type { Judgement } from "./judge.js";
+export interface Sample {
+    prompt: string;
+    wire: string;
+    inputTokens: number;
+    outputTokens: number;
+    ms: number;
+    genError?: string;
+    metrics: WireMetrics;
+    judge: Judgement;
+}
+export interface ArmSummary {
+    arm: string;
+    n: number;
+    compileOkRate: number;
+    meanRefErrors: number;
+    refErrorFreeRate: number;
+    meanBindingShapeErrors: number;
+    declaredUnusedRate: number;
+    formatMissRate: number;
+    meanFormatMiss: number;
+    fabricationRate: number;
+    answersAskRate: number;
+    meanQuality: number;
+    qualityStdev: number;
+    p50OutputTokens: number;
+    p50LatencyMs: number;
+    errorRate: number;
+}
+export declare const summarize: (arm: string, samples: Sample[]) => ArmSummary;
+/** Welch-style significance of the quality-mean difference between two arms.
+ *  Returns |difference| and whether it exceeds the combined standard error ×2
+ *  (~95%). A coarse but honest "outside noise?" gate for small n. */
+export declare const qualityDiffOutsideNoise: (a: Sample[], b: Sample[]) => {
+    diff: number;
+    se: number;
+    outside: boolean;
+};
+export declare const armTableRows: (rows: ArmSummary[]) => string;
+export declare const RAW_DIR: string;
+export declare const writeRaw: (name: string, data: unknown) => string;
+//# sourceMappingURL=report.d.ts.map

package/dist/bench/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/bench/report.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,WAAW,MAAM;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,CAAC,EAAE,MAAM,CAAC;IACV,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAeD,eAAO,MAAM,SAAS,GAAI,KAAK,MAAM,EAAE,SAAS,MAAM,EAAE,KAAG,UAoB1D,CAAC;AAEF;;qEAEqE;AACrE,eAAO,MAAM,uBAAuB,GAAI,GAAG,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,KAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAQ9G,CAAC;AAKF,eAAO,MAAM,YAAY,GAAI,MAAM,UAAU,EAAE,KAAG,MAQjD,CAAC;AAIF,eAAO,MAAM,OAAO,QAA2D,CAAC;AAEhF,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,MAAM,OAAO,KAAG,MAKtD,CAAC"}

package/dist/bench/report.js

@@ -0,0 +1,74 @@
+/**
+ * W1-bench (docs/verification/w1-bench) — per-arm aggregation, significance,
+ * and markdown/JSON artifact writing. Raw per-sample records are persisted so
+ * every number in VERDICTS.md is reproducible.
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0);
+const rate = (xs) => (xs.length ? xs.filter(Boolean).length / xs.length : 0);
+const stdev = (xs) => {
+    if (xs.length < 2)
+        return 0;
+    const m = mean(xs);
+    return Math.sqrt(mean(xs.map((x) => (x - m) ** 2)));
+};
+const p50 = (xs) => {
+    if (!xs.length)
+        return 0;
+    const s = [...xs].sort((a, b) => a - b);
+    return s[Math.floor(s.length / 2)];
+};
+export const summarize = (arm, samples) => {
+    const ok = samples.filter((s) => !s.genError);
+    return {
+        arm,
+        n: samples.length,
+        compileOkRate: rate(ok.map((s) => s.metrics.compileOk)),
+        meanRefErrors: mean(ok.map((s) => s.metrics.refErrors)),
+        refErrorFreeRate: rate(ok.map((s) => s.metrics.refErrors === 0 && s.metrics.compileOk)),
+        meanBindingShapeErrors: mean(ok.map((s) => s.metrics.bindingShapeErrors)),
+        declaredUnusedRate: rate(ok.map((s) => s.metrics.declaredButUnused > 0)),
+        formatMissRate: rate(ok.map((s) => s.metrics.formatMiss > 0)),
+        meanFormatMiss: mean(ok.map((s) => s.metrics.formatMiss)),
+        fabricationRate: rate(ok.map((s) => s.judge.fabricated)),
+        answersAskRate: rate(ok.map((s) => s.judge.answersAsk)),
+        meanQuality: mean(ok.map((s) => s.judge.quality)),
+        qualityStdev: stdev(ok.map((s) => s.judge.quality)),
+        p50OutputTokens: p50(ok.map((s) => s.outputTokens)),
+        p50LatencyMs: p50(ok.map((s) => s.ms)),
+        errorRate: rate(samples.map((s) => Boolean(s.genError))),
+    };
+};
+/** Welch-style significance of the quality-mean difference between two arms.
+ *  Returns |difference| and whether it exceeds the combined standard error ×2
+ *  (~95%). A coarse but honest "outside noise?" gate for small n. */
+export const qualityDiffOutsideNoise = (a, b) => {
+    const qa = a.filter((s) => !s.genError).map((s) => s.judge.quality);
+    const qb = b.filter((s) => !s.genError).map((s) => s.judge.quality);
+    const diff = mean(qa) - mean(qb);
+    const va = qa.length > 1 ? stdev(qa) ** 2 / qa.length : 0;
+    const vb = qb.length > 1 ? stdev(qb) ** 2 / qb.length : 0;
+    const se = Math.sqrt(va + vb);
+    return { diff, se, outside: Math.abs(diff) > 2 * se && se > 0 };
+};
+const pct = (x) => `${(x * 100).toFixed(0)}%`;
+const f2 = (x) => x.toFixed(2);
+export const armTableRows = (rows) => {
+    const header = `| arm | n | compile-ok | ref-error-free | mean ref-err | mean binding-err | declared-unused | format-miss | fabrication | answers-ask | mean quality (sd) | p50 out-tok | p50 latency |\n|---|---|---|---|---|---|---|---|---|---|---|---|---|`;
+    const body = rows
+        .map((r) => `| ${r.arm} | ${r.n} | ${pct(r.compileOkRate)} | ${pct(r.refErrorFreeRate)} | ${f2(r.meanRefErrors)} | ${f2(r.meanBindingShapeErrors)} | ${pct(r.declaredUnusedRate)} | ${pct(r.formatMissRate)} | ${pct(r.fabricationRate)} | ${pct(r.answersAskRate)} | ${f2(r.meanQuality)} (${f2(r.qualityStdev)}) | ${r.p50OutputTokens} | ${(r.p50LatencyMs / 1000).toFixed(1)}s |`)
+        .join("\n");
+    return `${header}\n${body}`;
+};
+const HERE = dirname(fileURLToPath(import.meta.url));
+// packages/apps/src/bench -> repo root
+export const RAW_DIR = join(HERE, "../../../../docs/verification/w1-bench/raw");
+export const writeRaw = (name, data) => {
+    mkdirSync(RAW_DIR, { recursive: true });
+    const path = join(RAW_DIR, name);
+    writeFileSync(path, JSON.stringify(data, null, 2));
+    return path;
+};
+//# sourceMappingURL=report.js.map

package/dist/bench/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/bench/report.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAkCzC,MAAM,IAAI,GAAG,CAAC,EAAY,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3F,MAAM,IAAI,GAAG,CAAC,EAAa,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACxF,MAAM,KAAK,GAAG,CAAC,EAAY,EAAE,EAAE;IAC7B,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACnB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC,CAAC;AACF,MAAM,GAAG,GAAG,CAAC,EAAY,EAAE,EAAE;IAC3B,IAAI,CAAC,EAAE,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAE,CAAC;AACtC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,OAAiB,EAAc,EAAE;IACtE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9C,OAAO;QACL,GAAG;QACH,CAAC,EAAE,OAAO,CAAC,MAAM;QACjB,aAAa,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACvD,aAAa,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACvD,gBAAgB,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACvF,sBAAsB,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACzE,kBAAkB,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,GAAG,CAAC,CAAC,CAAC;QACxE,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAC7D,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACzD,eAAe,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACxD,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACvD,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjD,YAAY,EAAE,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnD,eAAe,EAAE,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACnD,YAAY,EAAE,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;KACzD,CAAC;AACJ,CAAC,CAAC;AAEF;;qEAEqE;AACrE,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAW,EAAE,CAAW,EAAkD,EAAE;IAClH,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACpE,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACjC,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9B,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;AAClE,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACtD,MAAM,EAAE,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAEvC,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAkB,EAAU,EAAE;IACzD,MAAM,MAAM,GAAG,gPAAgP,CAAC;IAChQ,MAAM,IAAI,GAAG,IAAI;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACT,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,eAAe,MAAM,CAAC,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAC1W;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,OAAO,GAAG,MAAM,KAAK,IAAI,EAAE,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACrD,uCAAuC;AACvC,MAAM,CAAC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,4CAA4C,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,IAAa,EAAU,EAAE;IAC9D,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACjC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACnD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC"}

package/dist/bench/runner.d.ts

@@ -0,0 +1,15 @@
+/**
+ * W1-bench (docs/verification/w1-bench) — run one arm: generate a wire per
+ * prompt (bounded concurrency), compile+score it, and judge it. Pure glue over
+ * client/metrics/judge so each experiment file stays declarative.
+ */
+import { type GenOptions } from "./client.js";
+import type { Sample } from "./report.js";
+export interface ArmConfig {
+    system: string | ((prompt: string) => string);
+    inlineRefs?: boolean;
+    gen?: GenOptions;
+    concurrency?: number;
+}
+export declare const runArm: (prompts: string[], cfg: ArmConfig) => Promise<Sample[]>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/bench/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/bench/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAsB,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAIlE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC;IAC9C,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,eAAO,MAAM,MAAM,GAAU,SAAS,MAAM,EAAE,EAAE,KAAK,SAAS,KAAG,OAAO,CAAC,MAAM,EAAE,CAgB7E,CAAC"}

package/dist/bench/runner.js

@@ -0,0 +1,26 @@
+/**
+ * W1-bench (docs/verification/w1-bench) — run one arm: generate a wire per
+ * prompt (bounded concurrency), compile+score it, and judge it. Pure glue over
+ * client/metrics/judge so each experiment file stays declarative.
+ */
+import { generateWire, pool } from "./client.js";
+import { computeWireMetrics } from "./metrics.js";
+import { judge } from "./judge.js";
+import { MAPLE_TOOL_SHAPES } from "./fixtures.js";
+export const runArm = async (prompts, cfg) => pool(prompts, cfg.concurrency ?? 4, async (prompt) => {
+    const system = typeof cfg.system === "function" ? cfg.system(prompt) : cfg.system;
+    const g = await generateWire(system, `USER_REQUEST: ${prompt}`, cfg.gen ?? {});
+    const metrics = computeWireMetrics(g.wire, MAPLE_TOOL_SHAPES, { inlineRefs: cfg.inlineRefs });
+    const j = await judge(prompt, g.wire);
+    return {
+        prompt,
+        wire: g.wire,
+        inputTokens: g.inputTokens,
+        outputTokens: g.outputTokens,
+        ms: g.ms,
+        genError: g.error,
+        metrics,
+        judge: j,
+    };
+});
+//# sourceMappingURL=runner.js.map

package/dist/bench/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/bench/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,YAAY,EAAE,IAAI,EAAmB,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAUlD,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,EAAE,OAAiB,EAAE,GAAc,EAAqB,EAAE,CACnF,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAmB,EAAE;IACpE,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;IAClF,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,iBAAiB,MAAM,EAAE,EAAE,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,kBAAkB,CAAC,CAAC,CAAC,IAAI,EAAE,iBAAiB,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;IAC9F,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACtC,OAAO;QACL,MAAM;QACN,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,KAAK;QACjB,OAAO;QACP,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC,CAAC,CAAC"}