@adia-ai/gen-ui 0.8.37

package/compose/strategies/zettel/chunk-synthesizer.js

@@ -0,0 +1,458 @@
+/**
+ * Chunk-aware composition synthesizer.
+ *
+ * Mirrors the existing `synthesizer.js` (fragment-graph) but operates on the
+ * gen-UI training-chunk corpus instead of the fragment catalog. When pure
+ * retrieval can't surface a strong match for an intent, this asks the LLM to
+ * mix-and-match: pick a page-kind chunk, then bind block/panel chunks to its
+ * slots. Validator checks slot names + chunk-kind contracts before composing.
+ *
+ * Two-tier behavior:
+ *   1. Retrieval first — searchChunks returns a strong match (score > threshold)
+ *      → return that chunk directly. Fast path; no LLM call.
+ *   2. Synthesis fallback — retrieval is weak → invoke the LLM to compose
+ *      from the catalog. This is the creative mix-and-match path.
+ *
+ * Token-budget mitigation: pre-search filters the catalog to ~30 candidates
+ * before the LLM sees them. Mirrors the fragment-synthesizer technique.
+ *
+ * Plan: .claude/docs/plans/training-pipeline-chunk-harvest-2026-04-27.md (Phase C.2).
+ */
+
+import {
+  getChunk,
+  searchChunks,
+  searchChunksAsync,
+  listChunksByKind,
+  getAllChunks,
+  resolveNestedChunks,
+} from '../../../corpus/scripts/chunk-library.js';
+import { composeFromPlan, validatePlan } from './chunk-composer.js';
+import { transpileHTML } from '../../transpiler/transpiler.js';
+
+// TKT-0009: composeFromIntent used to return only a raw HTML string,
+// which every caller wrapped in a single node carrying an UNREGISTERED
+// A2UI type (`component: 'article'`) — meaning full schema/catalog
+// validation against it always reported invalid, regardless of the
+// composed content's actual quality (registry.has('article') is false).
+// Ratified direction: reuse the SAME transpileHTML() infrastructure the
+// docs-transpiler and harvest-chunks.mjs already run at scale, rather
+// than inventing a new passthrough construct — this makes every
+// composeFromIntent() result a REAL component graph, so it qualifies for
+// the shared validate-and-repair loop like every other engine. A chunk's
+// own precomputed `.template` (from harvest-chunks.mjs) is used directly
+// when available (accurate, no re-transpile cost); multi-instance chunk
+// records don't carry one per-instance, so this falls back to
+// transpiling the HTML on demand either way — every return path leaves
+// with a real template, never just a string.
+async function ensureTemplate(html, precomputedTemplate) {
+  if (Array.isArray(precomputedTemplate) && precomputedTemplate.length) return precomputedTemplate;
+  if (!html) return null;
+  const { messages } = await transpileHTML(html);
+  return messages?.[0]?.components ?? null;
+}
+
+const STRONG_RETRIEVAL_SCORE = 8;     // search-score threshold for fast path
+const PRE_SEARCH_LIMIT = 30;           // chunks shown to the LLM in the prompt
+const DEFAULT_MAX_ATTEMPTS = 2;
+// Scope-drift gate: composed HTML's component count vs. the sum of bound
+// chunks' component counts. A multiplier > SCOPE_DRIFT_RATIO trips a warning
+// + auto-fires a `scope-drift` issue. Floor prevents false positives on
+// small UIs where slot-wrapper noise dominates.
+export const SCOPE_DRIFT_RATIO = 1.5;
+export const SCOPE_DRIFT_MIN_ACTUAL = 20;
+
+const SYSTEM_PROMPT = `You compose web-app pages by binding training chunks into named slots.
+
+## YOUR TASK
+
+Given a user intent and a catalog of chunks, return a JSON object that:
+1. SELECTS the most appropriate page-kind shell
+2. BINDS block/panel chunks into its slots
+
+## STEP 1 — SELECT A PAGE SHELL
+
+Page shells (kind=page or kind=panel) define the layout topology. Match the
+intent domain to the right shell — NEVER default to dashboard-admin-page for
+non-dashboard intents:
+
+| Shell | Domain | Slots |
+|-------|--------|-------|
+| dashboard-admin-page | Admin/analytics dashboards | page-header, page-content |
+| settings-page-shell | Settings, preferences, configuration | page-header, page-tabs, page-content |
+| form-page-shell | Auth forms, sign-in, sign-up, profile | page-header, form-content, page-footer |
+| marketing-page-shell | Landing pages, heroes, features, CTAs | hero, features, cta |
+| error-page-shell | Error states (404, 500, permission denied) | error-content, navigation |
+| editor-page-shell | Split-pane editors (code + preview) | editor-pane, preview-pane |
+| onb-step-shell | Onboarding wizards, multi-step flows | page-story, page-header, page-content, page-footer |
+| reg-step-shell | Registration flows, multi-step sign-up | page-story, page-header, page-content, page-footer |
+
+Selection rule: the shell whose DOMAIN matches the intent keywords wins.
+
+## STEP 2 — SELECT BLOCKS TO FILL SLOTS
+
+- Search the block catalog for chunks whose name contains intent keywords.
+- Prefer blocks whose primary tag matches the desired component (e.g.
+  "stat-ui" for stat cards, "chart-ui" for charts).
+- If a slot is optional and no matching block exists, OMIT it rather
+  than forcing a random block.
+- Never invent chunk names. Every bound name must appear in the catalog.
+
+## STEP 3 — RETURN JSON
+
+Return ONLY a JSON object shaped exactly like:
+{
+  "page": "<name of page-kind chunk>",
+  "slot_bindings": {
+    "<slot-name>": "<bound chunk name>"
+    OR
+    "<slot-name>": ["<chunk1>", "<chunk2>"]
+  }
+}
+
+Rules:
+- The "page" must be a chunk with kind="page" or kind="panel".
+- Every slot key must be declared by the chosen page shell.
+- Every bound value must be a real chunk name from the catalog.
+- Return valid JSON. No prose, no comments, no markdown fences. Begin with "{".
+`;
+
+function buildCatalogSummary(chunks) {
+  return chunks.map((c) => ({
+    name: c.name,
+    kind: c.kind,
+    primary: c.primary || c.instances?.[0]?.primary,
+    slots: (c.slots || c.instances?.[0]?.slots || []).map((s) => s.name),
+  }));
+}
+
+function buildExamples() {
+  // Four canonical examples spanning different domains. Diverse examples
+  // prevent the LLM from defaulting to dashboard-shaped output.
+  return [
+    {
+      intent: 'admin dashboard with KPIs and a conversion funnel',
+      output: {
+        page: 'dashboard-admin-page',
+        slot_bindings: {
+          'page-header': 'dashboard-page-header',
+          'page-content': ['dashboard-overview-panel', 'dashboard-funnel'],
+        },
+      },
+    },
+    {
+      intent: 'sign-in form with email and password',
+      output: {
+        page: 'form-page-shell',
+        slot_bindings: {
+          'page-header': 'auth-signin-card-password',
+          'form-content': 'auth-email-entry',
+        },
+      },
+    },
+    {
+      intent: 'settings page with tabs for general and billing',
+      output: {
+        page: 'settings-page-shell',
+        slot_bindings: {
+          'page-header': 'settings-general-form',
+          'page-tabs': 'check-combinations-settings-group',
+          'page-content': 'settings-general-form',
+        },
+      },
+    },
+    {
+      intent: 'marketing landing hero with feature cards',
+      output: {
+        page: 'marketing-page-shell',
+        slot_bindings: {
+          hero: 'hero-cta-simple',
+          features: 'empty-state-action',
+          cta: 'button-primary',
+        },
+      },
+    },
+  ];
+}
+
+function buildUserPrompt(intent, catalog, examples) {
+  const lines = [];
+  lines.push(`Intent: ${intent}`);
+  lines.push('');
+  lines.push('## Available chunks');
+  lines.push('');
+  for (const c of catalog) {
+    const slots = c.slots?.length ? ` slots=[${c.slots.join(', ')}]` : '';
+    lines.push(`- ${c.name} (kind=${c.kind}, primary=${c.primary})${slots}`);
+  }
+  lines.push('');
+  lines.push('## Example bindings');
+  lines.push('');
+  for (const ex of examples) {
+    lines.push(`Intent: ${ex.intent}`);
+    lines.push('Output:');
+    lines.push(JSON.stringify(ex.output, null, 2));
+    lines.push('');
+  }
+  lines.push('## Your turn');
+  lines.push('');
+  lines.push('Return JSON binding for the intent above. JSON only.');
+  return lines.join('\n');
+}
+
+function extractJSON(raw) {
+  if (!raw || typeof raw !== 'string') return null;
+  const trimmed = raw.trim();
+  // Strip markdown fences if present
+  const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
+  const candidate = fenced ? fenced[1].trim() : trimmed;
+  try {
+    return JSON.parse(candidate);
+  } catch {
+    // Find the first { ... } balanced object
+    const m = candidate.match(/\{[\s\S]*\}/);
+    if (!m) return null;
+    try { return JSON.parse(m[0]); } catch { return null; }
+  }
+}
+
+/**
+ * Try retrieval-first; fall back to synthesis if retrieval is weak or missing.
+ *
+ * @param {object} opts
+ * @param {string} opts.intent
+ * @param {object} opts.llmAdapter — { complete: async ({ messages, systemPrompt }) => { content|text } }
+ * @param {number} [opts.maxAttempts=2]
+ * @returns {Promise<{ html: string, template: object[]|null, plan: object|null, source: 'retrieval'|'synthesis', score?: number, warnings: string[], synthesis?: object }>}
+ *   `template` (TKT-0009) is a real A2UI component array — derived from the
+ *   chunk's own precomputed template when available, else transpiled from
+ *   `html` on demand — so callers get a validatable component graph, not
+ *   just an HTML string to wrap opaquely.
+ */
+export async function composeFromIntent({ intent, llmAdapter, maxAttempts = DEFAULT_MAX_ATTEMPTS }) {
+  // Tier 1 — retrieval. Try semantic-blended hit first; fall back to keyword
+  // when embeddings are unavailable (no chunk-embeddings.json or no API key).
+  //
+  // Restricted to kind=block: page/panel chunks are SKELETONS that need
+  // slot-binding composition (Tier 2 handles them). Returning a skeleton
+  // Fast path — deterministic keyword-first; embeddings as tie-breaker only.
+  // Embeddings can drift (e.g. "pane" matching "panel" boosting unrelated
+  // chunks). Sync keyword search is stable and keyword-discoverability is
+  // the baseline guarantee. We run both and prefer the sync result when
+  // it meets the threshold; only use async when sync is weak but async
+  // is strong (e.g. semantic match on content not captured by keywords).
+  const syncHits = searchChunks(intent, { kind: 'block', limit: 5 });
+  const asyncHits = await searchChunksAsync(intent, { kind: 'block', limit: 5 });
+
+  const syncTop = syncHits[0];
+  const asyncTop = asyncHits[0];
+  const useSync = syncTop && syncTop.score >= STRONG_RETRIEVAL_SCORE;
+  const useAsync = !useSync && asyncTop && asyncTop.score >= STRONG_RETRIEVAL_SCORE;
+
+  if (useSync || useAsync) {
+    const hit = useSync ? syncTop : asyncTop;
+    const top = getChunk(hit.name);
+    // A block chunk harvested from a page with nested markers is itself a
+    // skeleton — its body is `<!-- nested: … -->` comments. Returning that
+    // verbatim ships an empty shell, so inline the referenced chunks.
+    const html = resolveNestedChunks(top.html || top.instances?.[0]?.html || '', {
+      page: top.page || top.source || top.instances?.[0]?.page,
+    });
+    const scopeDrift = computeScopeDrift(html, [top, ...(top.nested || []).map(getChunk)]);
+    return {
+      html,
+      template: await ensureTemplate(html, top.template),
+      // gh#832 — a retrieval hit IS a composition, so it carries a plan like
+      // every synthesis result does: the single-chunk plan that materializes
+      // back to exactly this HTML. It used to be `null`, which made both
+      // tiers' outputs structurally different and left `refineFromIntent` /
+      // `applyOps` with nothing to mutate — refine silently no-opped on the
+      // entire Tier-1 path, in the product as well as the eval.
+      plan: { page: top.name, slot_bindings: {} },
+      source: 'retrieval',
+      score: hit.score,
+      cosineScore: hit.cosineScore,
+      warnings: scopeDrift.drift
+        ? [`scope drift: ${scopeDrift.actual} components in HTML vs ${scopeDrift.expected} in bound chunk (ratio ${scopeDrift.ratio.toFixed(2)}×)`]
+        : [],
+      scopeDrift,
+    };
+  }
+
+  // Tier 2 — synthesis. Need an LLM adapter.
+  if (!llmAdapter?.complete) {
+    return {
+      html: null,
+      plan: null,
+      source: 'synthesis',
+      warnings: ['retrieval was weak and no llmAdapter provided — cannot synthesize'],
+    };
+  }
+
+  // Pre-search the catalog to keep the prompt token-budget bounded.
+  // Strategy: retrieve top N by semantic-blended search; ALWAYS include all
+  // page-kind chunks (small set, the LLM needs to see what shells are
+  // available). Block pre-search uses embeddings when available.
+  const pageChunks = listChunksByKind('page');
+  const panelChunks = listChunksByKind('panel');
+  const blockHits = await searchChunksAsync(intent, {
+    kind: 'block',
+    limit: PRE_SEARCH_LIMIT - pageChunks.length - panelChunks.length,
+  });
+  const blockChunks = blockHits.map((h) => getChunk(h.name)).filter(Boolean);
+
+  const filtered = [
+    ...pageChunks,
+    ...panelChunks,
+    ...blockChunks,
+  ];
+  const catalog = buildCatalogSummary(filtered);
+  const examples = buildExamples();
+
+  const userPrompt = buildUserPrompt(intent, catalog, examples);
+  let lastError = null;
+  const attempts = [];
+
+  // Trace: snapshot of the retrieval log for the issue-reporter to surface
+  // verbatim on bug tickets. Recorded once before the retry loop so it
+  // describes what the LLM actually saw.
+  const tier1HitList = useSync ? syncHits : (useAsync ? asyncHits : []);
+  const retrievalTrace = {
+    tier1Hits: tier1HitList.slice(0, 5).map((h) => ({
+      name: h.name,
+      score: Number(h.score.toFixed(3)),
+      kind: h.kind,
+      cosineScore: h.cosineScore != null ? Number(h.cosineScore.toFixed(3)) : null,
+    })),
+    tier1Threshold: STRONG_RETRIEVAL_SCORE,
+    tier1Pass: tier1HitList.length > 0 && tier1HitList[0].score >= STRONG_RETRIEVAL_SCORE,
+    catalogSize: filtered.length,
+    catalogPageNames: pageChunks.map((c) => c.name),
+    catalogPanelNames: panelChunks.map((c) => c.name),
+    catalogBlockTopN: blockHits.slice(0, 10).map((h) => ({
+      name: h.name,
+      score: Number(h.score.toFixed(3)),
+    })),
+    userPromptChars: userPrompt.length,
+    systemPromptChars: SYSTEM_PROMPT.length,
+  };
+
+  for (let i = 0; i < maxAttempts; i++) {
+    const retryNudge = lastError
+      ? `\n\nPREVIOUS ATTEMPT FAILED: ${lastError}. Return ONLY a JSON object shaped as { "page": "...", "slot_bindings": { ... } }. No prose, no questions.`
+      : '';
+    const messages = [{ role: 'user', content: userPrompt + retryNudge }];
+    const response = await llmAdapter.complete({ messages, systemPrompt: SYSTEM_PROMPT });
+    const raw = response?.content || response?.text || (typeof response === 'string' ? response : '');
+    attempts.push({ attempt: i + 1, raw });
+
+    const plan = extractJSON(raw);
+    if (!plan || !plan.page || !plan.slot_bindings) {
+      lastError = 'Response was not valid JSON with { page, slot_bindings } fields.';
+      continue;
+    }
+
+    const validation = validatePlan(plan);
+    if (!validation.ok) {
+      lastError = validation.errors.slice(0, 3).join('; ');
+      continue;
+    }
+
+    const composed = composeFromPlan(plan);
+    if (!composed.html) {
+      lastError = composed.warnings.join('; ') || 'composer returned null html';
+      continue;
+    }
+
+    // Compute scope-drift signal: composed envelope vs sum of bound chunks.
+    // Bound chunks = page chunk + every block/panel resolved into a slot.
+    const boundNames = new Set([plan.page]);
+    for (const v of Object.values(plan.slot_bindings || {})) {
+      const arr = Array.isArray(v) ? v : [v];
+      for (const n of arr) boundNames.add(n);
+    }
+    const boundChunks = [...boundNames].map((n) => getChunk(n)).filter(Boolean);
+    const scopeDrift = computeScopeDrift(composed.html, boundChunks);
+    const driftWarnings = scopeDrift.drift
+      ? [`scope drift: ${scopeDrift.actual} components in composed HTML vs ${scopeDrift.expected} in bound chunks (ratio ${scopeDrift.ratio.toFixed(2)}× exceeds gate ${SCOPE_DRIFT_RATIO}×)`]
+      : [];
+
+    return {
+      html: composed.html,
+      template: await ensureTemplate(composed.html, null),
+      plan,
+      source: 'synthesis',
+      warnings: [...composed.warnings, ...driftWarnings],
+      scopeDrift,
+      synthesis: {
+        attempts: i + 1,
+        attemptsLog: attempts,
+        validation,
+        retrievalTrace,
+        userPrompt,
+        systemPromptHash: hashString(SYSTEM_PROMPT),
+      },
+    };
+  }
+
+  return {
+    html: null,
+    plan: null,
+    source: 'synthesis',
+    warnings: [`synthesis failed after ${maxAttempts} attempts: ${lastError}`],
+    synthesis: {
+      attempts: maxAttempts,
+      attemptsLog: attempts,
+      retrievalTrace,
+      userPrompt,
+      systemPromptHash: hashString(SYSTEM_PROMPT),
+    },
+  };
+}
+
+// Cheap non-crypto hash for prompt-version fingerprinting in tickets.
+function hashString(s) {
+  let h = 5381;
+  for (let i = 0; i < s.length; i++) h = ((h * 33) ^ s.charCodeAt(i)) >>> 0;
+  return 'h' + h.toString(36);
+}
+
+/**
+ * Heuristic component count — number of opening tags in the HTML.
+ * Loose but stable proxy for A2UI envelope size; used by the scope-drift
+ * gate (above) and by issue-reporter's ticket-rendering counter.
+ */
+function countComponents(html) {
+  if (typeof html !== 'string') return 0;
+  return (html.match(/<[a-z][a-z0-9-]*/g) || []).length;
+}
+
+/**
+ * Compute the scope-drift signal for a composed result.
+ *
+ * Inputs: the final HTML string + the chunk records that fed into it
+ * (page chunk for Tier-2; the single block for Tier-1). Sums the bound
+ * chunks' component counts to get an "expected envelope," compares it to
+ * the actual composed count, and reports the ratio.
+ *
+ * Returns { actual, expected, ratio, drift } where `drift` is true when
+ * actual exceeds SCOPE_DRIFT_RATIO × expected AND actual ≥ SCOPE_DRIFT_MIN_ACTUAL.
+ */
+export function computeScopeDrift(html, boundChunks) {
+  const actual = countComponents(html);
+  let expected = 0;
+  for (const c of boundChunks) {
+    if (!c) continue;
+    const chunkHtml = c.html || c.instances?.[0]?.html;
+    expected += countComponents(chunkHtml);
+  }
+  // Guard against zero-expected: when bound chunks have no HTML accounted for
+  // (rare; should only happen in malformed corpora), skip the gate rather
+  // than report infinite drift.
+  if (expected === 0) {
+    return { actual, expected, ratio: null, drift: false };
+  }
+  const ratio = actual / expected;
+  const drift = actual >= SCOPE_DRIFT_MIN_ACTUAL && ratio > SCOPE_DRIFT_RATIO;
+  return { actual, expected, ratio, drift };
+}

package/compose/strategies/zettel/composer.js

@@ -0,0 +1,78 @@
+/**
+ * Composer — passes A2UI composition templates straight through.
+ *
+ * History: this file used to expand `$fragment` refs in compositions
+ * via a slot-binding mechanism (zettel-style fragment graph). As of
+ * §37 (2026-05-12) fragments are retired — all compositions are
+ * pre-inlined into flat A2UI templates by /tmp/inline-compositions.py.
+ * `resolveComposition` is now a defensive copy + strip pass; any
+ * lingering `$fragment` ref is treated as a placeholder (the LLM /
+ * renderer will fail noisily, surfacing corpus drift).
+ *
+ * Kept as a thin wrapper rather than deleted because zettel's
+ * generator-adapter still calls `resolveComposition(comp)` —
+ * downstream rename is a follow-up arc.
+ */
+
+const ID_PREFIX_SEPARATOR = '--';
+
+/**
+ * Return a defensive copy of a composition's flat template.
+ *
+ * Compositions are pre-inlined, so there's nothing to expand. We
+ * still strip zettel-era fields that may linger in the corpus —
+ * future-proof against accidental re-introduction.
+ *
+ * @param {object} composition - { template: [...] }
+ * @returns {Array<object>}
+ */
+export function resolveComposition(composition) {
+  const template = composition?.template || [];
+  const out = [];
+  for (const node of template) {
+    if (!node || typeof node !== 'object') continue;
+    if (node.$fragment) {
+      // Stale $fragment refs shouldn't exist after the §37 inline pass.
+      // Surface them as a visible placeholder so corpus drift is easy
+      // to spot at render time.
+      out.push({
+        id: node.id || `stale-${node.$fragment}`,
+        component: 'Text',
+        textContent: `⚠ stale $fragment ref (compositions are pre-inlined): ${node.$fragment}`,
+      });
+      continue;
+    }
+    const {
+      // Strip composition-only fields that never reach the renderer
+      $fragment, bindings, prependChildren, appendChildren,
+      ...rest
+    } = node;
+    out.push({ ...rest });
+  }
+  return out;
+}
+
+/**
+ * Convert a flat A2UI template into `beginComponent` messages for the
+ * validator / renderer. Strips composition-era fields.
+ */
+export function templateToMessages(template) {
+  return template.map((node) => {
+    const {
+      id, component, children,
+      $fragment, bindings,
+      prependChildren, appendChildren,
+      ...rest
+    } = node;
+    return {
+      messageType: 'beginComponent',
+      componentId: id,
+      componentType: component,
+      children: children || [],
+      ...rest,
+    };
+  });
+}
+
+// Re-export the separator for any downstream code that still uses it.
+export { ID_PREFIX_SEPARATOR };