@adia-ai/a2ui-retrieval 0.0.1

package/dialog-recorder.js

@@ -0,0 +1,179 @@
+/**
+ * Dialog Recorder — write every generation turn to disk for debugging,
+ * regression analysis, and training-data bootstrapping.
+ *
+ * Output: logs/dialogs/<sessionId>/<NNN>-<turnId>.json
+ *   - <sessionId>: storeId / executionId of the multi-turn session, or
+ *                  'standalone-<isodate>' for one-shot generations.
+ *   - <NNN>:        zero-padded turn index within the session (000, 001, …)
+ *   - <turnId>:     short slug from the intent + a random suffix for uniqueness
+ *
+ * Each file contains a flat JSON object with everything the pipeline knew at
+ * generation time: intent, mode, model, the analyzer's structured signals,
+ * pattern matches, the FULL system prompt, the raw LLM response, parsed A2UI
+ * messages, validation, drift, suggestions, and timing/token telemetry.
+ *
+ * Gated by ADIA_LOG_DIALOGS=1 — opt-in, zero overhead when disabled. Browser
+ * environment is also a no-op (no fs); only Node-side generation paths log.
+ *
+ * Use cases:
+ *   - replay a single bad turn locally without paying the LLM cost again
+ *   - diff two turns visually after wiring the headless renderer
+ *   - bootstrap an eval set from real dogfood — every turn becomes a labeled example
+ *   - regression detection on prompt / corpus / catalog changes
+ */
+
+const IS_NODE = typeof process !== 'undefined' && !!process.versions?.node;
+const ENABLED = IS_NODE && (process.env.ADIA_LOG_DIALOGS === '1' || process.env.ADIA_LOG_DIALOGS === 'true');
+
+let _fs = null;
+let _path = null;
+let _url = null;
+let _logsRoot = null;
+
+async function _ensureModules() {
+  if (_fs) return;
+  _fs = await import(/* @vite-ignore */ 'node:fs/promises');
+  _path = await import(/* @vite-ignore */ 'node:path');
+  _url = await import(/* @vite-ignore */ 'node:url');
+  // logs/ lives at the repo root: packages/a2ui/retrieval → up 3 → repo root
+  const __dirname = _path.dirname(_url.fileURLToPath(import.meta.url));
+  _logsRoot = _path.resolve(__dirname, '..', '..', '..', 'logs', 'dialogs');
+}
+
+// In-memory turn counter per session — keeps the on-disk turn ordering correct
+// even when timestamps collide (sub-millisecond turns from automated probes).
+const _turnCounter = new Map();
+// Tracks which sessions we've already written `_session.json` for. Per-process
+// memory; if the process restarts mid-session, the file is rewritten, which is
+// fine — content is idempotent.
+const _sessionMetaWritten = new Set();
+
+/**
+ * Record one generation turn. Safe to call unconditionally — when the env var
+ * is unset, this is a no-op that returns immediately.
+ *
+ * @param {object} record
+ * @param {string} record.sessionId      — multi-turn session handle (storeId / executionId)
+ * @param {string} record.intent         — user's prompt this turn
+ * @param {string} [record.mode]         — instant | pro | thinking | stream
+ * @param {string} [record.engine]       — monolithic | zettel | <custom>
+ * @param {string} [record.model]        — LLM model id
+ * @param {object} [record.analysis]     — output of analyzePrompt() (concepts, steelman, ...)
+ * @param {object} [record.currentCanvas] — { components } or { messages } provided by caller
+ * @param {object[]} [record.patterns]   — patterns retrieved by searchBlocks()
+ * @param {string} [record.systemPrompt] — full system prompt sent to LLM
+ * @param {string} [record.rawLLMResponse] — raw LLM text (pre-parse)
+ * @param {object[]} [record.messages]   — parsed A2UI messages (the result)
+ * @param {object} [record.validation]   — validateSchema() result
+ * @param {object} [record.drift]        — getDriftMetrics() result
+ * @param {string[]} [record.suggestions] — follow-up suggestions
+ * @param {object} [record.timing]       — { totalMs, llmMs, ... }
+ * @param {object} [record.tokens]       — { input, output }
+ * @param {object} [record.engineDebug]  — engine-specific extras (strategy, composition, fragmentsUsed for zettel; raw stage reports, etc.)
+ * @param {boolean} [record.isIteration] — derived from executionId / currentCanvas presence
+ * @returns {Promise<string|null>} the path written, or null when logging is disabled
+ */
+export async function recordTurn(record) {
+  if (!ENABLED) return null;
+
+  try {
+    await _ensureModules();
+
+    const sessionId = record.sessionId || `standalone-${new Date().toISOString().replace(/[:.]/g, '-')}`;
+    const turnIdx = (_turnCounter.get(sessionId) ?? 0);
+    _turnCounter.set(sessionId, turnIdx + 1);
+
+    const slug = String(record.intent || 'turn')
+      .toLowerCase()
+      .replace(/[^a-z0-9]+/g, '-')
+      .replace(/^-+|-+$/g, '')
+      .slice(0, 32) || 'turn';
+    const rand = Math.random().toString(36).slice(2, 6);
+    const fileName = `${String(turnIdx).padStart(3, '0')}-${slug}-${rand}.json`;
+
+    const sessionDir = _path.join(_logsRoot, sessionId);
+    await _fs.mkdir(sessionDir, { recursive: true });
+
+    // Write per-session header on the first turn so a directory listing is
+    // immediately legible — origin intent, model, mode, first-turn analysis,
+    // start time. Future headless renderer reads this for its session list UI.
+    if (!_sessionMetaWritten.has(sessionId) && turnIdx === 0) {
+      _sessionMetaWritten.add(sessionId);
+      const sessionMeta = {
+        sessionId,
+        startedAt: new Date().toISOString(),
+        originIntent: record.intent || null,
+        engine: record.engine || null,
+        mode: record.mode || null,
+        model: record.model || null,
+        // The analyzer's enriched brief + concept tags from turn 0 — these
+        // characterize the session's intent space, not just the latest turn.
+        originAnalysis: record.analysis ? {
+          steelman: record.analysis.steelman || null,
+          concepts: record.analysis.concepts || [],
+          impliedComponents: record.analysis.impliedComponents || [],
+          styleHints: record.analysis.styleHints || [],
+        } : null,
+      };
+      await _fs.writeFile(
+        _path.join(sessionDir, '_session.json'),
+        JSON.stringify(sessionMeta, null, 2) + '\n',
+      );
+    }
+
+    const payload = {
+      // ── Identity ────────────────────────────────────────────────
+      sessionId,
+      turnIndex: turnIdx,
+      timestamp: new Date().toISOString(),
+      isIteration: !!record.isIteration,
+
+      // ── Request ─────────────────────────────────────────────────
+      intent: record.intent,
+      mode: record.mode || null,
+      engine: record.engine || null,
+      model: record.model || null,
+      currentCanvas: record.currentCanvas || null,
+
+      // ── Pipeline-side reasoning ─────────────────────────────────
+      analysis: record.analysis || null,
+      patterns: (record.patterns || []).slice(0, 10).map(p => ({
+        name: p.name,
+        score: p.score ?? p.confidence ?? null,
+        keywords: p.keywords || null,
+      })),
+
+      // ── LLM I/O (the largest fields — keep last for jq usability) ─
+      systemPrompt: record.systemPrompt || null,
+      rawLLMResponse: record.rawLLMResponse || null,
+
+      // ── Result ──────────────────────────────────────────────────
+      messages: record.messages || [],
+      validation: record.validation || null,
+      drift: record.drift || null,
+      suggestions: record.suggestions || [],
+
+      // ── Telemetry ───────────────────────────────────────────────
+      timing: record.timing || null,
+      tokens: record.tokens || null,
+      // Engine-specific extras (zettel: strategy/composition/fragmentsUsed,
+      // monolithic: nothing yet — but reserved). Whatever the engine put on
+      // result._debug above and beyond the standard fields lands here.
+      engineDebug: record.engineDebug || null,
+    };
+
+    const filePath = _path.join(sessionDir, fileName);
+    await _fs.writeFile(filePath, JSON.stringify(payload, null, 2) + '\n');
+    return filePath;
+  } catch (err) {
+    // Logging must NEVER break a generation. Swallow + warn once per turn.
+    console.warn('[dialog-recorder] failed to record turn:', err.message);
+    return null;
+  }
+}
+
+/** True when logging is on. Useful for guarding expensive capture work. */
+export function isRecording() {
+  return ENABLED;
+}

package/domain-router.js

@@ -0,0 +1,172 @@
+/**
+ * Domain Router — Intent-to-domain classification via keyword matching.
+ *
+ * Five domains: forms, data, layout, agent, navigation.
+ * Each has signal keywords weighted for matching confidence.
+ */
+
+const domains = {
+  forms: {
+    keywords: [
+      'login', 'signup', 'sign up', 'register', 'input', 'field', 'form',
+      'validation', 'submit', 'password', 'email', 'checkbox', 'toggle',
+      'radio', 'select', 'dropdown', 'upload', 'otp', 'text field',
+      'date picker', 'slider', 'textarea', 'required', 'label',
+      'settings', 'configure', 'preference', 'margin', 'bleed',
+      'preview', 'approve', 'approval', 'photo', 'design system',
+      'button', 'cancel', 'action', 'reset',
+      'color picker', 'theme', 'swatch', 'two factor', 'authentication',
+      'verification', 'verify', 'subscribe', 'newsletter', 'survey',
+      'rbac', 'role', 'roles', 'permission', 'permissions', 'mfa', 'two-factor',
+      'session', 'sessions', 'device', 'devices', 'revoke', 'signing',
+      'destructive', 'confirm', 'confirmation', 'type to', 'type-to',
+      'mapping', 'mapper', 'csv', 'import',
+    ],
+    components: [
+      'Input', 'CheckBox', 'Toggle', 'Switch', 'Slider', 'Select', 'Radio',
+      'TextArea', 'Upload', 'OtpInput', 'CalendarPicker', 'ColorPicker',
+      'Range', 'Button',
+    ],
+  },
+  data: {
+    keywords: [
+      'table', 'chart', 'stat', 'metric', 'dashboard', 'graph', 'report',
+      'analytics', 'data', 'list', 'grid', 'sort', 'filter', 'paginate',
+      'pagination', 'row', 'column', 'progress', 'sparkline', 'kpi',
+      'kanban', 'board', 'activity', 'feed', 'timeline', 'leaderboard',
+      'inventory', 'project', 'task', 'tasks', 'team', 'calendar',
+      'tracker', 'tracking', 'monitor', 'monitoring', 'weather',
+      'pricing', 'product', 'bookmark', 'music', 'player',
+      'shopping', 'cart', 'order', 'invoice', 'receipt', 'checkout',
+      'inbox', 'notification', 'changelog', 'permission',
+      'cohort', 'retention', 'heatmap', 'funnel', 'conversion',
+      'audit', 'log', 'logs', 'viewer', 'audit log',
+      'integration', 'integrations', 'marketplace',
+      'webhook', 'webhooks', 'endpoint', 'delivery',
+      'usage', 'quota', 'meter', 'limit', 'limits',
+      'incident', 'status page', 'uptime',
+      'release', 'releases', 'release notes',
+      'flag', 'flags', 'feature flag', 'rollout',
+      'api key', 'api keys', 'token', 'tokens',
+      'object inspector', 'inspector', 'record',
+      'virtualized', 'virtual scroll', 'sticky',
+      'column manager', 'reorder',
+      'saved view', 'saved views', 'view',
+      'bulk', 'bulk action', 'selection',
+    ],
+    components: [
+      'Table', 'Chart', 'Stat', 'Progress', 'Timeline', 'List', 'Grid',
+      'Badge', 'Pagination', 'Avatar', 'Skeleton',
+    ],
+  },
+  layout: {
+    keywords: [
+      'page', 'grid', 'sidebar', 'header', 'footer', 'hero', 'section',
+      'column', 'row', 'card', 'panel', 'divider', 'stack', 'block',
+      'container', 'layout', 'spacing', 'gap', 'toolbar', 'image', 'embed',
+      'gallery', 'profile', 'testimonial', 'feature', 'cta', 'landing',
+      'swiper', 'carousel', 'slideshow', 'slides', 'slider',
+      'wizard', 'steps', 'stepper', 'accordion', 'faq',
+      'toast', 'alert', 'banner', 'badge', 'tag', 'code', 'snippet',
+      'skeleton', 'loading', 'empty', 'error', 'onboarding',
+      'avatar', 'popover', 'tooltip', 'modal', 'dialog', 'drawer',
+      'breadcrumb', 'recipe', 'blog', 'post', 'comparison',
+      'three-pane', 'three pane', 'list-detail', 'master-detail',
+      'split pane', 'bento', 'workspace', 'switcher', 'org',
+      'shell', 'app shell',
+    ],
+    components: [
+      'Row', 'Column', 'Grid', 'Card', 'Divider', 'Tabs', 'Tab',
+      'Accordion', 'Toolbar', 'Drawer', 'Modal', 'Pane',
+      'Image', 'Embed', 'Tag', 'Kbd', 'Code', 'Swiper',
+    ],
+  },
+  agent: {
+    keywords: [
+      'chat', 'message', 'stream', 'ai', 'prompt', 'generate', 'conversation',
+      'assistant', 'bot', 'agent', 'llm', 'response', 'streaming', 'a2ui',
+      'surface', 'render', 'dynamic', 'empty', 'error', 'loading',
+    ],
+    components: [
+      'Stream', 'Card', 'Text', 'Button', 'Column', 'Row', 'Alert', 'Toast',
+      'EmptyState', 'Skeleton', 'Tooltip', 'Popover',
+    ],
+  },
+  navigation: {
+    keywords: [
+      'menu', 'nav', 'breadcrumb', 'tab', 'tabs', 'sidebar', 'link', 'route',
+      'router', 'navigate', 'page', 'sitemap', 'command palette', 'command',
+      'segmented', 'breadcrumbs',
+    ],
+    components: [
+      'Nav', 'Breadcrumb', 'Tabs', 'Tab', 'Menu', 'Command',
+      'SegmentedControl', 'Segment', 'Pagination',
+    ],
+  },
+};
+
+/**
+ * Classify an intent string into a domain.
+ *
+ * @param {string} text — Natural language intent
+ * @returns {{ domain: string, confidence: number, matchedSignals: string[], suggestedComponents: string[] }}
+ */
+export function classifyIntent(text) {
+  const lower = text.toLowerCase();
+  const scores = {};
+  const matches = {};
+
+  for (const [domain, config] of Object.entries(domains)) {
+    scores[domain] = 0;
+    matches[domain] = [];
+
+    for (const keyword of config.keywords) {
+      if (lower.includes(keyword)) {
+        scores[domain]++;
+        matches[domain].push(keyword);
+      }
+    }
+  }
+
+  // Find the top domain
+  let bestDomain = 'layout'; // default fallback
+  let bestScore = 0;
+
+  for (const [domain, score] of Object.entries(scores)) {
+    if (score > bestScore) {
+      bestScore = score;
+      bestDomain = domain;
+    }
+  }
+
+  // Confidence: ratio of matched keywords to total keywords in the domain
+  const totalKeywords = domains[bestDomain].keywords.length;
+  const confidence = bestScore > 0
+    ? Math.min(1, bestScore / Math.max(3, totalKeywords * 0.3))
+    : 0;
+
+  return {
+    domain: bestDomain,
+    confidence: Math.round(confidence * 100) / 100,
+    matchedSignals: matches[bestDomain],
+    suggestedComponents: domains[bestDomain].components,
+  };
+}
+
+/**
+ * Get domain configuration (keywords + components) for a given domain.
+ *
+ * @param {string} domain — Domain name
+ * @returns {object|null}
+ */
+export function getDomain(domain) {
+  return domains[domain] || null;
+}
+
+/**
+ * Get all domain names.
+ * @returns {string[]}
+ */
+export function getAllDomains() {
+  return Object.keys(domains);
+}

package/embedding-provider.js

@@ -0,0 +1,108 @@
+/**
+ * Pluggable embedding provider. Two implementations:
+ *
+ *   voyage() — Voyage AI, voyage-3-lite by default (1024 dims).
+ *              Env: VOYAGE_API_KEY. Anthropic-recommended; generous free tier.
+ *
+ *   openai() — OpenAI text-embedding-3-small (1536 dims).
+ *              Env: OPENAI_API_KEY. Cheap ($0.02/1M tokens).
+ *
+ * detectProvider() picks Voyage if available, else OpenAI, else null.
+ * A null provider means "embeddings unavailable" — callers should fall back
+ * to keyword retrieval cleanly.
+ *
+ * Both implementations batch inputs to minimize round-trips. Every provider
+ * returns a Promise<Float32Array[]> of the same length as the input.
+ */
+
+const VOYAGE_URL = 'https://api.voyageai.com/v1/embeddings';
+const OPENAI_URL = 'https://api.openai.com/v1/embeddings';
+
+function getEnv(key) {
+  if (typeof process !== 'undefined' && process.env?.[key]) return process.env[key];
+  try {
+    const env = import.meta.env;
+    if (env?.[`VITE_${key}`]) return env[`VITE_${key}`];
+    if (env?.[key]) return env[key];
+  } catch {}
+  return '';
+}
+
+/**
+ * Voyage AI embedding provider.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.apiKey] — defaults to env VOYAGE_API_KEY
+ * @param {string} [opts.model]  — voyage-3-lite | voyage-3 | voyage-code-3
+ * @returns {(texts: string[]) => Promise<Float32Array[]>} embedder, or null if no key
+ */
+export function voyage({ apiKey, model = 'voyage-3-lite' } = {}) {
+  const key = apiKey || getEnv('VOYAGE_API_KEY');
+  if (!key) return null;
+
+  return async function embed(texts) {
+    if (!Array.isArray(texts) || texts.length === 0) return [];
+    const res = await fetch(VOYAGE_URL, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        'authorization': `Bearer ${key}`,
+      },
+      body: JSON.stringify({ input: texts, model, input_type: 'document' }),
+    });
+    if (!res.ok) throw new Error(`Voyage ${res.status}: ${(await res.text()).slice(0, 200)}`);
+    const data = await res.json();
+    return data.data.map(d => new Float32Array(d.embedding));
+  };
+}
+
+/**
+ * OpenAI text-embedding-3-small provider.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.apiKey] — defaults to env OPENAI_API_KEY
+ * @param {string} [opts.model]  — text-embedding-3-small | text-embedding-3-large
+ * @returns {(texts: string[]) => Promise<Float32Array[]>} embedder, or null if no key
+ */
+export function openai({ apiKey, model = 'text-embedding-3-small' } = {}) {
+  const key = apiKey || getEnv('OPENAI_API_KEY');
+  if (!key) return null;
+
+  return async function embed(texts) {
+    if (!Array.isArray(texts) || texts.length === 0) return [];
+    const res = await fetch(OPENAI_URL, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        'authorization': `Bearer ${key}`,
+      },
+      body: JSON.stringify({ input: texts, model }),
+    });
+    if (!res.ok) throw new Error(`OpenAI ${res.status}: ${(await res.text()).slice(0, 200)}`);
+    const data = await res.json();
+    return data.data.map(d => new Float32Array(d.embedding));
+  };
+}
+
+/**
+ * Auto-detect the best available provider. Voyage first (cheaper, dense vectors
+ * at 1024 dims), then OpenAI, then null.
+ */
+export function detectProvider() {
+  const v = voyage(); if (v) return { name: 'voyage',  model: 'voyage-3-lite',             embed: v };
+  const o = openai(); if (o) return { name: 'openai',  model: 'text-embedding-3-small',    embed: o };
+  return null;
+}
+
+/** Cosine similarity between two Float32Arrays of the same length. */
+export function cosine(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na  += a[i] * a[i];
+    nb  += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}

package/embedding-retriever.js

@@ -0,0 +1,120 @@
+/**
+ * Embedding retriever — loads the build-time pattern embedding index and
+ * scores a query against every pattern via cosine similarity.
+ *
+ * Index: packages/a2ui/corpus/pattern-embeddings.json (built by
+ * scripts/build-embeddings.mjs). When missing or empty, the retriever is
+ * effectively a no-op — callers see a scoreFor(name) of 0 and should fall
+ * back to keyword-only ranking.
+ *
+ * Query embedding uses the same provider as the index (baked in at build),
+ * so the query path also requires the provider's API key at runtime. If
+ * the key is absent, the retriever exposes a `available()` probe returning
+ * false and callers degrade gracefully.
+ */
+
+import { detectProvider, cosine, voyage, openai } from './embedding-provider.js';
+
+const IS_NODE = typeof process !== 'undefined' && !!process.versions?.node;
+
+let _index = null;
+let _indexByName = null;   // Map<string, Float32Array>
+let _loadPromise = null;
+let _embedFn = null;
+let _available = null;      // lazy, once the first probe runs
+
+async function _loadIndex() {
+  if (_index) return _index;
+  if (_loadPromise) return _loadPromise;
+  _loadPromise = (async () => {
+    try {
+      if (IS_NODE) {
+        const fs = await import(/* @vite-ignore */ 'node:fs/promises');
+        const path = await import(/* @vite-ignore */ 'node:path');
+        const url = await import(/* @vite-ignore */ 'node:url');
+        const here = path.dirname(url.fileURLToPath(import.meta.url));
+        const p = path.resolve(here, '../corpus/pattern-embeddings.json');
+        const raw = await fs.readFile(p, 'utf8');
+        _index = JSON.parse(raw);
+      } else {
+        const url = new URL('../corpus/pattern-embeddings.json', import.meta.url);
+        const res = await fetch(url).catch(() => null);
+        _index = res?.ok ? await res.json().catch(() => null) : null;
+      }
+    } catch {
+      _index = null;
+    }
+    if (_index?.patterns?.length) {
+      _indexByName = new Map();
+      for (const p of _index.patterns) {
+        if (p?.name && Array.isArray(p.vector)) {
+          _indexByName.set(p.name, Float32Array.from(p.vector));
+        }
+      }
+    }
+    return _index;
+  })();
+  return _loadPromise;
+}
+
+/** Resolve the embed function matching the index's provider. */
+function _resolveEmbed(providerName, model) {
+  if (providerName === 'voyage') return voyage({ model });
+  if (providerName === 'openai') return openai({ model });
+  // Unknown or null — fall through to whatever's available.
+  const auto = detectProvider();
+  return auto?.embed || null;
+}
+
+/**
+ * True when both the index AND the matching provider's API key are available.
+ * Callers use this to decide whether to include embedding scores in the blend.
+ */
+export async function available() {
+  if (_available !== null) return _available;
+  const idx = await _loadIndex();
+  if (!idx || !idx.patterns?.length) { _available = false; return false; }
+  _embedFn = _resolveEmbed(idx.provider, idx.model);
+  _available = !!_embedFn;
+  return _available;
+}
+
+/**
+ * Embed a query string and return a { patternName → cosineScore } map.
+ * Returns an empty Map when unavailable (no index or no API key).
+ *
+ * @param {string} query — the user's intent (steelman or raw)
+ * @returns {Promise<Map<string, number>>}
+ */
+export async function scoreAll(query) {
+  if (!query || typeof query !== 'string') return new Map();
+  if (!(await available())) return new Map();
+
+  let qVec;
+  try {
+    const [v] = await _embedFn([query]);
+    qVec = v;
+  } catch (e) {
+    // Don't let a runtime embedding failure nuke retrieval — fall back cleanly.
+    if (typeof console !== 'undefined') console.warn('[embedding-retriever]', e.message);
+    return new Map();
+  }
+
+  const out = new Map();
+  for (const [name, vec] of _indexByName) {
+    out.set(name, cosine(qVec, vec));
+  }
+  return out;
+}
+
+/** Number of patterns in the index. Useful for logging/diagnostics. */
+export async function size() {
+  const idx = await _loadIndex();
+  return idx?.patterns?.length || 0;
+}
+
+/** Diagnostics: the provider/model the index was built with. */
+export async function providerInfo() {
+  const idx = await _loadIndex();
+  return idx ? { provider: idx.provider, model: idx.model, dims: idx.dims } : null;
+}