@adia-ai/a2ui-retrieval 0.0.1

package/feedback-analyzer.js

@@ -0,0 +1,235 @@
+/**
+ * Feedback Analyzer
+ *
+ * Reads JSONL feedback files, aggregates by intent category,
+ * and surfaces weak intents, promotion candidates, and pattern gaps.
+ *
+ * Usage:
+ *   import { FeedbackAnalyzer } from './feedback-analyzer.js';
+ *   const analyzer = new FeedbackAnalyzer();
+ *   const entries = await analyzer.readRange(30);
+ *   const aggregated = analyzer.aggregateByIntent(entries);
+ *   const weak = analyzer.findWeakIntents(aggregated);
+ */
+
+import { feedbackStore } from './feedback-store.js';
+import { categorizeIntent } from './intent-categorizer.js';
+
+let fs, path;
+const IS_NODE = typeof process !== 'undefined' && process.versions?.node;
+if (IS_NODE) {
+  try {
+    fs = await import(/* @vite-ignore */ 'node:fs/promises');
+    path = await import(/* @vite-ignore */ 'node:path');
+  } catch {
+    // Node builtins unavailable
+  }
+}
+
+const FEEDBACK_DIR = path
+  ? path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', 'a2ui/corpus', 'feedback')
+  : null;
+
+export class FeedbackAnalyzer {
+  /**
+   * Read JSONL feedback files for the last N days.
+   *
+   * @param {number} days — Number of days to look back (default 30)
+   * @returns {Promise<object[]>} — Array of parsed feedback entries
+   */
+  async readRange(days = 30) {
+    if (!fs || !FEEDBACK_DIR) return [];
+
+    const entries = [];
+    const now = new Date();
+
+    // Build set of date strings we want
+    const dateStrings = new Set();
+    for (let i = 0; i < days; i++) {
+      const d = new Date(now);
+      d.setDate(d.getDate() - i);
+      dateStrings.add(d.toISOString().slice(0, 10));
+    }
+
+    try {
+      const files = await fs.readdir(FEEDBACK_DIR);
+      const jsonlFiles = files
+        .filter(f => f.endsWith('.jsonl'))
+        .filter(f => {
+          const dateStr = f.replace('.jsonl', '');
+          return dateStrings.has(dateStr);
+        })
+        .sort();
+
+      for (const file of jsonlFiles) {
+        try {
+          const content = await fs.readFile(path.join(FEEDBACK_DIR, file), 'utf8');
+          const lines = content.trim().split('\n').filter(Boolean);
+          for (const line of lines) {
+            try {
+              const entry = JSON.parse(line);
+              entry._file = file;
+              entries.push(entry);
+            } catch {
+              // Skip malformed lines
+            }
+          }
+        } catch {
+          // Skip unreadable files
+        }
+      }
+    } catch {
+      // Feedback dir doesn't exist yet
+    }
+
+    return entries;
+  }
+
+  /**
+   * Aggregate feedback entries by intent category.
+   *
+   * @param {object[]} entries — Array of feedback entries
+   * @returns {Map<string, { count: number, avgScore: number, avgRating: number, patternMatchRate: number, entries: object[] }>}
+   */
+  aggregateByIntent(entries) {
+    const buckets = new Map();
+
+    // First pass: group executions by category
+    const executions = entries.filter(e => e.type === 'execution');
+    const ratings = entries.filter(e => e.type === 'rating');
+
+    // Index ratings by executionId for fast lookup
+    const ratingsByExecId = new Map();
+    for (const r of ratings) {
+      if (!ratingsByExecId.has(r.executionId)) {
+        ratingsByExecId.set(r.executionId, []);
+      }
+      ratingsByExecId.get(r.executionId).push(r);
+    }
+
+    for (const exec of executions) {
+      const { category } = categorizeIntent(exec.intent);
+
+      if (!buckets.has(category)) {
+        buckets.set(category, {
+          count: 0,
+          totalScore: 0,
+          totalRating: 0,
+          ratingCount: 0,
+          patternMatchCount: 0,
+          entries: [],
+          sampleIntents: [],
+        });
+      }
+
+      const bucket = buckets.get(category);
+      bucket.count++;
+      bucket.totalScore += exec.score || 0;
+      bucket.entries.push(exec);
+
+      if (exec.patternMatch) {
+        bucket.patternMatchCount++;
+      }
+
+      // Collect unique sample intents (up to 5)
+      if (bucket.sampleIntents.length < 5 && exec.intent) {
+        const intentLower = exec.intent.toLowerCase();
+        if (!bucket.sampleIntents.some(s => s.toLowerCase() === intentLower)) {
+          bucket.sampleIntents.push(exec.intent);
+        }
+      }
+
+      // Attach ratings
+      const execRatings = ratingsByExecId.get(exec.executionId) || [];
+      for (const r of execRatings) {
+        bucket.totalRating += r.rating;
+        bucket.ratingCount++;
+      }
+    }
+
+    // Compute averages
+    const result = new Map();
+    for (const [category, bucket] of buckets) {
+      result.set(category, {
+        count: bucket.count,
+        avgScore: bucket.count > 0 ? Math.round(bucket.totalScore / bucket.count) : 0,
+        avgRating: bucket.ratingCount > 0 ? Math.round((bucket.totalRating / bucket.ratingCount) * 10) / 10 : 0,
+        patternMatchRate: bucket.count > 0 ? Math.round((bucket.patternMatchCount / bucket.count) * 100) : 0,
+        sampleIntents: bucket.sampleIntents,
+        entries: bucket.entries,
+      });
+    }
+
+    return result;
+  }
+
+  /**
+   * Find intent categories with weak performance.
+   *
+   * @param {Map} aggregated — Output of aggregateByIntent
+   * @param {number} threshold — Score threshold (default 60)
+   * @returns {Array<{ category: string, count: number, avgScore: number, avgRating: number, sampleIntents: string[] }>}
+   */
+  findWeakIntents(aggregated, threshold = 60) {
+    const weak = [];
+    for (const [category, data] of aggregated) {
+      if (data.avgScore < threshold) {
+        weak.push({
+          category,
+          count: data.count,
+          avgScore: data.avgScore,
+          avgRating: data.avgRating,
+          sampleIntents: data.sampleIntents,
+        });
+      }
+    }
+    return weak.sort((a, b) => a.avgScore - b.avgScore);
+  }
+
+  /**
+   * Find intent categories ready for pattern promotion.
+   * Criteria: avgScore >= 95, avgRating >= 4, count >= 3
+   *
+   * @param {Map} aggregated — Output of aggregateByIntent
+   * @returns {Array<{ category: string, count: number, avgScore: number, avgRating: number, sampleIntents: string[] }>}
+   */
+  findPromotionCandidates(aggregated) {
+    const candidates = [];
+    for (const [category, data] of aggregated) {
+      if (data.avgScore >= 95 && data.avgRating >= 4 && data.count >= 3) {
+        candidates.push({
+          category,
+          count: data.count,
+          avgScore: data.avgScore,
+          avgRating: data.avgRating,
+          patternMatchRate: data.patternMatchRate,
+          sampleIntents: data.sampleIntents,
+        });
+      }
+    }
+    return candidates.sort((a, b) => b.avgScore - a.avgScore);
+  }
+
+  /**
+   * Find intent categories with no pattern match AND low scores — gaps in pattern coverage.
+   *
+   * @param {Map} aggregated — Output of aggregateByIntent
+   * @returns {Array<{ category: string, count: number, avgScore: number, patternMatchRate: number, sampleIntents: string[] }>}
+   */
+  findPatternGaps(aggregated) {
+    const gaps = [];
+    for (const [category, data] of aggregated) {
+      if (data.patternMatchRate === 0 && data.avgScore < 70) {
+        gaps.push({
+          category,
+          count: data.count,
+          avgScore: data.avgScore,
+          avgRating: data.avgRating,
+          patternMatchRate: data.patternMatchRate,
+          sampleIntents: data.sampleIntents,
+        });
+      }
+    }
+    return gaps.sort((a, b) => a.avgScore - b.avgScore);
+  }
+}

package/feedback-store.js

@@ -0,0 +1,175 @@
+/**
+ * Persistent Feedback Store
+ *
+ * Writes execution metadata, ratings, LLM self-critique, and gap signals
+ * to JSONL files on disk. One file per day. Browser-safe (no-ops if no fs).
+ *
+ * Usage:
+ *   import { feedbackStore } from './feedback-store.js';
+ *   feedbackStore.logExecution({ executionId, intent, model, domain, ... });
+ *   feedbackStore.logRating({ executionId, rating, ... });
+ *   feedbackStore.logGap({ type: 'pattern', description: '...' });
+ *   const recent = await feedbackStore.readRecent(50);
+ */
+
+let fs, path;
+const IS_NODE = typeof process !== 'undefined' && process.versions?.node;
+if (IS_NODE) {
+  try {
+    fs = await import(/* @vite-ignore */ 'node:fs/promises');
+    path = await import(/* @vite-ignore */ 'node:path');
+  } catch {
+    // Node builtins unavailable
+  }
+}
+
+const FEEDBACK_DIR = path
+  ? path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', 'a2ui/corpus', 'feedback')
+  : null;
+
+function todayFile() {
+  const d = new Date().toISOString().slice(0, 10);
+  return path ? path.join(FEEDBACK_DIR, `${d}.jsonl`) : null;
+}
+
+async function append(entry) {
+  if (!fs || !FEEDBACK_DIR) return;
+  try {
+    await fs.mkdir(FEEDBACK_DIR, { recursive: true });
+    await fs.appendFile(todayFile(), JSON.stringify(entry) + '\n');
+  } catch (e) {
+    console.warn('FeedbackStore: write failed', e.message);
+  }
+}
+
+export const feedbackStore = {
+  /**
+   * Log a completed generation execution.
+   */
+  async logExecution({
+    executionId, intent, model, domain, mode,
+    patternMatch, patternConfidence,
+    score, componentCount, tokenCount,
+    meta, // LLM self-critique
+    messages, // A2UI output (optional — can be large)
+  }) {
+    await append({
+      type: 'execution',
+      timestamp: new Date().toISOString(),
+      executionId, intent, model, domain, mode,
+      patternMatch, patternConfidence,
+      score, componentCount, tokenCount,
+      meta: meta || null,
+    });
+  },
+
+  /**
+   * Log a user rating (👍/👎).
+   */
+  async logRating({ executionId, rating, intent }) {
+    await append({
+      type: 'rating',
+      timestamp: new Date().toISOString(),
+      executionId, rating, intent,
+    });
+  },
+
+  /**
+   * Log a pattern save action.
+   */
+  async logPatternSave({ executionId, patternName, intent }) {
+    await append({
+      type: 'pattern_save',
+      timestamp: new Date().toISOString(),
+      executionId, patternName, intent,
+    });
+  },
+
+  /**
+   * Log a training gap identified by LLM meta or heuristics.
+   */
+  async logGap({ type, description, source, executionId }) {
+    await append({
+      type: 'gap',
+      gapType: type, // 'pattern' | 'domain' | 'component' | 'prompt'
+      timestamp: new Date().toISOString(),
+      description, source, executionId,
+    });
+  },
+
+  /**
+   * Read recent feedback entries (Node only).
+   */
+  async readRecent(limit = 100) {
+    if (!fs || !FEEDBACK_DIR) return [];
+    try {
+      const files = (await fs.readdir(FEEDBACK_DIR))
+        .filter(f => f.endsWith('.jsonl'))
+        .sort()
+        .reverse();
+
+      const entries = [];
+      for (const file of files) {
+        if (entries.length >= limit) break;
+        const content = await fs.readFile(path.join(FEEDBACK_DIR, file), 'utf8');
+        const lines = content.trim().split('\n').filter(Boolean).reverse();
+        for (const line of lines) {
+          if (entries.length >= limit) break;
+          try { entries.push(JSON.parse(line)); } catch {}
+        }
+      }
+      return entries;
+    } catch { return []; }
+  },
+
+  /**
+   * Get gap summary — aggregate gap signals for training improvement.
+   */
+  async getGapSummary() {
+    const entries = await this.readRecent(500);
+    const gaps = entries.filter(e => e.type === 'gap');
+    const byType = {};
+    for (const g of gaps) {
+      byType[g.gapType] = byType[g.gapType] || [];
+      byType[g.gapType].push(g.description);
+    }
+    return byType;
+  },
+
+  /**
+   * Get quality metrics — aggregate from recent executions.
+   */
+  async getQualityMetrics() {
+    const entries = await this.readRecent(500);
+    const executions = entries.filter(e => e.type === 'execution');
+    const ratings = entries.filter(e => e.type === 'rating');
+
+    if (executions.length === 0) return { executions: 0, avgScore: 0, avgTokens: 0, thumbUpRate: 0 };
+
+    const avgScore = executions.reduce((s, e) => s + (e.score || 0), 0) / executions.length;
+    const avgTokens = executions.reduce((s, e) => s + (e.tokenCount || 0), 0) / executions.length;
+    const thumbsUp = ratings.filter(r => r.rating >= 4).length;
+    const thumbsDown = ratings.filter(r => r.rating < 4).length;
+    const thumbUpRate = (thumbsUp + thumbsDown) > 0 ? thumbsUp / (thumbsUp + thumbsDown) : 0;
+
+    // Per-domain breakdown
+    const byDomain = {};
+    for (const e of executions) {
+      const d = e.domain || 'unknown';
+      if (!byDomain[d]) byDomain[d] = { count: 0, totalScore: 0 };
+      byDomain[d].count++;
+      byDomain[d].totalScore += e.score || 0;
+    }
+
+    return {
+      executions: executions.length,
+      avgScore: Math.round(avgScore),
+      avgTokens: Math.round(avgTokens),
+      thumbUpRate: Math.round(thumbUpRate * 100),
+      byDomain: Object.fromEntries(
+        Object.entries(byDomain).map(([d, v]) => [d, { count: v.count, avgScore: Math.round(v.totalScore / v.count) }])
+      ),
+      gaps: await this.getGapSummary(),
+    };
+  },
+};

package/feedback.js

@@ -0,0 +1,198 @@
+/**
+ * FeedbackCollector — Structured feedback for the evolution engine.
+ *
+ * Captures per-generation feedback across multiple dimensions:
+ *   - Overall rating (1-5)
+ *   - Intent alignment, visual quality, component choice (1-5 each)
+ *   - Whether the user edited the output
+ *   - Pattern promotion signals ("this should become a pattern")
+ *
+ * Exports as a structured JSON log for the training cycle.
+ */
+
+/**
+ * @typedef {object} FeedbackEntry
+ * @property {string} executionId
+ * @property {string} intent
+ * @property {string} domain
+ * @property {string} mode
+ * @property {number} timestamp
+ * @property {object} generation
+ * @property {number} generation.componentCount
+ * @property {string[]} generation.componentTypes
+ * @property {number} generation.score
+ * @property {{ name: string, passed: boolean }[]} generation.validationChecks
+ * @property {{ structural: number, completeness: number, idiomatic: number, minimal: number }} generation.qualityDimensions
+ * @property {object} feedback
+ * @property {number} [feedback.rating]
+ * @property {number} [feedback.intentAlignment]
+ * @property {number} [feedback.visualQuality]
+ * @property {number} [feedback.componentChoice]
+ * @property {boolean} [feedback.userEdited]
+ * @property {string} [feedback.editSummary]
+ * @property {string} [feedback.notes]
+ * @property {object} patterns
+ * @property {string} [patterns.patternUsed]
+ * @property {boolean} [patterns.shouldBePattern]
+ * @property {string} [patterns.suggestedName]
+ */
+
+export class FeedbackCollector {
+  /** @type {Map<string, FeedbackEntry>} */
+  #entries = new Map();
+
+  /**
+   * Initialize a feedback entry from generation results.
+   * Called automatically after each generation completes.
+   *
+   * @param {string} executionId
+   * @param {object} data
+   * @param {string} data.intent
+   * @param {string} data.domain
+   * @param {string} data.mode
+   * @param {object[]} data.messages
+   * @param {object} data.validation
+   */
+  initFromGeneration(executionId, { intent, domain, mode, messages, validation }) {
+    const components = messages?.[0]?.components || [];
+    const checks = validation?.checks || [];
+
+    // Compute quality dimensions (same logic as score_quality in mcp-tools.js)
+    const failedChecks = checks.filter(c => !c.passed);
+    const structural = failedChecks.some(c =>
+      ['hasRootComponent', 'noOrphanedChildren', 'flatAdjacency'].includes(c.name)
+    ) ? 0.5 : 1;
+    const completeness = Math.max(0, 1 - (
+      failedChecks.filter(c => ['textContentSet', 'allTypesRegistered'].includes(c.name)).length * 0.1
+    ));
+    const idiomatic = failedChecks.some(c =>
+      ['noBareDivs', 'noBareInputs', 'cardStructure'].includes(c.name)
+    ) ? 0.5 : 1;
+    const minimal = failedChecks.some(c =>
+      ['noHardcodedColors', 'noInlineLayout'].includes(c.name)
+    ) ? 0.5 : 1;
+
+    this.#entries.set(executionId, {
+      executionId,
+      intent: intent || '',
+      domain: domain || '',
+      mode: mode || 'instant',
+      timestamp: Date.now(),
+      generation: {
+        componentCount: components.length,
+        componentTypes: [...new Set(components.map(c => c.component).filter(Boolean))],
+        score: validation?.score ?? 0,
+        validationChecks: checks.map(c => ({ name: c.name, passed: c.passed })),
+        qualityDimensions: { structural, completeness, idiomatic, minimal },
+      },
+      feedback: {},
+      patterns: {},
+    });
+  }
+
+  /**
+   * Collect user feedback for an execution.
+   *
+   * @param {string} executionId
+   * @param {object} feedback
+   * @param {number} [feedback.rating] — 1-5
+   * @param {number} [feedback.intentAlignment] — 1-5
+   * @param {number} [feedback.visualQuality] — 1-5
+   * @param {number} [feedback.componentChoice] — 1-5
+   * @param {boolean} [feedback.userEdited]
+   * @param {string} [feedback.editSummary]
+   * @param {string} [feedback.notes]
+   */
+  collectFeedback(executionId, feedback) {
+    const entry = this.#entries.get(executionId);
+    if (!entry) {
+      // Create a minimal entry if init wasn't called
+      this.#entries.set(executionId, {
+        executionId,
+        intent: '', domain: '', mode: '', timestamp: Date.now(),
+        generation: { componentCount: 0, componentTypes: [], score: 0, validationChecks: [], qualityDimensions: {} },
+        feedback: {},
+        patterns: {},
+      });
+    }
+    const e = this.#entries.get(executionId);
+    e.feedback = { ...e.feedback, ...feedback };
+  }
+
+  /**
+   * Collect pattern-related feedback.
+   *
+   * @param {string} executionId
+   * @param {object} patternFeedback
+   * @param {string} [patternFeedback.patternUsed]
+   * @param {boolean} [patternFeedback.shouldBePattern]
+   * @param {string} [patternFeedback.suggestedName]
+   */
+  collectPatternFeedback(executionId, patternFeedback) {
+    const entry = this.#entries.get(executionId);
+    if (!entry) return;
+    entry.patterns = { ...entry.patterns, ...patternFeedback };
+  }
+
+  /**
+   * Get a single feedback entry.
+   * @param {string} executionId
+   * @returns {FeedbackEntry|null}
+   */
+  get(executionId) {
+    return this.#entries.get(executionId) ?? null;
+  }
+
+  /**
+   * Get all feedback entries.
+   * @returns {FeedbackEntry[]}
+   */
+  getAll() {
+    return [...this.#entries.values()];
+  }
+
+  /** Number of entries. */
+  get size() {
+    return this.#entries.size;
+  }
+
+  /**
+   * Export all feedback as structured JSON.
+   * In browser: triggers a file download.
+   * In Node: returns the JSON string.
+   *
+   * @returns {string} — JSON string of all entries
+   */
+  exportFeedback() {
+    const data = {
+      exportedAt: new Date().toISOString(),
+      entryCount: this.#entries.size,
+      entries: this.getAll(),
+    };
+    const json = JSON.stringify(data, null, 2);
+
+    // Browser download
+    if (typeof document !== 'undefined') {
+      const date = new Date().toISOString().slice(0, 10);
+      const blob = new Blob([json], { type: 'application/json' });
+      const url = URL.createObjectURL(blob);
+      const a = document.createElement('a');
+      a.href = url;
+      a.download = `gen-ui-feedback-${date}.json`;
+      a.style.display = 'none';
+      // Prevent SPA router from intercepting the blob URL click
+      a.addEventListener('click', (e) => e.stopPropagation());
+      document.body.appendChild(a);
+      a.click();
+      document.body.removeChild(a);
+      URL.revokeObjectURL(url);
+    }
+
+    return json;
+  }
+
+  /** Clear all entries. */
+  clear() {
+    this.#entries.clear();
+  }
+}