@adia-ai/a2ui-retrieval 0.6.6 → 0.6.7

package/feedback/feedback-analyzer.ts

@@ -1,250 +0,0 @@
-/**
- * Feedback Analyzer
- *
- * Reads JSONL feedback files, aggregates by intent category,
- * and surfaces weak intents, promotion candidates, and pattern gaps.
- *
- * Usage:
- *   import { FeedbackAnalyzer } from './feedback-analyzer.js';
- *   const analyzer = new FeedbackAnalyzer();
- *   const entries = await analyzer.readRange(30);
- *   const aggregated = analyzer.aggregateByIntent(entries);
- *   const weak = analyzer.findWeakIntents(aggregated);
- */
-
-import { feedbackStore } from './feedback-store.js';
-import { categorizeIntent } from '../intent/intent-categorizer.js';
-
-// `process` is not in scope under "types": []
-declare const process: { versions?: { node?: string } } | undefined;
-
-// Lazy top-level import pattern for Node-only modules
-let fs: typeof import('node:fs/promises') | null = null;
-let path: typeof import('node:path') | null = null;
-const IS_NODE = typeof process !== 'undefined' && process.versions?.node;
-if (IS_NODE) {
-  try {
-    fs = await import(/* @vite-ignore */ 'node:fs/promises');
-    path = await import(/* @vite-ignore */ 'node:path');
-  } catch {
-    // Node builtins unavailable
-  }
-}
-
-// packages/a2ui/retrieval/feedback → up 3 → packages/a2ui → corpus/feedback
-const FEEDBACK_DIR: string | null = path
-  ? path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', '..', 'a2ui/corpus', 'feedback')
-  : null;
-
-type FeedbackEntry = Record<string, unknown> & { type?: string; executionId?: string; intent?: string; patternMatch?: boolean; score?: number; rating?: number; _file?: string };
-
-type AggregatedBucket = {
-  count: number;
-  avgScore: number;
-  avgRating: number;
-  patternMatchRate: number;
-  sampleIntents: string[];
-  entries: FeedbackEntry[];
-};
-
-type InternalBucket = {
-  count: number;
-  totalScore: number;
-  totalRating: number;
-  ratingCount: number;
-  patternMatchCount: number;
-  entries: FeedbackEntry[];
-  sampleIntents: string[];
-};
-
-export class FeedbackAnalyzer {
-  /**
-   * Read JSONL feedback files for the last N days.
-   */
-  async readRange(days = 30): Promise<FeedbackEntry[]> {
-    if (!fs || !FEEDBACK_DIR) return [];
-
-    const entries: FeedbackEntry[] = [];
-    const now = new Date();
-
-    // Build set of date strings we want
-    const dateStrings = new Set<string>();
-    for (let i = 0; i < days; i++) {
-      const d = new Date(now);
-      d.setDate(d.getDate() - i);
-      dateStrings.add(d.toISOString().slice(0, 10));
-    }
-
-    try {
-      const files = await fs.readdir(FEEDBACK_DIR);
-      const jsonlFiles = files
-        .filter((f: string) => f.endsWith('.jsonl'))
-        .filter((f: string) => {
-          const dateStr = f.replace('.jsonl', '');
-          return dateStrings.has(dateStr);
-        })
-        .sort();
-
-      for (const file of jsonlFiles) {
-        try {
-          const content = await fs.readFile(path!.join(FEEDBACK_DIR, file), 'utf8');
-          const lines = content.trim().split('\n').filter(Boolean);
-          for (const line of lines) {
-            try {
-              const entry = JSON.parse(line) as FeedbackEntry;
-              entry['_file'] = file;
-              entries.push(entry);
-            } catch {
-              // Skip malformed lines
-            }
-          }
-        } catch {
-          // Skip unreadable files
-        }
-      }
-    } catch {
-      // Feedback dir doesn't exist yet
-    }
-
-    return entries;
-  }
-
-  /**
-   * Aggregate feedback entries by intent category.
-   */
-  aggregateByIntent(entries: FeedbackEntry[]): Map<string, AggregatedBucket> {
-    const buckets = new Map<string, InternalBucket>();
-
-    // First pass: group executions by category
-    const executions = entries.filter(e => e.type === 'execution');
-    const ratings = entries.filter(e => e.type === 'rating');
-
-    // Index ratings by executionId for fast lookup
-    const ratingsByExecId = new Map<string, FeedbackEntry[]>();
-    for (const r of ratings) {
-      if (!r.executionId) continue;
-      if (!ratingsByExecId.has(r.executionId)) {
-        ratingsByExecId.set(r.executionId, []);
-      }
-      ratingsByExecId.get(r.executionId)!.push(r);
-    }
-
-    for (const exec of executions) {
-      const { category } = categorizeIntent(exec.intent ?? '');
-
-      if (!buckets.has(category)) {
-        buckets.set(category, {
-          count: 0,
-          totalScore: 0,
-          totalRating: 0,
-          ratingCount: 0,
-          patternMatchCount: 0,
-          entries: [],
-          sampleIntents: [],
-        });
-      }
-
-      const bucket = buckets.get(category)!;
-      bucket.count++;
-      bucket.totalScore += exec.score ?? 0;
-      bucket.entries.push(exec);
-
-      if (exec.patternMatch) {
-        bucket.patternMatchCount++;
-      }
-
-      // Collect unique sample intents (up to 5)
-      if (bucket.sampleIntents.length < 5 && exec.intent) {
-        const intentLower = exec.intent.toLowerCase();
-        if (!bucket.sampleIntents.some(s => s.toLowerCase() === intentLower)) {
-          bucket.sampleIntents.push(exec.intent);
-        }
-      }
-
-      // Attach ratings
-      const execRatings = ratingsByExecId.get(exec.executionId ?? '') ?? [];
-      for (const r of execRatings) {
-        bucket.totalRating += (r.rating ?? 0);
-        bucket.ratingCount++;
-      }
-    }
-
-    // Compute averages
-    const result = new Map<string, AggregatedBucket>();
-    for (const [category, bucket] of buckets) {
-      result.set(category, {
-        count: bucket.count,
-        avgScore: bucket.count > 0 ? Math.round(bucket.totalScore / bucket.count) : 0,
-        avgRating: bucket.ratingCount > 0 ? Math.round((bucket.totalRating / bucket.ratingCount) * 10) / 10 : 0,
-        patternMatchRate: bucket.count > 0 ? Math.round((bucket.patternMatchCount / bucket.count) * 100) : 0,
-        sampleIntents: bucket.sampleIntents,
-        entries: bucket.entries,
-      });
-    }
-
-    return result;
-  }
-
-  /**
-   * Find intent categories with weak performance.
-   */
-  findWeakIntents(aggregated: Map<string, AggregatedBucket>, threshold = 60): Array<{ category: string; count: number; avgScore: number; avgRating: number; sampleIntents: string[] }> {
-    const weak = [];
-    for (const [category, data] of aggregated) {
-      if (data.avgScore < threshold) {
-        weak.push({
-          category,
-          count: data.count,
-          avgScore: data.avgScore,
-          avgRating: data.avgRating,
-          sampleIntents: data.sampleIntents,
-        });
-      }
-    }
-    return weak.sort((a, b) => a.avgScore - b.avgScore);
-  }
-
-  /**
-   * Find intent categories ready for pattern promotion.
-   * Criteria: avgScore >= 95, avgRating >= 4, count >= 3
-   */
-  findPromotionCandidates(aggregated: Map<string, AggregatedBucket>): Array<{ category: string; count: number; avgScore: number; avgRating: number; patternMatchRate: number; sampleIntents: string[] }> {
-    const candidates = [];
-    for (const [category, data] of aggregated) {
-      if (data.avgScore >= 95 && data.avgRating >= 4 && data.count >= 3) {
-        candidates.push({
-          category,
-          count: data.count,
-          avgScore: data.avgScore,
-          avgRating: data.avgRating,
-          patternMatchRate: data.patternMatchRate,
-          sampleIntents: data.sampleIntents,
-        });
-      }
-    }
-    return candidates.sort((a, b) => b.avgScore - a.avgScore);
-  }
-
-  /**
-   * Find intent categories with no pattern match AND low scores — gaps in pattern coverage.
-   */
-  findPatternGaps(aggregated: Map<string, AggregatedBucket>): Array<{ category: string; count: number; avgScore: number; avgRating: number; patternMatchRate: number; sampleIntents: string[] }> {
-    const gaps = [];
-    for (const [category, data] of aggregated) {
-      if (data.patternMatchRate === 0 && data.avgScore < 70) {
-        gaps.push({
-          category,
-          count: data.count,
-          avgScore: data.avgScore,
-          avgRating: data.avgRating,
-          patternMatchRate: data.patternMatchRate,
-          sampleIntents: data.sampleIntents,
-        });
-      }
-    }
-    return gaps.sort((a, b) => a.avgScore - b.avgScore);
-  }
-}
-
-// Export feedbackStore re-export so the module stays importable
-export { feedbackStore };

package/feedback/feedback-store.ts

@@ -1,229 +0,0 @@
-/**
- * Persistent Feedback Store
- *
- * Writes execution metadata, ratings, LLM self-critique, and gap signals
- * to JSONL files on disk. One file per day. Browser-safe (no-ops if no fs).
- *
- * Usage:
- *   import { feedbackStore } from './feedback-store.js';
- *   feedbackStore.logExecution({ executionId, intent, model, domain, ... });
- *   feedbackStore.logRating({ executionId, rating, ... });
- *   feedbackStore.logGap({ type: 'pattern', description: '...' });
- *   const recent = await feedbackStore.readRecent(50);
- */
-
-// `process` is not in scope under "types": []
-declare const process: { versions?: { node?: string } } | undefined;
-
-// Lazy top-level import pattern for Node-only modules
-let fs: typeof import('node:fs/promises') | null = null;
-let path: typeof import('node:path') | null = null;
-const IS_NODE = typeof process !== 'undefined' && process.versions?.node;
-if (IS_NODE) {
-  try {
-    fs = await import(/* @vite-ignore */ 'node:fs/promises');
-    path = await import(/* @vite-ignore */ 'node:path');
-  } catch {
-    // Node builtins unavailable
-  }
-}
-
-// packages/a2ui/retrieval/feedback → up 3 → packages/a2ui → corpus/feedback
-const FEEDBACK_DIR: string | null = path
-  ? path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', '..', 'a2ui/corpus', 'feedback')
-  : null;
-
-function todayFile(): string | null {
-  const d = new Date().toISOString().slice(0, 10);
-  return path ? path.join(FEEDBACK_DIR!, `${d}.jsonl`) : null;
-}
-
-async function append(entry: unknown): Promise<void> {
-  if (!fs || !FEEDBACK_DIR) return;
-  try {
-    await fs.mkdir(FEEDBACK_DIR, { recursive: true });
-    await fs.appendFile(todayFile()!, JSON.stringify(entry) + '\n');
-  } catch (e) {
-    console.warn('FeedbackStore: write failed', (e as Error).message);
-  }
-}
-
-type ExecutionEntry = {
-  type: 'execution';
-  timestamp: string;
-  executionId: string;
-  intent: string;
-  model: string;
-  domain: string;
-  mode: string;
-  patternMatch: boolean | undefined;
-  patternConfidence: number | undefined;
-  score: number | undefined;
-  componentCount: number | undefined;
-  tokenCount: number | undefined;
-  meta: unknown;
-};
-
-type RatingEntry = {
-  type: 'rating';
-  timestamp: string;
-  executionId: string;
-  rating: number;
-  intent: string;
-};
-
-type PatternSaveEntry = {
-  type: 'pattern_save';
-  timestamp: string;
-  executionId: string;
-  patternName: string;
-  intent: string;
-};
-
-type GapEntry = {
-  type: 'gap';
-  gapType: string;
-  timestamp: string;
-  description: string | undefined;
-  source: string | undefined;
-  executionId: string | undefined;
-};
-
-type AnyEntry = ExecutionEntry | RatingEntry | PatternSaveEntry | GapEntry | Record<string, unknown>;
-
-export const feedbackStore = {
-  /** Log a completed generation execution. */
-  async logExecution({
-    executionId, intent, model, domain, mode,
-    patternMatch, patternConfidence,
-    score, componentCount, tokenCount,
-    meta,
-  }: {
-    executionId: string;
-    intent: string;
-    model: string;
-    domain: string;
-    mode: string;
-    patternMatch?: boolean;
-    patternConfidence?: number;
-    score?: number;
-    componentCount?: number;
-    tokenCount?: number;
-    meta?: unknown;
-    messages?: unknown; // optional, not stored
-  }): Promise<void> {
-    await append({
-      type: 'execution',
-      timestamp: new Date().toISOString(),
-      executionId, intent, model, domain, mode,
-      patternMatch, patternConfidence,
-      score, componentCount, tokenCount,
-      meta: meta ?? null,
-    });
-  },
-
-  /** Log a user rating (👍/👎). */
-  async logRating({ executionId, rating, intent }: { executionId: string; rating: number; intent: string }): Promise<void> {
-    await append({
-      type: 'rating',
-      timestamp: new Date().toISOString(),
-      executionId, rating, intent,
-    });
-  },
-
-  /** Log a pattern save action. */
-  async logPatternSave({ executionId, patternName, intent }: { executionId: string; patternName: string; intent: string }): Promise<void> {
-    await append({
-      type: 'pattern_save',
-      timestamp: new Date().toISOString(),
-      executionId, patternName, intent,
-    });
-  },
-
-  /** Log a training gap identified by LLM meta or heuristics. */
-  async logGap({ type, description, source, executionId }: { type: string; description?: string; source?: string; executionId?: string }): Promise<void> {
-    await append({
-      type: 'gap',
-      gapType: type, // 'pattern' | 'domain' | 'component' | 'prompt'
-      timestamp: new Date().toISOString(),
-      description, source, executionId,
-    });
-  },
-
-  /** Read recent feedback entries (Node only). */
-  async readRecent(limit = 100): Promise<AnyEntry[]> {
-    if (!fs || !FEEDBACK_DIR) return [];
-    try {
-      const files = (await fs.readdir(FEEDBACK_DIR))
-        .filter((f: string) => f.endsWith('.jsonl'))
-        .sort()
-        .reverse();
-
-      const entries: AnyEntry[] = [];
-      for (const file of files) {
-        if (entries.length >= limit) break;
-        const content = await fs.readFile(path!.join(FEEDBACK_DIR, file), 'utf8');
-        const lines = content.trim().split('\n').filter(Boolean).reverse();
-        for (const line of lines) {
-          if (entries.length >= limit) break;
-          try { entries.push(JSON.parse(line) as AnyEntry); } catch {}
-        }
-      }
-      return entries;
-    } catch { return []; }
-  },
-
-  /** Get gap summary — aggregate gap signals for training improvement. */
-  async getGapSummary(): Promise<Record<string, string[]>> {
-    const entries = await this.readRecent(500);
-    const gaps = entries.filter(e => (e as Record<string, unknown>)['type'] === 'gap') as GapEntry[];
-    const byType: Record<string, string[]> = {};
-    for (const g of gaps) {
-      if (!byType[g.gapType]) byType[g.gapType] = [];
-      byType[g.gapType]!.push(g.description ?? '');
-    }
-    return byType;
-  },
-
-  /** Get quality metrics — aggregate from recent executions. */
-  async getQualityMetrics(): Promise<{
-    executions: number;
-    avgScore: number;
-    avgTokens: number;
-    thumbUpRate: number;
-    byDomain?: Record<string, { count: number; avgScore: number }>;
-    gaps?: Record<string, string[]>;
-  }> {
-    const entries = await this.readRecent(500);
-    const executions = entries.filter(e => (e as Record<string, unknown>)['type'] === 'execution') as ExecutionEntry[];
-    const ratings = entries.filter(e => (e as Record<string, unknown>)['type'] === 'rating') as RatingEntry[];
-
-    if (executions.length === 0) return { executions: 0, avgScore: 0, avgTokens: 0, thumbUpRate: 0 };
-
-    const avgScore = executions.reduce((s, e) => s + (e.score ?? 0), 0) / executions.length;
-    const avgTokens = executions.reduce((s, e) => s + (e.tokenCount ?? 0), 0) / executions.length;
-    const thumbsUp = ratings.filter(r => r.rating >= 4).length;
-    const thumbsDown = ratings.filter(r => r.rating < 4).length;
-    const thumbUpRate = (thumbsUp + thumbsDown) > 0 ? thumbsUp / (thumbsUp + thumbsDown) : 0;
-
-    // Per-domain breakdown
-    const byDomain: Record<string, { count: number; totalScore: number }> = {};
-    for (const e of executions) {
-      const d = e.domain || 'unknown';
-      if (!byDomain[d]) byDomain[d] = { count: 0, totalScore: 0 };
-      byDomain[d]!.count++;
-      byDomain[d]!.totalScore += e.score ?? 0;
-    }
-
-    return {
-      executions: executions.length,
-      avgScore: Math.round(avgScore),
-      avgTokens: Math.round(avgTokens),
-      thumbUpRate: Math.round(thumbUpRate * 100),
-      byDomain: Object.fromEntries(
-        Object.entries(byDomain).map(([d, v]) => [d, { count: v.count, avgScore: Math.round(v.totalScore / v.count) }])
-      ),
-      gaps: await this.getGapSummary(),
-    };
-  },
-};

package/feedback/feedback.ts

@@ -1,201 +0,0 @@
-/**
- * FeedbackCollector — Structured feedback for the evolution engine.
- *
- * Captures per-generation feedback across multiple dimensions:
- *   - Overall rating (1-5)
- *   - Intent alignment, visual quality, component choice (1-5 each)
- *   - Whether the user edited the output
- *   - Pattern promotion signals ("this should become a pattern")
- *
- * Exports as a structured JSON log for the training cycle.
- */
-
-export type QualityDimensions = {
-  structural: number;
-  completeness: number;
-  idiomatic: number;
-  minimal: number;
-};
-
-export type ValidationCheck = { name: string; passed: boolean };
-
-export type GenerationData = {
-  componentCount: number;
-  componentTypes: string[];
-  score: number;
-  validationChecks: ValidationCheck[];
-  qualityDimensions: QualityDimensions;
-};
-
-export type FeedbackData = {
-  rating?: number;
-  intentAlignment?: number;
-  visualQuality?: number;
-  componentChoice?: number;
-  userEdited?: boolean;
-  editSummary?: string;
-  notes?: string;
-};
-
-export type PatternData = {
-  patternUsed?: string;
-  shouldBePattern?: boolean;
-  suggestedName?: string;
-};
-
-export type FeedbackEntry = {
-  executionId: string;
-  intent: string;
-  domain: string;
-  mode: string;
-  timestamp: number;
-  generation: GenerationData;
-  feedback: FeedbackData;
-  patterns: PatternData;
-};
-
-type ComponentLike = {
-  component?: string;
-  [key: string]: unknown;
-};
-
-type ValidationResult = {
-  score?: number;
-  checks?: Array<{ name: string; passed: boolean }>;
-};
-
-export class FeedbackCollector {
-  readonly #entries = new Map<string, FeedbackEntry>();
-
-  /**
-   * Initialize a feedback entry from generation results.
-   * Called automatically after each generation completes.
-   */
-  initFromGeneration(executionId: string, { intent, domain, mode, messages, validation }: {
-    intent?: string;
-    domain?: string;
-    mode?: string;
-    messages?: Array<{ components?: ComponentLike[] }>;
-    validation?: ValidationResult;
-  }): void {
-    const components: ComponentLike[] = messages?.[0]?.components ?? [];
-    const checks = validation?.checks ?? [];
-
-    // Compute quality dimensions
-    const failedChecks = checks.filter(c => !c.passed);
-    const structural = failedChecks.some(c =>
-      ['hasRootComponent', 'noOrphanedChildren', 'flatAdjacency'].includes(c.name)
-    ) ? 0.5 : 1;
-    const completeness = Math.max(0, 1 - (
-      failedChecks.filter(c => ['textContentSet', 'allTypesRegistered'].includes(c.name)).length * 0.1
-    ));
-    const idiomatic = failedChecks.some(c =>
-      ['noBareDivs', 'noBareInputs', 'cardStructure'].includes(c.name)
-    ) ? 0.5 : 1;
-    const minimal = failedChecks.some(c =>
-      ['noHardcodedColors', 'noInlineLayout'].includes(c.name)
-    ) ? 0.5 : 1;
-
-    this.#entries.set(executionId, {
-      executionId,
-      intent: intent ?? '',
-      domain: domain ?? '',
-      mode: mode ?? 'instant',
-      timestamp: Date.now(),
-      generation: {
-        componentCount: components.length,
-        componentTypes: [...new Set(components.map(c => c.component).filter((x): x is string => Boolean(x)))],
-        score: validation?.score ?? 0,
-        validationChecks: checks.map(c => ({ name: c.name, passed: c.passed })),
-        qualityDimensions: { structural, completeness, idiomatic, minimal },
-      },
-      feedback: {},
-      patterns: {},
-    });
-  }
-
-  /**
-   * Collect user feedback for an execution.
-   */
-  collectFeedback(executionId: string, feedback: FeedbackData): void {
-    const entry = this.#entries.get(executionId);
-    if (!entry) {
-      // Create a minimal entry if init wasn't called
-      this.#entries.set(executionId, {
-        executionId,
-        intent: '', domain: '', mode: '', timestamp: Date.now(),
-        generation: { componentCount: 0, componentTypes: [], score: 0, validationChecks: [], qualityDimensions: { structural: 0, completeness: 0, idiomatic: 0, minimal: 0 } },
-        feedback: {},
-        patterns: {},
-      });
-    }
-    const e = this.#entries.get(executionId)!;
-    e.feedback = { ...e.feedback, ...feedback };
-  }
-
-  /**
-   * Collect pattern-related feedback.
-   */
-  collectPatternFeedback(executionId: string, patternFeedback: PatternData): void {
-    const entry = this.#entries.get(executionId);
-    if (!entry) return;
-    entry.patterns = { ...entry.patterns, ...patternFeedback };
-  }
-
-  /**
-   * Get a single feedback entry.
-   */
-  get(executionId: string): FeedbackEntry | null {
-    return this.#entries.get(executionId) ?? null;
-  }
-
-  /**
-   * Get all feedback entries.
-   */
-  getAll(): FeedbackEntry[] {
-    return [...this.#entries.values()];
-  }
-
-  /** Number of entries. */
-  get size(): number {
-    return this.#entries.size;
-  }
-
-  /**
-   * Export all feedback as structured JSON.
-   * In browser: triggers a file download.
-   * In Node: returns the JSON string.
-   */
-  exportFeedback(): string {
-    const data = {
-      exportedAt: new Date().toISOString(),
-      entryCount: this.#entries.size,
-      entries: this.getAll(),
-    };
-    const json = JSON.stringify(data, null, 2);
-
-    // Browser download
-    if (typeof document !== 'undefined') {
-      const date = new Date().toISOString().slice(0, 10);
-      const blob = new Blob([json], { type: 'application/json' });
-      const url = URL.createObjectURL(blob);
-      const a = document.createElement('a');
-      a.href = url;
-      a.download = `gen-ui-feedback-${date}.json`;
-      a.style.display = 'none';
-      // Prevent SPA router from intercepting the blob URL click
-      a.addEventListener('click', (e) => e.stopPropagation());
-      document.body.appendChild(a);
-      a.click();
-      document.body.removeChild(a);
-      URL.revokeObjectURL(url);
-    }
-
-    return json;
-  }
-
-  /** Clear all entries. */
-  clear(): void {
-    this.#entries.clear();
-  }
-}