@adia-ai/a2ui-retrieval 0.6.4 → 0.6.7

package/feedback/dialog-recorder.ts

@@ -1,172 +0,0 @@
-/**
- * Dialog Recorder — write every generation turn to disk for debugging,
- * regression analysis, and training-data bootstrapping.
- *
- * Output: logs/dialogs/<sessionId>/<NNN>-<turnId>.json
- *   - <sessionId>: storeId / executionId of the multi-turn session, or
- *                  'standalone-<isodate>' for one-shot generations.
- *   - <NNN>:        zero-padded turn index within the session (000, 001, …)
- *   - <turnId>:     short slug from the intent + a random suffix for uniqueness
- *
- * Gated by ADIA_LOG_DIALOGS=1 — opt-in, zero overhead when disabled. Browser
- * environment is also a no-op (no fs); only Node-side generation paths log.
- */
-
-// `process` is not in scope under "types": []
-declare const process: { versions?: { node?: string }; env?: Record<string, string | undefined> } | undefined;
-
-const IS_NODE = typeof process !== 'undefined' && !!process.versions?.node;
-const ENABLED = IS_NODE && (process?.env?.['ADIA_LOG_DIALOGS'] === '1' || process?.env?.['ADIA_LOG_DIALOGS'] === 'true');
-
-// Lazy-loaded Node modules (only when ENABLED)
-let _fs: typeof import('node:fs/promises') | null = null;
-let _path: typeof import('node:path') | null = null;
-let _url: typeof import('node:url') | null = null;
-let _logsRoot: string | null = null;
-
-async function _ensureModules(): Promise<void> {
-  if (_fs) return;
-  _fs = await import(/* @vite-ignore */ 'node:fs/promises');
-  _path = await import(/* @vite-ignore */ 'node:path');
-  _url = await import(/* @vite-ignore */ 'node:url');
-  // logs/ lives at the repo root: packages/a2ui/retrieval/feedback → up 4 → repo root
-  const __dirname = _path.dirname(_url.fileURLToPath(import.meta.url));
-  _logsRoot = _path.resolve(__dirname, '..', '..', '..', '..', 'logs', 'dialogs');
-}
-
-// In-memory turn counter per session
-const _turnCounter = new Map<string, number>();
-const _sessionMetaWritten = new Set<string>();
-
-export type TurnRecord = {
-  sessionId?: string;
-  intent?: string;
-  mode?: string;
-  engine?: string;
-  model?: string;
-  analysis?: {
-    steelman?: string;
-    concepts?: string[];
-    impliedComponents?: string[];
-    styleHints?: string[];
-  };
-  currentCanvas?: unknown;
-  patterns?: Array<{ name?: string; score?: number; confidence?: number; keywords?: string[] | null }>;
-  systemPrompt?: string;
-  rawLLMResponse?: string;
-  messages?: unknown[];
-  validation?: unknown;
-  drift?: unknown;
-  suggestions?: string[];
-  timing?: unknown;
-  tokens?: unknown;
-  engineDebug?: unknown;
-  isIteration?: boolean;
-};
-
-/**
- * Record one generation turn. Safe to call unconditionally — when the env var
- * is unset, this is a no-op that returns immediately.
- *
- * @returns The path written, or null when logging is disabled.
- */
-export async function recordTurn(record: TurnRecord): Promise<string | null> {
-  if (!ENABLED) return null;
-
-  try {
-    await _ensureModules();
-
-    const sessionId = record.sessionId ?? `standalone-${new Date().toISOString().replace(/[:.]/g, '-')}`;
-    const turnIdx = (_turnCounter.get(sessionId) ?? 0);
-    _turnCounter.set(sessionId, turnIdx + 1);
-
-    const slug = String(record.intent ?? 'turn')
-      .toLowerCase()
-      .replace(/[^a-z0-9]+/g, '-')
-      .replace(/^-+|-+$/g, '')
-      .slice(0, 32) || 'turn';
-    const rand = Math.random().toString(36).slice(2, 6);
-    const fileName = `${String(turnIdx).padStart(3, '0')}-${slug}-${rand}.json`;
-
-    const sessionDir = _path!.join(_logsRoot!, sessionId);
-    await _fs!.mkdir(sessionDir, { recursive: true });
-
-    if (!_sessionMetaWritten.has(sessionId) && turnIdx === 0) {
-      _sessionMetaWritten.add(sessionId);
-      const sessionMeta = {
-        sessionId,
-        startedAt: new Date().toISOString(),
-        originIntent: record.intent ?? null,
-        engine: record.engine ?? null,
-        mode: record.mode ?? null,
-        model: record.model ?? null,
-        originAnalysis: record.analysis ? {
-          steelman: record.analysis.steelman ?? null,
-          concepts: record.analysis.concepts ?? [],
-          impliedComponents: record.analysis.impliedComponents ?? [],
-          styleHints: record.analysis.styleHints ?? [],
-        } : null,
-      };
-      await _fs!.writeFile(
-        _path!.join(sessionDir, '_session.json'),
-        JSON.stringify(sessionMeta, null, 2) + '\n',
-      );
-    }
-
-    const payload = {
-      // ── Identity ────────────────────────────────────────────────
-      sessionId,
-      turnIndex: turnIdx,
-      timestamp: new Date().toISOString(),
-      isIteration: !!record.isIteration,
-
-      // ── Request ─────────────────────────────────────────────────
-      intent: record.intent,
-      mode: record.mode ?? null,
-      engine: record.engine ?? null,
-      model: record.model ?? null,
-      currentCanvas: record.currentCanvas ?? null,
-
-      // ── Pipeline-side reasoning ─────────────────────────────────
-      analysis: record.analysis ?? null,
-      patterns: (record.patterns ?? []).slice(0, 10).map(p => ({
-        name: p.name,
-        score: p.score ?? p.confidence ?? null,
-        keywords: p.keywords ?? null,
-      })),
-
-      // ── LLM I/O ─────────────────────────────────────────────────
-      systemPrompt: record.systemPrompt ?? null,
-      rawLLMResponse: record.rawLLMResponse ?? null,
-
-      // ── Result ──────────────────────────────────────────────────
-      messages: record.messages ?? [],
-      validation: record.validation ?? null,
-      drift: record.drift ?? null,
-      suggestions: record.suggestions ?? [],
-
-      // ── Telemetry ───────────────────────────────────────────────
-      timing: record.timing ?? null,
-      tokens: record.tokens ?? null,
-      engineDebug: record.engineDebug ?? null,
-    };
-
-    const filePath = _path!.join(sessionDir, fileName);
-    await _fs!.writeFile(filePath, JSON.stringify(payload, null, 2) + '\n');
-    return filePath;
-  } catch (err) {
-    // Logging must NEVER break a generation. Swallow + warn once per turn.
-    console.warn('[dialog-recorder] failed to record turn:', (err as Error).message);
-    return null;
-  }
-}
-
-/** True when logging is on. Useful for guarding expensive capture work. */
-export function isRecording(): boolean {
-  // ENABLED is already gated by IS_NODE at module init. The A2UI_COMPOSE_TRACE
-  // bypass must guard `process` again — without it, browser callers throw
-  // `ReferenceError: process is not defined` after a successful LLM round-trip.
-  if (ENABLED) return true;
-  const IS_NODE_RT = typeof process !== 'undefined' && process.versions?.node;
-  return IS_NODE_RT ? !!(process?.env?.['A2UI_COMPOSE_TRACE']) : false;
-}

package/feedback/feedback-analyzer.ts

@@ -1,250 +0,0 @@
-/**
- * Feedback Analyzer
- *
- * Reads JSONL feedback files, aggregates by intent category,
- * and surfaces weak intents, promotion candidates, and pattern gaps.
- *
- * Usage:
- *   import { FeedbackAnalyzer } from './feedback-analyzer.js';
- *   const analyzer = new FeedbackAnalyzer();
- *   const entries = await analyzer.readRange(30);
- *   const aggregated = analyzer.aggregateByIntent(entries);
- *   const weak = analyzer.findWeakIntents(aggregated);
- */
-
-import { feedbackStore } from './feedback-store.js';
-import { categorizeIntent } from '../intent/intent-categorizer.js';
-
-// `process` is not in scope under "types": []
-declare const process: { versions?: { node?: string } } | undefined;
-
-// Lazy top-level import pattern for Node-only modules
-let fs: typeof import('node:fs/promises') | null = null;
-let path: typeof import('node:path') | null = null;
-const IS_NODE = typeof process !== 'undefined' && process.versions?.node;
-if (IS_NODE) {
-  try {
-    fs = await import(/* @vite-ignore */ 'node:fs/promises');
-    path = await import(/* @vite-ignore */ 'node:path');
-  } catch {
-    // Node builtins unavailable
-  }
-}
-
-// packages/a2ui/retrieval/feedback → up 3 → packages/a2ui → corpus/feedback
-const FEEDBACK_DIR: string | null = path
-  ? path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', '..', 'a2ui/corpus', 'feedback')
-  : null;
-
-type FeedbackEntry = Record<string, unknown> & { type?: string; executionId?: string; intent?: string; patternMatch?: boolean; score?: number; rating?: number; _file?: string };
-
-type AggregatedBucket = {
-  count: number;
-  avgScore: number;
-  avgRating: number;
-  patternMatchRate: number;
-  sampleIntents: string[];
-  entries: FeedbackEntry[];
-};
-
-type InternalBucket = {
-  count: number;
-  totalScore: number;
-  totalRating: number;
-  ratingCount: number;
-  patternMatchCount: number;
-  entries: FeedbackEntry[];
-  sampleIntents: string[];
-};
-
-export class FeedbackAnalyzer {
-  /**
-   * Read JSONL feedback files for the last N days.
-   */
-  async readRange(days = 30): Promise<FeedbackEntry[]> {
-    if (!fs || !FEEDBACK_DIR) return [];
-
-    const entries: FeedbackEntry[] = [];
-    const now = new Date();
-
-    // Build set of date strings we want
-    const dateStrings = new Set<string>();
-    for (let i = 0; i < days; i++) {
-      const d = new Date(now);
-      d.setDate(d.getDate() - i);
-      dateStrings.add(d.toISOString().slice(0, 10));
-    }
-
-    try {
-      const files = await fs.readdir(FEEDBACK_DIR);
-      const jsonlFiles = files
-        .filter(f => f.endsWith('.jsonl'))
-        .filter(f => {
-          const dateStr = f.replace('.jsonl', '');
-          return dateStrings.has(dateStr);
-        })
-        .sort();
-
-      for (const file of jsonlFiles) {
-        try {
-          const content = await fs.readFile(path!.join(FEEDBACK_DIR, file), 'utf8');
-          const lines = content.trim().split('\n').filter(Boolean);
-          for (const line of lines) {
-            try {
-              const entry = JSON.parse(line) as FeedbackEntry;
-              entry['_file'] = file;
-              entries.push(entry);
-            } catch {
-              // Skip malformed lines
-            }
-          }
-        } catch {
-          // Skip unreadable files
-        }
-      }
-    } catch {
-      // Feedback dir doesn't exist yet
-    }
-
-    return entries;
-  }
-
-  /**
-   * Aggregate feedback entries by intent category.
-   */
-  aggregateByIntent(entries: FeedbackEntry[]): Map<string, AggregatedBucket> {
-    const buckets = new Map<string, InternalBucket>();
-
-    // First pass: group executions by category
-    const executions = entries.filter(e => e.type === 'execution');
-    const ratings = entries.filter(e => e.type === 'rating');
-
-    // Index ratings by executionId for fast lookup
-    const ratingsByExecId = new Map<string, FeedbackEntry[]>();
-    for (const r of ratings) {
-      if (!r.executionId) continue;
-      if (!ratingsByExecId.has(r.executionId)) {
-        ratingsByExecId.set(r.executionId, []);
-      }
-      ratingsByExecId.get(r.executionId)!.push(r);
-    }
-
-    for (const exec of executions) {
-      const { category } = categorizeIntent(exec.intent ?? '');
-
-      if (!buckets.has(category)) {
-        buckets.set(category, {
-          count: 0,
-          totalScore: 0,
-          totalRating: 0,
-          ratingCount: 0,
-          patternMatchCount: 0,
-          entries: [],
-          sampleIntents: [],
-        });
-      }
-
-      const bucket = buckets.get(category)!;
-      bucket.count++;
-      bucket.totalScore += exec.score ?? 0;
-      bucket.entries.push(exec);
-
-      if (exec.patternMatch) {
-        bucket.patternMatchCount++;
-      }
-
-      // Collect unique sample intents (up to 5)
-      if (bucket.sampleIntents.length < 5 && exec.intent) {
-        const intentLower = exec.intent.toLowerCase();
-        if (!bucket.sampleIntents.some(s => s.toLowerCase() === intentLower)) {
-          bucket.sampleIntents.push(exec.intent);
-        }
-      }
-
-      // Attach ratings
-      const execRatings = ratingsByExecId.get(exec.executionId ?? '') ?? [];
-      for (const r of execRatings) {
-        bucket.totalRating += (r.rating ?? 0);
-        bucket.ratingCount++;
-      }
-    }
-
-    // Compute averages
-    const result = new Map<string, AggregatedBucket>();
-    for (const [category, bucket] of buckets) {
-      result.set(category, {
-        count: bucket.count,
-        avgScore: bucket.count > 0 ? Math.round(bucket.totalScore / bucket.count) : 0,
-        avgRating: bucket.ratingCount > 0 ? Math.round((bucket.totalRating / bucket.ratingCount) * 10) / 10 : 0,
-        patternMatchRate: bucket.count > 0 ? Math.round((bucket.patternMatchCount / bucket.count) * 100) : 0,
-        sampleIntents: bucket.sampleIntents,
-        entries: bucket.entries,
-      });
-    }
-
-    return result;
-  }
-
-  /**
-   * Find intent categories with weak performance.
-   */
-  findWeakIntents(aggregated: Map<string, AggregatedBucket>, threshold = 60): Array<{ category: string; count: number; avgScore: number; avgRating: number; sampleIntents: string[] }> {
-    const weak = [];
-    for (const [category, data] of aggregated) {
-      if (data.avgScore < threshold) {
-        weak.push({
-          category,
-          count: data.count,
-          avgScore: data.avgScore,
-          avgRating: data.avgRating,
-          sampleIntents: data.sampleIntents,
-        });
-      }
-    }
-    return weak.sort((a, b) => a.avgScore - b.avgScore);
-  }
-
-  /**
-   * Find intent categories ready for pattern promotion.
-   * Criteria: avgScore >= 95, avgRating >= 4, count >= 3
-   */
-  findPromotionCandidates(aggregated: Map<string, AggregatedBucket>): Array<{ category: string; count: number; avgScore: number; avgRating: number; patternMatchRate: number; sampleIntents: string[] }> {
-    const candidates = [];
-    for (const [category, data] of aggregated) {
-      if (data.avgScore >= 95 && data.avgRating >= 4 && data.count >= 3) {
-        candidates.push({
-          category,
-          count: data.count,
-          avgScore: data.avgScore,
-          avgRating: data.avgRating,
-          patternMatchRate: data.patternMatchRate,
-          sampleIntents: data.sampleIntents,
-        });
-      }
-    }
-    return candidates.sort((a, b) => b.avgScore - a.avgScore);
-  }
-
-  /**
-   * Find intent categories with no pattern match AND low scores — gaps in pattern coverage.
-   */
-  findPatternGaps(aggregated: Map<string, AggregatedBucket>): Array<{ category: string; count: number; avgScore: number; avgRating: number; patternMatchRate: number; sampleIntents: string[] }> {
-    const gaps = [];
-    for (const [category, data] of aggregated) {
-      if (data.patternMatchRate === 0 && data.avgScore < 70) {
-        gaps.push({
-          category,
-          count: data.count,
-          avgScore: data.avgScore,
-          avgRating: data.avgRating,
-          patternMatchRate: data.patternMatchRate,
-          sampleIntents: data.sampleIntents,
-        });
-      }
-    }
-    return gaps.sort((a, b) => a.avgScore - b.avgScore);
-  }
-}
-
-// Export feedbackStore re-export so the module stays importable
-export { feedbackStore };

package/feedback/feedback-store.ts

@@ -1,229 +0,0 @@
-/**
- * Persistent Feedback Store
- *
- * Writes execution metadata, ratings, LLM self-critique, and gap signals
- * to JSONL files on disk. One file per day. Browser-safe (no-ops if no fs).
- *
- * Usage:
- *   import { feedbackStore } from './feedback-store.js';
- *   feedbackStore.logExecution({ executionId, intent, model, domain, ... });
- *   feedbackStore.logRating({ executionId, rating, ... });
- *   feedbackStore.logGap({ type: 'pattern', description: '...' });
- *   const recent = await feedbackStore.readRecent(50);
- */
-
-// `process` is not in scope under "types": []
-declare const process: { versions?: { node?: string } } | undefined;
-
-// Lazy top-level import pattern for Node-only modules
-let fs: typeof import('node:fs/promises') | null = null;
-let path: typeof import('node:path') | null = null;
-const IS_NODE = typeof process !== 'undefined' && process.versions?.node;
-if (IS_NODE) {
-  try {
-    fs = await import(/* @vite-ignore */ 'node:fs/promises');
-    path = await import(/* @vite-ignore */ 'node:path');
-  } catch {
-    // Node builtins unavailable
-  }
-}
-
-// packages/a2ui/retrieval/feedback → up 3 → packages/a2ui → corpus/feedback
-const FEEDBACK_DIR: string | null = path
-  ? path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', '..', 'a2ui/corpus', 'feedback')
-  : null;
-
-function todayFile(): string | null {
-  const d = new Date().toISOString().slice(0, 10);
-  return path ? path.join(FEEDBACK_DIR!, `${d}.jsonl`) : null;
-}
-
-async function append(entry: unknown): Promise<void> {
-  if (!fs || !FEEDBACK_DIR) return;
-  try {
-    await fs.mkdir(FEEDBACK_DIR, { recursive: true });
-    await fs.appendFile(todayFile()!, JSON.stringify(entry) + '\n');
-  } catch (e) {
-    console.warn('FeedbackStore: write failed', (e as Error).message);
-  }
-}
-
-type ExecutionEntry = {
-  type: 'execution';
-  timestamp: string;
-  executionId: string;
-  intent: string;
-  model: string;
-  domain: string;
-  mode: string;
-  patternMatch: boolean | undefined;
-  patternConfidence: number | undefined;
-  score: number | undefined;
-  componentCount: number | undefined;
-  tokenCount: number | undefined;
-  meta: unknown;
-};
-
-type RatingEntry = {
-  type: 'rating';
-  timestamp: string;
-  executionId: string;
-  rating: number;
-  intent: string;
-};
-
-type PatternSaveEntry = {
-  type: 'pattern_save';
-  timestamp: string;
-  executionId: string;
-  patternName: string;
-  intent: string;
-};
-
-type GapEntry = {
-  type: 'gap';
-  gapType: string;
-  timestamp: string;
-  description: string | undefined;
-  source: string | undefined;
-  executionId: string | undefined;
-};
-
-type AnyEntry = ExecutionEntry | RatingEntry | PatternSaveEntry | GapEntry | Record<string, unknown>;
-
-export const feedbackStore = {
-  /** Log a completed generation execution. */
-  async logExecution({
-    executionId, intent, model, domain, mode,
-    patternMatch, patternConfidence,
-    score, componentCount, tokenCount,
-    meta,
-  }: {
-    executionId: string;
-    intent: string;
-    model: string;
-    domain: string;
-    mode: string;
-    patternMatch?: boolean;
-    patternConfidence?: number;
-    score?: number;
-    componentCount?: number;
-    tokenCount?: number;
-    meta?: unknown;
-    messages?: unknown; // optional, not stored
-  }): Promise<void> {
-    await append({
-      type: 'execution',
-      timestamp: new Date().toISOString(),
-      executionId, intent, model, domain, mode,
-      patternMatch, patternConfidence,
-      score, componentCount, tokenCount,
-      meta: meta ?? null,
-    });
-  },
-
-  /** Log a user rating (👍/👎). */
-  async logRating({ executionId, rating, intent }: { executionId: string; rating: number; intent: string }): Promise<void> {
-    await append({
-      type: 'rating',
-      timestamp: new Date().toISOString(),
-      executionId, rating, intent,
-    });
-  },
-
-  /** Log a pattern save action. */
-  async logPatternSave({ executionId, patternName, intent }: { executionId: string; patternName: string; intent: string }): Promise<void> {
-    await append({
-      type: 'pattern_save',
-      timestamp: new Date().toISOString(),
-      executionId, patternName, intent,
-    });
-  },
-
-  /** Log a training gap identified by LLM meta or heuristics. */
-  async logGap({ type, description, source, executionId }: { type: string; description?: string; source?: string; executionId?: string }): Promise<void> {
-    await append({
-      type: 'gap',
-      gapType: type, // 'pattern' | 'domain' | 'component' | 'prompt'
-      timestamp: new Date().toISOString(),
-      description, source, executionId,
-    });
-  },
-
-  /** Read recent feedback entries (Node only). */
-  async readRecent(limit = 100): Promise<AnyEntry[]> {
-    if (!fs || !FEEDBACK_DIR) return [];
-    try {
-      const files = (await fs.readdir(FEEDBACK_DIR))
-        .filter(f => f.endsWith('.jsonl'))
-        .sort()
-        .reverse();
-
-      const entries: AnyEntry[] = [];
-      for (const file of files) {
-        if (entries.length >= limit) break;
-        const content = await fs.readFile(path!.join(FEEDBACK_DIR, file), 'utf8');
-        const lines = content.trim().split('\n').filter(Boolean).reverse();
-        for (const line of lines) {
-          if (entries.length >= limit) break;
-          try { entries.push(JSON.parse(line) as AnyEntry); } catch {}
-        }
-      }
-      return entries;
-    } catch { return []; }
-  },
-
-  /** Get gap summary — aggregate gap signals for training improvement. */
-  async getGapSummary(): Promise<Record<string, string[]>> {
-    const entries = await this.readRecent(500);
-    const gaps = entries.filter(e => (e as Record<string, unknown>)['type'] === 'gap') as GapEntry[];
-    const byType: Record<string, string[]> = {};
-    for (const g of gaps) {
-      if (!byType[g.gapType]) byType[g.gapType] = [];
-      byType[g.gapType]!.push(g.description ?? '');
-    }
-    return byType;
-  },
-
-  /** Get quality metrics — aggregate from recent executions. */
-  async getQualityMetrics(): Promise<{
-    executions: number;
-    avgScore: number;
-    avgTokens: number;
-    thumbUpRate: number;
-    byDomain?: Record<string, { count: number; avgScore: number }>;
-    gaps?: Record<string, string[]>;
-  }> {
-    const entries = await this.readRecent(500);
-    const executions = entries.filter(e => (e as Record<string, unknown>)['type'] === 'execution') as ExecutionEntry[];
-    const ratings = entries.filter(e => (e as Record<string, unknown>)['type'] === 'rating') as RatingEntry[];
-
-    if (executions.length === 0) return { executions: 0, avgScore: 0, avgTokens: 0, thumbUpRate: 0 };
-
-    const avgScore = executions.reduce((s, e) => s + (e.score ?? 0), 0) / executions.length;
-    const avgTokens = executions.reduce((s, e) => s + (e.tokenCount ?? 0), 0) / executions.length;
-    const thumbsUp = ratings.filter(r => r.rating >= 4).length;
-    const thumbsDown = ratings.filter(r => r.rating < 4).length;
-    const thumbUpRate = (thumbsUp + thumbsDown) > 0 ? thumbsUp / (thumbsUp + thumbsDown) : 0;
-
-    // Per-domain breakdown
-    const byDomain: Record<string, { count: number; totalScore: number }> = {};
-    for (const e of executions) {
-      const d = e.domain || 'unknown';
-      if (!byDomain[d]) byDomain[d] = { count: 0, totalScore: 0 };
-      byDomain[d]!.count++;
-      byDomain[d]!.totalScore += e.score ?? 0;
-    }
-
-    return {
-      executions: executions.length,
-      avgScore: Math.round(avgScore),
-      avgTokens: Math.round(avgTokens),
-      thumbUpRate: Math.round(thumbUpRate * 100),
-      byDomain: Object.fromEntries(
-        Object.entries(byDomain).map(([d, v]) => [d, { count: v.count, avgScore: Math.round(v.totalScore / v.count) }])
-      ),
-      gaps: await this.getGapSummary(),
-    };
-  },
-};