@massu/core 1.10.8 → 1.12.0

package/src/observation-extractor.ts

@@ -1,418 +0,0 @@
-// Copyright (c) 2026 Massu. All rights reserved.
-// Licensed under BSL 1.1 - see LICENSE file for details.
-
-import type { ParsedToolCall, TranscriptEntry, ExtractedDecision, ExtractedFailedAttempt } from './transcript-parser.ts';
-import {
-  extractToolCalls,
-  extractDecisions,
-  extractFailedAttempts,
-  extractFileOperations,
-  extractVerificationCommands,
-  extractUserMessages,
-  estimateTokens,
-} from './transcript-parser.ts';
-import type { AddObservationOpts } from './memory-db.ts';
-import { assignImportance } from './memory-db.ts';
-import { detectDecisionPatterns } from './adr-generator.ts';
-import { getProjectRoot, getConfig, getResolvedPaths } from './config.ts';
-import { homedir } from 'os';
-
-// ============================================================
-// P2-002: Observation Extractor
-// ============================================================
-
-/**
- * Visibility classification for observations.
- * - 'public': Safe to share across teams and sync to cloud (no secrets, no absolute paths)
- * - 'private': Contains potentially sensitive data (file paths, env vars, credentials)
- */
-export type ObservationVisibility = 'public' | 'private';
-
-/**
- * A structured observation ready for DB insertion.
- */
-export interface ExtractedObservation {
-  type: string;
-  title: string;
-  detail: string | null;
-  visibility: ObservationVisibility;
-  opts: AddObservationOpts;
-}
-
-/**
- * Patterns that indicate private/sensitive content.
- * If any of these match the observation title or detail, it is classified as 'private'.
- */
-const PRIVATE_PATTERNS = [
-  /\/Users\/\w+/,              // Absolute macOS paths
-  /\/home\/\w+/,               // Absolute Linux paths
-  /[A-Z]:\\/,                  // Windows paths
-  /\b(api[_-]?key|secret|token|password|credential|dsn)\b/i,  // Secrets
-  /\b(STRIPE_|SUPABASE_|SENTRY_|AWS_|DATABASE_URL)\b/,        // Env var names
-  /\.(env|pem|key|cert)\b/,   // Sensitive file extensions
-  /Bearer\s+\S+/,             // Auth tokens
-  /sk_live_|sk_test_|whsec_/, // Stripe keys
-];
-
-/**
- * Classify whether an observation is safe for public sharing or should remain private.
- */
-export function classifyVisibility(title: string, detail: string | null): ObservationVisibility {
-  const text = `${title} ${detail ?? ''}`;
-  for (const pattern of PRIVATE_PATTERNS) {
-    if (pattern.test(text)) return 'private';
-  }
-  return 'public';
-}
-
-/**
- * Noise filtering rules - these tool calls add no memory value.
- */
-export function isNoisyToolCall(tc: ParsedToolCall, seenReads: Set<string>): boolean {
-  // Glob/Grep tool calls (search operations, not actions)
-  if (tc.toolName === 'Glob' || tc.toolName === 'Grep') return true;
-
-  // Duplicate Read calls to the same file within same session
-  if (tc.toolName === 'Read') {
-    const filePath = tc.input.file_path as string ?? '';
-    if (seenReads.has(filePath)) return true;
-    seenReads.add(filePath);
-
-    // Read of node_modules files
-    if (filePath.includes('node_modules')) return true;
-  }
-
-  // Bash with trivial commands
-  if (tc.toolName === 'Bash') {
-    const cmd = (tc.input.command as string ?? '').trim();
-    const trivialPatterns = /^(ls|pwd|echo|cat\s|head\s|tail\s|wc\s)/;
-    if (trivialPatterns.test(cmd)) return true;
-  }
-
-  // Empty or error-only tool responses
-  if (!tc.result || tc.result.trim() === '') return true;
-
-  return false;
-}
-
-/**
- * Extract all observations from transcript entries.
- * This is the main entry point for both real-time (P3-002) and backfill (P6-001).
- */
-export function extractObservationsFromEntries(entries: TranscriptEntry[]): ExtractedObservation[] {
-  const observations: ExtractedObservation[] = [];
-  const seenReads = new Set<string>();
-
-  // 1. Extract from tool calls
-  const toolCalls = extractToolCalls(entries);
-  for (const tc of toolCalls) {
-    if (isNoisyToolCall(tc, seenReads)) continue;
-    const obs = classifyToolCall(tc);
-    if (obs) observations.push(obs);
-  }
-
-  // 2. Extract decisions from assistant text
-  const decisions = extractDecisions(entries);
-  for (const decision of decisions) {
-    const title = decision.text.slice(0, 200);
-    const detail = decision.context;
-    observations.push({
-      type: 'decision',
-      title,
-      detail,
-      visibility: classifyVisibility(title, detail),
-      opts: {
-        importance: assignImportance('decision'),
-        originalTokens: estimateTokens(decision.text + decision.context),
-      },
-    });
-  }
-
-  // 3. Extract failed attempts from assistant text
-  const failures = extractFailedAttempts(entries);
-  for (const failure of failures) {
-    const title = failure.text.slice(0, 200);
-    const detail = failure.context;
-    observations.push({
-      type: 'failed_attempt',
-      title,
-      detail,
-      visibility: classifyVisibility(title, detail),
-      opts: {
-        importance: assignImportance('failed_attempt'),
-        originalTokens: estimateTokens(failure.text + failure.context),
-      },
-    });
-  }
-
-  // 4. Extract verification commands
-  const verifications = extractVerificationCommands(toolCalls);
-  for (const vr of verifications) {
-    const title = `${vr.vrType}: ${vr.passed ? 'PASS' : 'FAIL'}`;
-    const detail = vr.command;
-    observations.push({
-      type: 'vr_check',
-      title,
-      detail,
-      visibility: classifyVisibility(title, detail),
-      opts: {
-        vrType: vr.vrType,
-        evidence: vr.result,
-        importance: assignImportance('vr_check', vr.passed ? 'PASS' : 'FAIL'),
-        originalTokens: estimateTokens(vr.result),
-      },
-    });
-  }
-
-  return observations;
-}
-
-/**
- * Classify a single tool call into an observation (or null if not observation-worthy).
- */
-function classifyToolCall(tc: ParsedToolCall): ExtractedObservation | null {
-  const result = tc.result ?? '';
-
-  switch (tc.toolName) {
-    case 'Write': {
-      const filePath = tc.input.file_path as string ?? 'unknown';
-      const title = `Created/wrote: ${shortenPath(filePath)}`;
-      return {
-        type: 'file_change',
-        title,
-        detail: null,
-        visibility: classifyVisibility(title, filePath),
-        opts: {
-          filesInvolved: [filePath],
-          importance: assignImportance('file_change'),
-          originalTokens: estimateTokens(result),
-          ...extractLinkedReferences(result + filePath),
-        },
-      };
-    }
-
-    case 'Edit': {
-      const filePath = tc.input.file_path as string ?? 'unknown';
-      const title = `Edited: ${shortenPath(filePath)}`;
-      return {
-        type: 'file_change',
-        title,
-        detail: null,
-        visibility: classifyVisibility(title, filePath),
-        opts: {
-          filesInvolved: [filePath],
-          importance: assignImportance('file_change'),
-          originalTokens: estimateTokens(result),
-          ...extractLinkedReferences(result + filePath),
-        },
-      };
-    }
-
-    case 'Read': {
-      const filePath = tc.input.file_path as string ?? 'unknown';
-      // Only keep reads of interesting files (plan files, knowledge source files, etc.)
-      const knowledgeSourceFiles = getConfig().conventions?.knowledgeSourceFiles ?? ['CLAUDE.md', 'MEMORY.md', 'corrections.md'];
-      const plansDir = getResolvedPaths().plansDir;
-      if (filePath.includes(plansDir) || knowledgeSourceFiles.some(f => filePath.includes(f))) {
-        const title = `Read: ${shortenPath(filePath)}`;
-        return {
-          type: 'discovery',
-          title,
-          detail: null,
-          visibility: classifyVisibility(title, filePath),
-          opts: {
-            filesInvolved: [filePath],
-            importance: assignImportance('discovery'),
-            originalTokens: estimateTokens(result),
-          },
-        };
-      }
-      return null;
-    }
-
-    case 'Bash': {
-      const cmd = (tc.input.command as string ?? '').trim();
-
-      // Git commit
-      if (cmd.includes('git commit')) {
-        const commitMsg = extractCommitMessage(cmd);
-        const isfix = commitMsg.toLowerCase().includes('fix');
-        const title = `Commit: ${commitMsg.slice(0, 150)}`;
-        return {
-          type: isfix ? 'bugfix' : 'feature',
-          title,
-          detail: cmd,
-          visibility: classifyVisibility(title, cmd),
-          opts: {
-            importance: assignImportance(isfix ? 'bugfix' : 'feature'),
-            originalTokens: estimateTokens(result),
-          },
-        };
-      }
-
-      // Pattern scanner
-      if (cmd.includes('pattern-scanner')) {
-        const passed = !result.includes('FAIL') && !result.includes('BLOCKED');
-        const title = `Pattern Scanner: ${passed ? 'PASS' : 'FAIL'}`;
-        const detail = result.slice(0, 500);
-        return {
-          type: 'pattern_compliance',
-          title,
-          detail,
-          visibility: classifyVisibility(title, detail),
-          opts: {
-            evidence: result.slice(0, 500),
-            importance: assignImportance('pattern_compliance', passed ? 'PASS' : 'FAIL'),
-            originalTokens: estimateTokens(result),
-          },
-        };
-      }
-
-      // npm test / vitest
-      if (cmd.includes('npm test') || cmd.includes('vitest')) {
-        const passed = !tc.isError && !result.includes('FAIL');
-        const title = `Tests: ${passed ? 'PASS' : 'FAIL'}`;
-        return {
-          type: 'vr_check',
-          title,
-          detail: cmd,
-          visibility: classifyVisibility(title, cmd),
-          opts: {
-            vrType: 'VR-TEST',
-            evidence: result.slice(0, 500),
-            importance: assignImportance('vr_check', passed ? 'PASS' : 'FAIL'),
-            originalTokens: estimateTokens(result),
-          },
-        };
-      }
-
-      // npm run build / tsc
-      if (cmd.includes('npm run build') || cmd.includes('tsc --noEmit')) {
-        const vrType = cmd.includes('tsc') ? 'VR-TYPE' : 'VR-BUILD';
-        const passed = !tc.isError && !result.includes('error');
-        const title = `${vrType}: ${passed ? 'PASS' : 'FAIL'}`;
-        return {
-          type: 'vr_check',
-          title,
-          detail: cmd,
-          visibility: classifyVisibility(title, cmd),
-          opts: {
-            vrType,
-            evidence: result.slice(0, 500),
-            importance: assignImportance('vr_check', passed ? 'PASS' : 'FAIL'),
-            originalTokens: estimateTokens(result),
-          },
-        };
-      }
-
-      return null;
-    }
-
-    default:
-      return null;
-  }
-}
-
-/**
- * Extract CR rule, VR type, and plan item references from text.
- */
-function extractLinkedReferences(text: string): Partial<AddObservationOpts> {
-  const result: Partial<AddObservationOpts> = {};
-
-  const crMatch = text.match(/CR-(\d+)/);
-  if (crMatch) result.crRule = `CR-${crMatch[1]}`;
-
-  const vrMatch = text.match(/VR-([A-Z_]+)/);
-  if (vrMatch) result.vrType = `VR-${vrMatch[1]}`;
-
-  const planMatch = text.match(/P(\d+)-(\d+)/);
-  if (planMatch) result.planItem = `P${planMatch[1]}-${planMatch[2]}`;
-
-  return result;
-}
-
-/**
- * Extract commit message from a git commit command.
- */
-function extractCommitMessage(cmd: string): string {
-  // Match -m "message" or -m 'message'
-  const match = cmd.match(/-m\s+["'](.+?)["']/);
-  if (match) return match[1];
-
-  // Match heredoc pattern
-  const heredocMatch = cmd.match(/<<['"]?EOF['"]?\s*\n?([\s\S]*?)EOF/);
-  if (heredocMatch) return heredocMatch[1].trim().split('\n')[0];
-
-  return 'Unknown commit';
-}
-
-/**
- * Shorten a file path for display.
- */
-function shortenPath(filePath: string): string {
-  // Remove project root prefix, then common home dir prefix
-  const root = getProjectRoot();
-  if (filePath.startsWith(root + '/')) {
-    return filePath.slice(root.length + 1);
-  }
-  const home = homedir();
-  if (filePath.startsWith(home + '/')) {
-    return '~/' + filePath.slice(home.length + 1);
-  }
-  return filePath;
-}
-
-/**
- * Classify a single tool call for real-time capture (used by PostToolUse hook P3-002).
- * Applies noise filtering.
- */
-export function classifyRealTimeToolCall(
-  toolName: string,
-  toolInput: Record<string, unknown>,
-  toolResponse: string,
-  seenReads: Set<string>
-): ExtractedObservation | null {
-  const tc: ParsedToolCall = {
-    toolName,
-    toolUseId: '',
-    input: toolInput,
-    result: toolResponse,
-    isError: false,
-  };
-
-  if (isNoisyToolCall(tc, seenReads)) return null;
-
-  // P2-003: Detect architecture decision patterns in tool responses
-  if (toolResponse && detectDecisionPatterns(toolResponse)) {
-    const firstLine = toolResponse.split('\n')[0].slice(0, 200);
-    const title = `Architecture decision: ${firstLine}`;
-    const detail = toolResponse.slice(0, 1000);
-    return {
-      type: 'decision',
-      title,
-      detail,
-      visibility: classifyVisibility(title, detail),
-      opts: {
-        importance: assignImportance('decision'),
-        originalTokens: estimateTokens(toolResponse),
-        ...extractLinkedReferences(toolResponse),
-      },
-    };
-  }
-
-  return classifyToolCall(tc);
-}
-
-/**
- * Detect plan progress references in tool responses.
- * Returns plan items that appear to be completed.
- */
-export function detectPlanProgress(toolResponse: string): Array<{ planItem: string; status: string }> {
-  const results: Array<{ planItem: string; status: string }> = [];
-  const progressPattern = /(P\d+-\d+)\s*[:\-]?\s*(COMPLETE|PASS|DONE|complete|pass|done)/g;
-  let match;
-  while ((match = progressPattern.exec(toolResponse)) !== null) {
-    results.push({ planItem: match[1], status: 'complete' });
-  }
-  return results;
-}

package/src/page-deps.ts

@@ -1,285 +0,0 @@
-// Copyright (c) 2026 Massu. All rights reserved.
-// Licensed under BSL 1.1 - see LICENSE file for details.
-
-import { readFileSync, existsSync } from 'fs';
-import { resolve } from 'path';
-import type Database from 'better-sqlite3';
-import { getConfig, getProjectRoot } from './config.ts';
-import { ensureWithinRoot } from './security-utils.ts';
-import { t } from './lib/sql-table-names.ts';
-
-export interface PageChain {
-  page: string;
-  route: string;
-  portal: string;
-  components: string[];
-  hooks: string[];
-  routers: string[];
-  tables: string[];
-}
-
-/**
- * Derive the URL route from a Next.js page file path.
- * e.g., src/app/orders/page.tsx -> /orders
- * e.g., src/app/orders/[id]/page.tsx -> /orders/[id]
- */
-export function deriveRoute(pageFile: string): string {
-  let route = pageFile
-    .replace(/^src\/app/, '')
-    .replace(/\/page\.tsx?$/, '')
-    .replace(/\/page\.jsx?$/, '');
-  return route || '/';
-}
-
-/**
- * Determine which portal/scope a page belongs to based on its route.
- * Uses accessScopes from config if available, otherwise infers from route prefix.
- */
-export function derivePortal(route: string): string {
-  const scopes = getConfig().accessScopes;
-  if (scopes && scopes.length > 0) {
-    for (const scope of scopes) {
-      if (route.startsWith('/' + scope)) return scope;
-    }
-  }
-  // Fallback: use the first path segment as the scope
-  const parts = route.split('/').filter(Boolean);
-  return parts[0] ?? 'default';
-}
-
-/**
- * Recursively trace imports from a file, collecting components and hooks.
- */
-function traceImports(
-  startFile: string,
-  dataDb: Database.Database,
-  visited: Set<string>,
-  components: Set<string>,
-  hooks: Set<string>,
-  maxDepth: number = 5
-): void {
-  if (maxDepth <= 0 || visited.has(startFile)) return;
-  visited.add(startFile);
-
-  const imports = dataDb.prepare(
-    `SELECT target_file, imported_names FROM ${t('imports')} WHERE source_file = ?`
-  ).all(startFile) as { target_file: string; imported_names: string }[];
-
-  for (const imp of imports) {
-    const target = imp.target_file;
-
-    // Classify the import
-    if (target.includes('/components/')) {
-      components.add(target);
-    }
-    if (target.includes('/hooks/') || target.match(/use[A-Z]/)) {
-      hooks.add(target);
-    }
-
-    // Recurse into local imports (not node_modules)
-    if (target.startsWith('src/')) {
-      traceImports(target, dataDb, visited, components, hooks, maxDepth - 1);
-    }
-  }
-}
-
-/**
- * Find routers called by hooks/components via api.* patterns.
- */
-function findRouterCalls(files: string[]): string[] {
-  const routers = new Set<string>();
-  const projectRoot = getProjectRoot();
-
-  for (const file of files) {
-    const absPath = ensureWithinRoot(resolve(projectRoot, file), projectRoot);
-    if (!existsSync(absPath)) continue;
-
-    try {
-      const source = readFileSync(absPath, 'utf-8');
-      const apiCallRegex = /api\.(\w+)\.\w+/g;
-      let match;
-      while ((match = apiCallRegex.exec(source)) !== null) {
-        routers.add(match[1]);
-      }
-    } catch {
-      // Skip unreadable component files
-    }
-  }
-
-  return [...routers];
-}
-
-/**
- * Find database tables touched by routers.
- */
-function findTablesFromRouters(routerNames: string[], dataDb: Database.Database): string[] {
-  const tables = new Set<string>();
-
-  // Look up router files from the tRPC index
-  for (const routerName of routerNames) {
-    const procs = dataDb.prepare(
-      `SELECT DISTINCT router_file FROM ${t('trpc_procedures')} WHERE router_name = ?`
-    ).all(routerName) as { router_file: string }[];
-
-    for (const proc of procs) {
-      const absPath = ensureWithinRoot(resolve(getProjectRoot(), proc.router_file), getProjectRoot());
-      if (!existsSync(absPath)) continue;
-
-      try {
-        const source = readFileSync(absPath, 'utf-8');
-        // Match database access patterns from config (default: ctx.db.{table})
-        const dbPattern = getConfig().dbAccessPattern ?? 'ctx.db.{table}';
-        const regexStr = dbPattern
-          .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-          .replace('\\{table\\}', '(\\w+)');
-        const tableRegex = new RegExp(regexStr + '\\.', 'g');
-        let match;
-        while ((match = tableRegex.exec(source)) !== null) {
-          tables.add(match[1]);
-        }
-      } catch {
-        // Skip unreadable source files
-      }
-    }
-  }
-
-  return [...tables];
-}
-
-/**
- * Build page dependency chains for all page.tsx files.
- */
-export function buildPageDeps(dataDb: Database.Database, codegraphDb: Database.Database): number {
-  // Clear existing data
-  dataDb.exec(`DELETE FROM ${t('page_deps')}`);
-
-  // Find all page.tsx files from CodeGraph
-  const pages = codegraphDb.prepare(
-    "SELECT path FROM files WHERE path LIKE 'src/app/%/page.tsx' OR path = 'src/app/page.tsx'"
-  ).all() as { path: string }[];
-
-  const insertStmt = dataDb.prepare(
-    `INSERT INTO ${t('page_deps')} (page_file, route, portal, components, hooks, routers, tables_touched) VALUES (?, ?, ?, ?, ?, ?, ?)`
-  );
-
-  let count = 0;
-
-  const insertAll = dataDb.transaction(() => {
-    for (const page of pages) {
-      const route = deriveRoute(page.path);
-      const portal = derivePortal(route);
-
-      const visited = new Set<string>();
-      const components = new Set<string>();
-      const hooks = new Set<string>();
-
-      traceImports(page.path, dataDb, visited, components, hooks);
-
-      const allFiles = [...visited];
-      const routers = findRouterCalls(allFiles);
-      const tables = findTablesFromRouters(routers, dataDb);
-
-      insertStmt.run(
-        page.path,
-        route,
-        portal,
-        JSON.stringify([...components]),
-        JSON.stringify([...hooks]),
-        JSON.stringify(routers),
-        JSON.stringify(tables)
-      );
-
-      count++;
-    }
-  });
-
-  insertAll();
-  return count;
-}
-
-/**
- * Get the dependency chain for a specific page.
- */
-export function getPageChain(dataDb: Database.Database, pageFile: string): PageChain | null {
-  const row = dataDb.prepare(`SELECT * FROM ${t('page_deps')} WHERE page_file = ?`).get(pageFile) as {
-    page_file: string; route: string; portal: string;
-    components: string; hooks: string; routers: string; tables_touched: string;
-  } | undefined;
-
-  if (!row) return null;
-
-  return {
-    page: row.page_file,
-    route: row.route,
-    portal: row.portal,
-    components: JSON.parse(row.components),
-    hooks: JSON.parse(row.hooks),
-    routers: JSON.parse(row.routers),
-    tables: JSON.parse(row.tables_touched),
-  };
-}
-
-/**
- * Find all pages affected by a given file (reverse lookup).
- */
-export function findAffectedPages(dataDb: Database.Database, file: string): PageChain[] {
-  // Check if this file is directly a page
-  const directPage = getPageChain(dataDb, file);
-  if (directPage) return [directPage];
-
-  // Find all pages that import this file (directly or transitively)
-  // First, find who imports this file
-  const importers = dataDb.prepare(
-    `SELECT source_file FROM ${t('imports')} WHERE target_file = ?`
-  ).all(file) as { source_file: string }[];
-
-  const affectedFiles = new Set<string>([file, ...importers.map(i => i.source_file)]);
-
-  // Walk up the import tree to find pages
-  let frontier = [...importers.map(i => i.source_file)];
-  const visited = new Set(frontier);
-  const maxDepth = 10;
-  let depth = 0;
-
-  while (frontier.length > 0 && depth < maxDepth) {
-    const next: string[] = [];
-    for (const f of frontier) {
-      const upstreamImporters = dataDb.prepare(
-        `SELECT source_file FROM ${t('imports')} WHERE target_file = ?`
-      ).all(f) as { source_file: string }[];
-
-      for (const imp of upstreamImporters) {
-        if (!visited.has(imp.source_file)) {
-          visited.add(imp.source_file);
-          affectedFiles.add(imp.source_file);
-          next.push(imp.source_file);
-        }
-      }
-    }
-    frontier = next;
-    depth++;
-  }
-
-  // Now find which pages are in the affected set
-  const allPages = dataDb.prepare(`SELECT * FROM ${t('page_deps')}`).all() as {
-    page_file: string; route: string; portal: string;
-    components: string; hooks: string; routers: string; tables_touched: string;
-  }[];
-
-  const results: PageChain[] = [];
-  for (const row of allPages) {
-    if (affectedFiles.has(row.page_file)) {
-      results.push({
-        page: row.page_file,
-        route: row.route,
-        portal: row.portal,
-        components: JSON.parse(row.components),
-        hooks: JSON.parse(row.hooks),
-        routers: JSON.parse(row.routers),
-        tables: JSON.parse(row.tables_touched),
-      });
-    }
-  }
-
-  return results;
-}