@massu/core 0.1.0 → 0.1.2

package/src/observation-extractor.ts

@@ -0,0 +1,411 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+import type { ParsedToolCall, TranscriptEntry, ExtractedDecision, ExtractedFailedAttempt } from './transcript-parser.ts';
+import {
+  extractToolCalls,
+  extractDecisions,
+  extractFailedAttempts,
+  extractFileOperations,
+  extractVerificationCommands,
+  extractUserMessages,
+  estimateTokens,
+} from './transcript-parser.ts';
+import type { AddObservationOpts } from './memory-db.ts';
+import { assignImportance } from './memory-db.ts';
+import { detectDecisionPatterns } from './adr-generator.ts';
+import { getProjectRoot } from './config.ts';
+
+// ============================================================
+// P2-002: Observation Extractor
+// ============================================================
+
+/**
+ * Visibility classification for observations.
+ * - 'public': Safe to share across teams and sync to cloud (no secrets, no absolute paths)
+ * - 'private': Contains potentially sensitive data (file paths, env vars, credentials)
+ */
+export type ObservationVisibility = 'public' | 'private';
+
+/**
+ * A structured observation ready for DB insertion.
+ */
+export interface ExtractedObservation {
+  type: string;
+  title: string;
+  detail: string | null;
+  visibility: ObservationVisibility;
+  opts: AddObservationOpts;
+}
+
+/**
+ * Patterns that indicate private/sensitive content.
+ * If any of these match the observation title or detail, it is classified as 'private'.
+ */
+const PRIVATE_PATTERNS = [
+  /\/Users\/\w+/,              // Absolute macOS paths
+  /\/home\/\w+/,               // Absolute Linux paths
+  /[A-Z]:\\/,                  // Windows paths
+  /\b(api[_-]?key|secret|token|password|credential|dsn)\b/i,  // Secrets
+  /\b(STRIPE_|SUPABASE_|SENTRY_|AWS_|DATABASE_URL)\b/,        // Env var names
+  /\.(env|pem|key|cert)\b/,   // Sensitive file extensions
+  /Bearer\s+\S+/,             // Auth tokens
+  /sk_live_|sk_test_|whsec_/, // Stripe keys
+];
+
+/**
+ * Classify whether an observation is safe for public sharing or should remain private.
+ */
+export function classifyVisibility(title: string, detail: string | null): ObservationVisibility {
+  const text = `${title} ${detail ?? ''}`;
+  for (const pattern of PRIVATE_PATTERNS) {
+    if (pattern.test(text)) return 'private';
+  }
+  return 'public';
+}
+
+/**
+ * Noise filtering rules - these tool calls add no memory value.
+ */
+export function isNoisyToolCall(tc: ParsedToolCall, seenReads: Set<string>): boolean {
+  // Glob/Grep tool calls (search operations, not actions)
+  if (tc.toolName === 'Glob' || tc.toolName === 'Grep') return true;
+
+  // Duplicate Read calls to the same file within same session
+  if (tc.toolName === 'Read') {
+    const filePath = tc.input.file_path as string ?? '';
+    if (seenReads.has(filePath)) return true;
+    seenReads.add(filePath);
+
+    // Read of node_modules files
+    if (filePath.includes('node_modules')) return true;
+  }
+
+  // Bash with trivial commands
+  if (tc.toolName === 'Bash') {
+    const cmd = (tc.input.command as string ?? '').trim();
+    const trivialPatterns = /^(ls|pwd|echo|cat\s|head\s|tail\s|wc\s)/;
+    if (trivialPatterns.test(cmd)) return true;
+  }
+
+  // Empty or error-only tool responses
+  if (!tc.result || tc.result.trim() === '') return true;
+
+  return false;
+}
+
+/**
+ * Extract all observations from transcript entries.
+ * This is the main entry point for both real-time (P3-002) and backfill (P6-001).
+ */
+export function extractObservationsFromEntries(entries: TranscriptEntry[]): ExtractedObservation[] {
+  const observations: ExtractedObservation[] = [];
+  const seenReads = new Set<string>();
+
+  // 1. Extract from tool calls
+  const toolCalls = extractToolCalls(entries);
+  for (const tc of toolCalls) {
+    if (isNoisyToolCall(tc, seenReads)) continue;
+    const obs = classifyToolCall(tc);
+    if (obs) observations.push(obs);
+  }
+
+  // 2. Extract decisions from assistant text
+  const decisions = extractDecisions(entries);
+  for (const decision of decisions) {
+    const title = decision.text.slice(0, 200);
+    const detail = decision.context;
+    observations.push({
+      type: 'decision',
+      title,
+      detail,
+      visibility: classifyVisibility(title, detail),
+      opts: {
+        importance: assignImportance('decision'),
+        originalTokens: estimateTokens(decision.text + decision.context),
+      },
+    });
+  }
+
+  // 3. Extract failed attempts from assistant text
+  const failures = extractFailedAttempts(entries);
+  for (const failure of failures) {
+    const title = failure.text.slice(0, 200);
+    const detail = failure.context;
+    observations.push({
+      type: 'failed_attempt',
+      title,
+      detail,
+      visibility: classifyVisibility(title, detail),
+      opts: {
+        importance: assignImportance('failed_attempt'),
+        originalTokens: estimateTokens(failure.text + failure.context),
+      },
+    });
+  }
+
+  // 4. Extract verification commands
+  const verifications = extractVerificationCommands(toolCalls);
+  for (const vr of verifications) {
+    const title = `${vr.vrType}: ${vr.passed ? 'PASS' : 'FAIL'}`;
+    const detail = vr.command;
+    observations.push({
+      type: 'vr_check',
+      title,
+      detail,
+      visibility: classifyVisibility(title, detail),
+      opts: {
+        vrType: vr.vrType,
+        evidence: vr.result,
+        importance: assignImportance('vr_check', vr.passed ? 'PASS' : 'FAIL'),
+        originalTokens: estimateTokens(vr.result),
+      },
+    });
+  }
+
+  return observations;
+}
+
+/**
+ * Classify a single tool call into an observation (or null if not observation-worthy).
+ */
+function classifyToolCall(tc: ParsedToolCall): ExtractedObservation | null {
+  const result = tc.result ?? '';
+
+  switch (tc.toolName) {
+    case 'Write': {
+      const filePath = tc.input.file_path as string ?? 'unknown';
+      const title = `Created/wrote: ${shortenPath(filePath)}`;
+      return {
+        type: 'file_change',
+        title,
+        detail: null,
+        visibility: classifyVisibility(title, filePath),
+        opts: {
+          filesInvolved: [filePath],
+          importance: assignImportance('file_change'),
+          originalTokens: estimateTokens(result),
+          ...extractLinkedReferences(result + filePath),
+        },
+      };
+    }
+
+    case 'Edit': {
+      const filePath = tc.input.file_path as string ?? 'unknown';
+      const title = `Edited: ${shortenPath(filePath)}`;
+      return {
+        type: 'file_change',
+        title,
+        detail: null,
+        visibility: classifyVisibility(title, filePath),
+        opts: {
+          filesInvolved: [filePath],
+          importance: assignImportance('file_change'),
+          originalTokens: estimateTokens(result),
+          ...extractLinkedReferences(result + filePath),
+        },
+      };
+    }
+
+    case 'Read': {
+      const filePath = tc.input.file_path as string ?? 'unknown';
+      // Only keep reads of interesting files (plan files, CLAUDE.md, etc.)
+      if (filePath.includes('/plans/') || filePath.includes('CLAUDE.md') || filePath.includes('CURRENT.md')) {
+        const title = `Read: ${shortenPath(filePath)}`;
+        return {
+          type: 'discovery',
+          title,
+          detail: null,
+          visibility: classifyVisibility(title, filePath),
+          opts: {
+            filesInvolved: [filePath],
+            importance: assignImportance('discovery'),
+            originalTokens: estimateTokens(result),
+          },
+        };
+      }
+      return null;
+    }
+
+    case 'Bash': {
+      const cmd = (tc.input.command as string ?? '').trim();
+
+      // Git commit
+      if (cmd.includes('git commit')) {
+        const commitMsg = extractCommitMessage(cmd);
+        const isfix = commitMsg.toLowerCase().includes('fix');
+        const title = `Commit: ${commitMsg.slice(0, 150)}`;
+        return {
+          type: isfix ? 'bugfix' : 'feature',
+          title,
+          detail: cmd,
+          visibility: classifyVisibility(title, cmd),
+          opts: {
+            importance: assignImportance(isfix ? 'bugfix' : 'feature'),
+            originalTokens: estimateTokens(result),
+          },
+        };
+      }
+
+      // Pattern scanner
+      if (cmd.includes('pattern-scanner')) {
+        const passed = !result.includes('FAIL') && !result.includes('BLOCKED');
+        const title = `Pattern Scanner: ${passed ? 'PASS' : 'FAIL'}`;
+        const detail = result.slice(0, 500);
+        return {
+          type: 'pattern_compliance',
+          title,
+          detail,
+          visibility: classifyVisibility(title, detail),
+          opts: {
+            evidence: result.slice(0, 500),
+            importance: assignImportance('pattern_compliance', passed ? 'PASS' : 'FAIL'),
+            originalTokens: estimateTokens(result),
+          },
+        };
+      }
+
+      // npm test / vitest
+      if (cmd.includes('npm test') || cmd.includes('vitest')) {
+        const passed = !tc.isError && !result.includes('FAIL');
+        const title = `Tests: ${passed ? 'PASS' : 'FAIL'}`;
+        return {
+          type: 'vr_check',
+          title,
+          detail: cmd,
+          visibility: classifyVisibility(title, cmd),
+          opts: {
+            vrType: 'VR-TEST',
+            evidence: result.slice(0, 500),
+            importance: assignImportance('vr_check', passed ? 'PASS' : 'FAIL'),
+            originalTokens: estimateTokens(result),
+          },
+        };
+      }
+
+      // npm run build / tsc
+      if (cmd.includes('npm run build') || cmd.includes('tsc --noEmit')) {
+        const vrType = cmd.includes('tsc') ? 'VR-TYPE' : 'VR-BUILD';
+        const passed = !tc.isError && !result.includes('error');
+        const title = `${vrType}: ${passed ? 'PASS' : 'FAIL'}`;
+        return {
+          type: 'vr_check',
+          title,
+          detail: cmd,
+          visibility: classifyVisibility(title, cmd),
+          opts: {
+            vrType,
+            evidence: result.slice(0, 500),
+            importance: assignImportance('vr_check', passed ? 'PASS' : 'FAIL'),
+            originalTokens: estimateTokens(result),
+          },
+        };
+      }
+
+      return null;
+    }
+
+    default:
+      return null;
+  }
+}
+
+/**
+ * Extract CR rule, VR type, and plan item references from text.
+ */
+function extractLinkedReferences(text: string): Partial<AddObservationOpts> {
+  const result: Partial<AddObservationOpts> = {};
+
+  const crMatch = text.match(/CR-(\d+)/);
+  if (crMatch) result.crRule = `CR-${crMatch[1]}`;
+
+  const vrMatch = text.match(/VR-([A-Z_]+)/);
+  if (vrMatch) result.vrType = `VR-${vrMatch[1]}`;
+
+  const planMatch = text.match(/P(\d+)-(\d+)/);
+  if (planMatch) result.planItem = `P${planMatch[1]}-${planMatch[2]}`;
+
+  return result;
+}
+
+/**
+ * Extract commit message from a git commit command.
+ */
+function extractCommitMessage(cmd: string): string {
+  // Match -m "message" or -m 'message'
+  const match = cmd.match(/-m\s+["'](.+?)["']/);
+  if (match) return match[1];
+
+  // Match heredoc pattern
+  const heredocMatch = cmd.match(/<<['"]?EOF['"]?\s*\n?([\s\S]*?)EOF/);
+  if (heredocMatch) return heredocMatch[1].trim().split('\n')[0];
+
+  return 'Unknown commit';
+}
+
+/**
+ * Shorten a file path for display.
+ */
+function shortenPath(filePath: string): string {
+  // Remove project root prefix, then common home dir prefix
+  const root = getProjectRoot();
+  if (filePath.startsWith(root + '/')) {
+    return filePath.slice(root.length + 1);
+  }
+  return filePath.replace(/^\/Users\/\w+\//, '~/');
+}
+
+/**
+ * Classify a single tool call for real-time capture (used by PostToolUse hook P3-002).
+ * Applies noise filtering.
+ */
+export function classifyRealTimeToolCall(
+  toolName: string,
+  toolInput: Record<string, unknown>,
+  toolResponse: string,
+  seenReads: Set<string>
+): ExtractedObservation | null {
+  const tc: ParsedToolCall = {
+    toolName,
+    toolUseId: '',
+    input: toolInput,
+    result: toolResponse,
+    isError: false,
+  };
+
+  if (isNoisyToolCall(tc, seenReads)) return null;
+
+  // P2-003: Detect architecture decision patterns in tool responses
+  if (toolResponse && detectDecisionPatterns(toolResponse)) {
+    const firstLine = toolResponse.split('\n')[0].slice(0, 200);
+    const title = `Architecture decision: ${firstLine}`;
+    const detail = toolResponse.slice(0, 1000);
+    return {
+      type: 'decision',
+      title,
+      detail,
+      visibility: classifyVisibility(title, detail),
+      opts: {
+        importance: assignImportance('decision'),
+        originalTokens: estimateTokens(toolResponse),
+        ...extractLinkedReferences(toolResponse),
+      },
+    };
+  }
+
+  return classifyToolCall(tc);
+}
+
+/**
+ * Detect plan progress references in tool responses.
+ * Returns plan items that appear to be completed.
+ */
+export function detectPlanProgress(toolResponse: string): Array<{ planItem: string; status: string }> {
+  const results: Array<{ planItem: string; status: string }> = [];
+  const progressPattern = /(P\d+-\d+)\s*[:\-]?\s*(COMPLETE|PASS|DONE|complete|pass|done)/g;
+  let match;
+  while ((match = progressPattern.exec(toolResponse)) !== null) {
+    results.push({ planItem: match[1], status: 'complete' });
+  }
+  return results;
+}

package/src/page-deps.ts

@@ -0,0 +1,283 @@
+// Copyright (c) 2026 Massu. All rights reserved.
+// Licensed under BSL 1.1 - see LICENSE file for details.
+
+import { readFileSync, existsSync } from 'fs';
+import { resolve } from 'path';
+import type Database from 'better-sqlite3';
+import { getConfig, getProjectRoot } from './config.ts';
+
+export interface PageChain {
+  page: string;
+  route: string;
+  portal: string;
+  components: string[];
+  hooks: string[];
+  routers: string[];
+  tables: string[];
+}
+
+/**
+ * Derive the URL route from a Next.js page file path.
+ * e.g., src/app/orders/page.tsx -> /orders
+ * e.g., src/app/orders/[id]/page.tsx -> /orders/[id]
+ */
+export function deriveRoute(pageFile: string): string {
+  let route = pageFile
+    .replace(/^src\/app/, '')
+    .replace(/\/page\.tsx?$/, '')
+    .replace(/\/page\.jsx?$/, '');
+  return route || '/';
+}
+
+/**
+ * Determine which portal/scope a page belongs to based on its route.
+ * Uses accessScopes from config if available, otherwise infers from route prefix.
+ */
+export function derivePortal(route: string): string {
+  const scopes = getConfig().accessScopes;
+  if (scopes && scopes.length > 0) {
+    for (const scope of scopes) {
+      if (route.startsWith('/' + scope)) return scope;
+    }
+  }
+  // Fallback: use the first path segment as the scope
+  const parts = route.split('/').filter(Boolean);
+  return parts[0] ?? 'default';
+}
+
+/**
+ * Recursively trace imports from a file, collecting components and hooks.
+ */
+function traceImports(
+  startFile: string,
+  dataDb: Database.Database,
+  visited: Set<string>,
+  components: Set<string>,
+  hooks: Set<string>,
+  maxDepth: number = 5
+): void {
+  if (maxDepth <= 0 || visited.has(startFile)) return;
+  visited.add(startFile);
+
+  const imports = dataDb.prepare(
+    'SELECT target_file, imported_names FROM massu_imports WHERE source_file = ?'
+  ).all(startFile) as { target_file: string; imported_names: string }[];
+
+  for (const imp of imports) {
+    const target = imp.target_file;
+
+    // Classify the import
+    if (target.includes('/components/')) {
+      components.add(target);
+    }
+    if (target.includes('/hooks/') || target.match(/use[A-Z]/)) {
+      hooks.add(target);
+    }
+
+    // Recurse into local imports (not node_modules)
+    if (target.startsWith('src/')) {
+      traceImports(target, dataDb, visited, components, hooks, maxDepth - 1);
+    }
+  }
+}
+
+/**
+ * Find routers called by hooks/components via api.* patterns.
+ */
+function findRouterCalls(files: string[]): string[] {
+  const routers = new Set<string>();
+  const projectRoot = getProjectRoot();
+
+  for (const file of files) {
+    const absPath = resolve(projectRoot, file);
+    if (!existsSync(absPath)) continue;
+
+    try {
+      const source = readFileSync(absPath, 'utf-8');
+      const apiCallRegex = /api\.(\w+)\.\w+/g;
+      let match;
+      while ((match = apiCallRegex.exec(source)) !== null) {
+        routers.add(match[1]);
+      }
+    } catch {
+      // Skip unreadable component files
+    }
+  }
+
+  return [...routers];
+}
+
+/**
+ * Find database tables touched by routers.
+ */
+function findTablesFromRouters(routerNames: string[], dataDb: Database.Database): string[] {
+  const tables = new Set<string>();
+
+  // Look up router files from the tRPC index
+  for (const routerName of routerNames) {
+    const procs = dataDb.prepare(
+      'SELECT DISTINCT router_file FROM massu_trpc_procedures WHERE router_name = ?'
+    ).all(routerName) as { router_file: string }[];
+
+    for (const proc of procs) {
+      const absPath = resolve(getProjectRoot(), proc.router_file);
+      if (!existsSync(absPath)) continue;
+
+      try {
+        const source = readFileSync(absPath, 'utf-8');
+        // Match database access patterns from config (default: ctx.db.{table})
+        const dbPattern = getConfig().dbAccessPattern ?? 'ctx.db.{table}';
+        const regexStr = dbPattern
+          .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+          .replace('\\{table\\}', '(\\w+)');
+        const tableRegex = new RegExp(regexStr + '\\.', 'g');
+        let match;
+        while ((match = tableRegex.exec(source)) !== null) {
+          tables.add(match[1]);
+        }
+      } catch {
+        // Skip unreadable source files
+      }
+    }
+  }
+
+  return [...tables];
+}
+
+/**
+ * Build page dependency chains for all page.tsx files.
+ */
+export function buildPageDeps(dataDb: Database.Database, codegraphDb: Database.Database): number {
+  // Clear existing data
+  dataDb.exec('DELETE FROM massu_page_deps');
+
+  // Find all page.tsx files from CodeGraph
+  const pages = codegraphDb.prepare(
+    "SELECT path FROM files WHERE path LIKE 'src/app/%/page.tsx' OR path = 'src/app/page.tsx'"
+  ).all() as { path: string }[];
+
+  const insertStmt = dataDb.prepare(
+    'INSERT INTO massu_page_deps (page_file, route, portal, components, hooks, routers, tables_touched) VALUES (?, ?, ?, ?, ?, ?, ?)'
+  );
+
+  let count = 0;
+
+  const insertAll = dataDb.transaction(() => {
+    for (const page of pages) {
+      const route = deriveRoute(page.path);
+      const portal = derivePortal(route);
+
+      const visited = new Set<string>();
+      const components = new Set<string>();
+      const hooks = new Set<string>();
+
+      traceImports(page.path, dataDb, visited, components, hooks);
+
+      const allFiles = [...visited];
+      const routers = findRouterCalls(allFiles);
+      const tables = findTablesFromRouters(routers, dataDb);
+
+      insertStmt.run(
+        page.path,
+        route,
+        portal,
+        JSON.stringify([...components]),
+        JSON.stringify([...hooks]),
+        JSON.stringify(routers),
+        JSON.stringify(tables)
+      );
+
+      count++;
+    }
+  });
+
+  insertAll();
+  return count;
+}
+
+/**
+ * Get the dependency chain for a specific page.
+ */
+export function getPageChain(dataDb: Database.Database, pageFile: string): PageChain | null {
+  const row = dataDb.prepare('SELECT * FROM massu_page_deps WHERE page_file = ?').get(pageFile) as {
+    page_file: string; route: string; portal: string;
+    components: string; hooks: string; routers: string; tables_touched: string;
+  } | undefined;
+
+  if (!row) return null;
+
+  return {
+    page: row.page_file,
+    route: row.route,
+    portal: row.portal,
+    components: JSON.parse(row.components),
+    hooks: JSON.parse(row.hooks),
+    routers: JSON.parse(row.routers),
+    tables: JSON.parse(row.tables_touched),
+  };
+}
+
+/**
+ * Find all pages affected by a given file (reverse lookup).
+ */
+export function findAffectedPages(dataDb: Database.Database, file: string): PageChain[] {
+  // Check if this file is directly a page
+  const directPage = getPageChain(dataDb, file);
+  if (directPage) return [directPage];
+
+  // Find all pages that import this file (directly or transitively)
+  // First, find who imports this file
+  const importers = dataDb.prepare(
+    'SELECT source_file FROM massu_imports WHERE target_file = ?'
+  ).all(file) as { source_file: string }[];
+
+  const affectedFiles = new Set<string>([file, ...importers.map(i => i.source_file)]);
+
+  // Walk up the import tree to find pages
+  let frontier = [...importers.map(i => i.source_file)];
+  const visited = new Set(frontier);
+  const maxDepth = 10;
+  let depth = 0;
+
+  while (frontier.length > 0 && depth < maxDepth) {
+    const next: string[] = [];
+    for (const f of frontier) {
+      const upstreamImporters = dataDb.prepare(
+        'SELECT source_file FROM massu_imports WHERE target_file = ?'
+      ).all(f) as { source_file: string }[];
+
+      for (const imp of upstreamImporters) {
+        if (!visited.has(imp.source_file)) {
+          visited.add(imp.source_file);
+          affectedFiles.add(imp.source_file);
+          next.push(imp.source_file);
+        }
+      }
+    }
+    frontier = next;
+    depth++;
+  }
+
+  // Now find which pages are in the affected set
+  const allPages = dataDb.prepare('SELECT * FROM massu_page_deps').all() as {
+    page_file: string; route: string; portal: string;
+    components: string; hooks: string; routers: string; tables_touched: string;
+  }[];
+
+  const results: PageChain[] = [];
+  for (const row of allPages) {
+    if (affectedFiles.has(row.page_file)) {
+      results.push({
+        page: row.page_file,
+        route: row.route,
+        portal: row.portal,
+        components: JSON.parse(row.components),
+        hooks: JSON.parse(row.hooks),
+        routers: JSON.parse(row.routers),
+        tables: JSON.parse(row.tables_touched),
+      });
+    }
+  }
+
+  return results;
+}