@steno-ai/engine 0.1.11 → 0.1.13

package/src/extraction/codebase-extractor.ts

@@ -0,0 +1,217 @@
+import type { LLMAdapter } from '../adapters/llm.js';
+import type { ExtractionResult, ExtractedFact, ExtractedEntity, ExtractedEdge } from './types.js';
+import type { ExtractionTier, EdgeType, SourceType } from '../config.js';
+import { buildCodebaseFactExtractionPrompt, buildCodebaseGraphExtractionPrompt, CODEBASE_ENTITY_TYPES } from './codebase-prompts.js';
+import { normalizeEntityName } from './llm-extractor.js';
+
+export interface CodebaseExtractorConfig {
+  llm: LLMAdapter;
+  tier: ExtractionTier;
+}
+
+const VALID_CODEBASE_SOURCE_TYPES = new Set<string>([
+  'file_analysis', 'ast_parse', 'git_history', 'dependency_scan',
+]);
+
+const VALID_CODEBASE_EDGE_TYPES = new Set<string>([
+  'imports', 'implements', 'owns', 'configures', 'tested_by',
+  'depends_on', 'extends', 'associative', 'hierarchical',
+]);
+
+/**
+ * Extract architectural facts from code content.
+ * Similar to extractWithLLM but uses codebase-specific prompts
+ * and skips conversation-oriented logic.
+ */
+export async function extractCodebaseFacts(
+  config: CodebaseExtractorConfig,
+  input: string,
+): Promise<ExtractionResult> {
+  let totalTokensIn = 0;
+  let totalTokensOut = 0;
+
+  // ── PASS 1: Codebase fact extraction — ONE LLM call (no sliding window) ──
+  // Code doesn't need pronoun resolution. Send the full input as one chunk
+  // to minimize LLM calls (1 call vs 6+ with sliding window).
+  // Cap at 30K chars (~7.5K tokens) to fit gpt-4.1-mini's context.
+  const cappedInput = input.slice(0, 30_000);
+
+  let factEntries: Array<{
+    text: string;
+    importance: number;
+    sourceChunk: string;
+    sourceType: SourceType;
+  }> = [];
+
+  try {
+    const messages = buildCodebaseFactExtractionPrompt(cappedInput);
+    const response = await config.llm.complete(messages, { temperature: 0, responseFormat: 'json' });
+    totalTokensIn += response.tokensInput;
+    totalTokensOut += response.tokensOutput;
+
+    const parsed = JSON.parse(response.content) as Record<string, unknown>;
+    const rawFacts = Array.isArray(parsed.facts) ? parsed.facts : [];
+
+    for (const f of rawFacts) {
+      if (typeof f === 'string') {
+        const trimmed = f.trim();
+        if (trimmed.length > 0) {
+          factEntries.push({ text: trimmed, importance: 0.5, sourceChunk: cappedInput.slice(0, 500), sourceType: 'file_analysis' });
+        }
+      } else if (f && typeof f === 'object') {
+        const obj = f as Record<string, unknown>;
+        const text = (typeof obj.t === 'string' ? obj.t : typeof obj.text === 'string' ? obj.text : '').trim();
+        const importance = typeof obj.i === 'number' ? obj.i : typeof obj.importance === 'number' ? obj.importance : 0.5;
+        const rawSrc = typeof obj.src === 'string' ? obj.src : 'file_analysis';
+        const sourceType: SourceType = VALID_CODEBASE_SOURCE_TYPES.has(rawSrc)
+          ? rawSrc as SourceType
+          : 'file_analysis';
+
+        if (text.length > 0) {
+          factEntries.push({
+            text,
+            importance: Math.max(0, Math.min(1, importance)),
+            sourceChunk: cappedInput.slice(0, 500),
+            sourceType,
+          });
+        }
+      }
+    }
+  } catch (err) {
+    console.error('[steno-codebase] Fact extraction failed:', err instanceof Error ? err.message : err);
+  }
+
+  // Deduplicate facts by content similarity
+  const seenContent = new Set<string>();
+  factEntries = factEntries.filter(e => {
+    const key = e.text.toLowerCase().trim();
+    if (seenContent.has(key)) return false;
+    seenContent.add(key);
+    return true;
+  });
+
+  if (factEntries.length === 0) {
+    return emptyResult(config.tier, config.llm.model);
+  }
+
+  // Build ExtractedFact objects
+  const facts: ExtractedFact[] = factEntries.map(({ text, importance, sourceChunk, sourceType }) => ({
+    content: text,
+    importance,
+    confidence: 0.8,
+    sourceType,
+    modality: 'code' as const,
+    tags: ['codebase'],
+    originalContent: input,
+    entityCanonicalNames: [],
+    sourceChunk,
+  }));
+
+  // Add contextual embeddings
+  for (const fact of facts) {
+    const src = fact.sourceChunk ?? input;
+    const contextPrefix = src.length > 100
+      ? `Codebase context: ${src.slice(0, 200).trim()}... | Fact: `
+      : `Codebase context: ${src.trim()} | Fact: `;
+    fact.contextualContent = contextPrefix + fact.content;
+  }
+
+  // ── PASS 2: Architecture graph extraction ──
+  let entities: ExtractedEntity[] = [];
+  let edges: ExtractedEdge[] = [];
+
+  try {
+    const factStrings = factEntries.map(e => e.text);
+    const graphMessages = buildCodebaseGraphExtractionPrompt(factStrings);
+    const graphResponse = await config.llm.complete(graphMessages, { temperature: 0, responseFormat: 'json' });
+    totalTokensIn += graphResponse.tokensInput;
+    totalTokensOut += graphResponse.tokensOutput;
+
+    const graphParsed = JSON.parse(graphResponse.content) as Record<string, unknown>;
+
+    // Parse entities
+    const seenEntities = new Set<string>();
+    if (Array.isArray(graphParsed.entities)) {
+      for (const e of graphParsed.entities as unknown[]) {
+        if (!e || typeof (e as Record<string, unknown>).name !== 'string') continue;
+        const entity = e as Record<string, unknown>;
+        const canonical = normalizeEntityName(entity.name as string);
+        if (canonical.length === 0 || seenEntities.has(canonical)) continue;
+        seenEntities.add(canonical);
+
+        const rawType = String(entity.entity_type ?? entity.type ?? 'module');
+        const entityType = CODEBASE_ENTITY_TYPES.includes(rawType) ? rawType : 'module';
+
+        entities.push({
+          name: entity.name as string,
+          entityType,
+          canonicalName: canonical,
+          properties: {},
+        });
+      }
+    }
+
+    // Parse edges
+    if (Array.isArray(graphParsed.edges)) {
+      for (const r of graphParsed.edges as unknown[]) {
+        if (!r) continue;
+        const rel = r as Record<string, unknown>;
+        const rawSource = typeof rel.source === 'string' ? rel.source : null;
+        const rawTarget = typeof rel.target === 'string' ? rel.target : null;
+        if (!rawSource || !rawTarget) continue;
+        const source = normalizeEntityName(rawSource);
+        const target = normalizeEntityName(rawTarget);
+        if (!source || !target) continue;
+
+        const rawEdgeType = typeof rel.edge_type === 'string' ? rel.edge_type : 'associative';
+        const edgeType: EdgeType = VALID_CODEBASE_EDGE_TYPES.has(rawEdgeType)
+          ? rawEdgeType as EdgeType
+          : 'associative';
+
+        edges.push({
+          sourceName: source,
+          targetName: target,
+          relation: String(rel.relation ?? 'related_to'),
+          edgeType,
+          confidence: 0.8,
+        });
+      }
+    }
+
+    // Link entities to facts by text match
+    for (const fact of facts) {
+      const contentLower = fact.content.toLowerCase();
+      for (const entity of entities) {
+        if (entity.canonicalName.length >= 3 && contentLower.includes(entity.canonicalName)) {
+          fact.entityCanonicalNames!.push(entity.canonicalName);
+        }
+      }
+    }
+  } catch {
+    // Graph pass failed — facts are still useful without graph
+  }
+
+  return {
+    facts,
+    entities,
+    edges,
+    tier: config.tier,
+    confidence: 0.8,
+    tokensInput: totalTokensIn,
+    tokensOutput: totalTokensOut,
+    model: config.llm.model,
+  };
+}
+
+function emptyResult(tier: ExtractionTier, model: string): ExtractionResult {
+  return {
+    facts: [],
+    entities: [],
+    edges: [],
+    tier,
+    confidence: 0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model,
+  };
+}

package/src/extraction/codebase-prompts.ts

@@ -0,0 +1,141 @@
+import type { LLMMessage } from '../adapters/llm.js';
+
+// =============================================================================
+// PASS 1: CODEBASE FACT EXTRACTION — Architectural facts from code
+// =============================================================================
+
+export const CODEBASE_FACT_EXTRACTION_PROMPT = `You are a codebase analysis engine. Extract architectural and structural facts from source code, configuration files, and project documentation.
+
+## WHAT TO EXTRACT
+
+Focus on facts that help an AI assistant understand the codebase:
+
+1. **File structure patterns**: How the project is organized (monorepo layout, feature-based folders, barrel exports, etc.)
+2. **Framework choices**: Which frameworks, runtimes, and major libraries are used, and how they're configured
+3. **API conventions**: Route naming patterns, middleware usage, request/response shapes, authentication patterns
+4. **Error handling patterns**: How errors are caught, logged, and propagated — custom error classes, error boundaries, fallback strategies
+5. **Dependencies**: Key packages and their roles (ORM, auth, state management, testing, etc.)
+6. **Build & deploy config**: Build tools, CI/CD setup, environment variable patterns, deployment targets
+7. **Code conventions**: Naming conventions, module boundaries, shared utilities, type patterns
+8. **Data flow**: How data moves through the system — stores, caches, API layers, database access patterns
+
+## RULES
+
+1. Extract SELF-CONTAINED atomic facts. Each fact must be understandable without the original code.
+2. Be SPECIFIC — reference actual file paths, package names, and function signatures when visible.
+   BAD: "The project uses a database"
+   GOOD: "The project uses Supabase via @supabase/supabase-js with row-level security enabled in lib/supabase/client.ts"
+3. Write all facts in third person using the project/codebase as the subject.
+   - "The project uses Next.js App Router with server components"
+   - "The API routes follow REST conventions under app/api/"
+4. Focus on ARCHITECTURAL facts — patterns, conventions, and structure. Skip trivial implementation details.
+5. For configuration files, extract the meaningful settings, not every line.
+
+## OUTPUT
+
+Return ONLY a JSON object:
+{"facts": [{"t": "fact text here", "i": 0.7, "src": "file_analysis"}, ...]}
+
+Score importance (i) from 0.0 to 1.0:
+- 0.9-1.0: Core framework choice, primary database, authentication strategy, deployment target
+- 0.7-0.8: API patterns, state management, key dependencies, build configuration
+- 0.4-0.6: Utility patterns, naming conventions, folder structure details
+- 0.1-0.3: Minor config settings, dev-only tooling, formatting preferences
+
+Source type (src) must be one of: "file_analysis", "ast_parse", "git_history", "dependency_scan"
+
+Nothing else. No explanation, no markdown.`;
+
+// =============================================================================
+// PASS 2: CODEBASE GRAPH EXTRACTION — Architecture graph from facts
+// =============================================================================
+
+export const CODEBASE_GRAPH_EXTRACTION_PROMPT = `You are an architecture graph builder. Given facts about a codebase, extract entities and their relationships to build an architecture graph.
+
+## ENTITIES
+
+Extract packages, modules, services, configurations, and other structural components.
+
+Entity types to use: package, module, config, service, api_route, database, library, test_suite, build_tool, runtime
+
+Examples:
+- {"name": "Next.js App Router", "entity_type": "runtime"}
+- {"name": "lib/supabase", "entity_type": "module"}
+- {"name": "Prisma", "entity_type": "library"}
+- {"name": "app/api/auth", "entity_type": "api_route"}
+- {"name": "jest.config.ts", "entity_type": "config"}
+
+DO extract: Packages, modules, config files, services, API route groups, databases, key libraries
+DO NOT extract: Individual variables, single functions (unless they are exported utilities), line-level details
+
+## RELATIONSHIPS
+
+Extract structural relationships between entities. Use these edge types:
+
+- imports: A imports/depends on B at code level
+- implements: A implements interface/pattern B
+- owns: A contains/owns B (package owns modules, module owns routes)
+- configures: A configures/sets up B (config file configures a library)
+- tested_by: A is tested by B (module has corresponding test suite)
+- depends_on: A requires B at build/runtime level
+- extends: A extends/inherits from B
+
+## OUTPUT
+
+Return ONLY a JSON object:
+{
+  "entities": [
+    {"name": "lib/supabase", "entity_type": "module"},
+    {"name": "Prisma", "entity_type": "library"}
+  ],
+  "edges": [
+    {"source": "lib/supabase", "target": "supabase-js", "relation": "imports", "edge_type": "imports"},
+    {"source": "app/api", "target": "lib/supabase", "relation": "uses_for_data", "edge_type": "depends_on"},
+    {"source": "jest.config.ts", "target": "jest", "relation": "configures", "edge_type": "configures"}
+  ]
+}
+
+Entity names should be clean identifiers — file paths, package names, or descriptive labels.
+Return ONLY valid JSON.`;
+
+// =============================================================================
+// CODEBASE ENTITY TYPES
+// =============================================================================
+
+export const CODEBASE_ENTITY_TYPES = [
+  'package',
+  'module',
+  'config',
+  'service',
+  'api_route',
+  'database',
+  'library',
+  'test_suite',
+  'build_tool',
+  'runtime',
+];
+
+// =============================================================================
+// PROMPT BUILDERS
+// =============================================================================
+
+/**
+ * Build the codebase fact extraction prompt (Pass 1).
+ */
+export function buildCodebaseFactExtractionPrompt(input: string): LLMMessage[] {
+  return [
+    { role: 'system', content: CODEBASE_FACT_EXTRACTION_PROMPT },
+    { role: 'user', content: `Extract architectural facts from this code/configuration:\n\n${input}` },
+  ];
+}
+
+/**
+ * Build the codebase graph extraction prompt (Pass 2).
+ */
+export function buildCodebaseGraphExtractionPrompt(facts: string[]): LLMMessage[] {
+  const factsList = facts.map((f, i) => `${i + 1}. ${f}`).join('\n');
+  return [
+    { role: 'system', content: CODEBASE_GRAPH_EXTRACTION_PROMPT },
+    { role: 'user', content: `Build an architecture graph from these codebase facts:\n\n${factsList}` },
+  ];
+}

package/src/extraction/index.ts

@@ -1,9 +1,11 @@
 export * from './types.js';
 export * from './hasher.js';
 export * from './prompts.js';
+export * from './codebase-prompts.js';
 export * from './heuristic.js';
 export * from './contradiction.js';
 export * from './entity-extractor.js';
 export * from './llm-extractor.js';
+export * from './codebase-extractor.js';
 export * from './dedup.js';
 export * from './pipeline.js';

package/src/extraction/llm-extractor.ts

@@ -246,7 +246,11 @@ function emptyResult(tier: ExtractionTier, model: string): ExtractionResult {
 function isValidEdgeType(t: unknown): t is EdgeType {
   return (
     typeof t === 'string' &&
-    ['associative', 'causal', 'temporal', 'contradictory', 'hierarchical'].includes(t)
+    [
+      'associative', 'causal', 'temporal', 'contradictory', 'hierarchical',
+      'updates', 'extends', 'derives', 'precedes', 'depends_on', 'deadline',
+      'imports', 'implements', 'owns', 'configures', 'tested_by',
+    ].includes(t)
   );
 }