agentmap 0.7.1 → 0.9.0

package/src/extract/git-status.ts

@@ -0,0 +1,359 @@
+// Parse git diff output and calculate definition-level diff stats.
+// Uses defensive git options for cross-platform reliability.
+
+import { execSync } from 'child_process'
+import { createConsoleLogger } from '../logger.js'
+import type { Definition, DefinitionDiff, DiffHunk, FileDiff, FileDiffStats } from '../types.js'
+import type { Logger } from '../logger.js'
+
+/**
+ * Defensive git options to ensure consistent output across platforms/configs
+ */
+const GIT_DIFF_OPTIONS = [
+  '--no-color',      // No ANSI color codes
+  '--no-ext-diff',   // No external diff tools
+  '--no-textconv',   // No text conversion filters
+  '--no-renames',    // Don't detect renames (simpler parsing)
+].join(' ')
+
+/**
+ * Normalize file path for cross-platform compatibility
+ * - Converts backslashes to forward slashes
+ * - Handles quoted paths from git (e.g., paths with spaces/unicode)
+ */
+function normalizePath(path: string): string {
+  // Git quotes paths with special characters: "path/with spaces/file.ts"
+  if (path.startsWith('"') && path.endsWith('"')) {
+    path = path.slice(1, -1)
+    // Handle escaped characters in quoted paths
+    path = path.replace(/\\"/g, '"').replace(/\\\\/g, '\\')
+  }
+  // Normalize to forward slashes
+  return path.replace(/\\/g, '/')
+}
+
+/**
+ * Safely execute a git command, returning empty string on any error
+ */
+function safeExec(cmd: string, dir: string, logger: Logger): string {
+  try {
+    return execSync(cmd, {
+      cwd: dir,
+      encoding: 'utf8',
+      maxBuffer: 1024 * 1024 * 10, // 10MB
+      stdio: ['pipe', 'pipe', 'pipe'], // Capture stderr too
+    })
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err)
+    logger.warn(`Warning: git diff failed: ${message}`)
+    return ''
+  }
+}
+
+/**
+ * Parse git diff --numstat output for file-level stats
+ * Format: "added<TAB>deleted<TAB>path" or "-<TAB>-<TAB>path" for binary
+ * 
+ * This is much more reliable than parsing full diff output.
+ */
+export function parseNumstat(numstatOutput: string): Map<string, FileDiffStats> {
+  const stats = new Map<string, FileDiffStats>()
+  
+  if (!numstatOutput.trim()) {
+    return stats
+  }
+
+  const lines = numstatOutput.split('\n')
+  
+  for (const line of lines) {
+    if (!line.trim()) continue
+    
+    // Split by tab - format is: added<TAB>deleted<TAB>path
+    const parts = line.split('\t')
+    if (parts.length < 3) continue
+    
+    const [addedStr, deletedStr, ...pathParts] = parts
+    const path = normalizePath(pathParts.join('\t')) // Path might contain tabs (rare but possible)
+    
+    // Binary files show as "-" for both counts - skip them
+    if (addedStr === '-' || deletedStr === '-') {
+      continue
+    }
+    
+    const added = parseInt(addedStr, 10)
+    const deleted = parseInt(deletedStr, 10)
+    
+    // Skip if parsing failed or no changes
+    if (isNaN(added) || isNaN(deleted)) continue
+    if (added === 0 && deleted === 0) continue
+    
+    stats.set(path, { added, deleted })
+  }
+  
+  return stats
+}
+
+/**
+ * Parse a hunk header like "@@ -10,5 +12,7 @@" or "@@ -10 +12,7 @@"
+ */
+export function parseHunkHeader(line: string): DiffHunk | null {
+  // Match: @@ -oldStart[,oldCount] +newStart[,newCount] @@
+  const match = line.match(/@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/)
+  if (!match) return null
+
+  return {
+    oldStart: parseInt(match[1], 10),
+    oldCount: match[2] ? parseInt(match[2], 10) : 1,
+    newStart: parseInt(match[3], 10),
+    newCount: match[4] ? parseInt(match[4], 10) : 1,
+  }
+}
+
+/**
+ * Parse git diff output into structured file diffs (for definition-level analysis)
+ * Only extracts hunk positions, not content.
+ */
+export function parseDiff(diffOutput: string): Map<string, FileDiff> {
+  const files = new Map<string, FileDiff>()
+  
+  if (!diffOutput.trim()) {
+    return files
+  }
+
+  const lines = diffOutput.split('\n')
+
+  let currentFile: string | null = null
+  let hunks: DiffHunk[] = []
+
+  for (const line of lines) {
+    // New file header: "diff --git a/path b/path"
+    if (line.startsWith('diff --git ')) {
+      // Save previous file
+      if (currentFile && hunks.length > 0) {
+        files.set(currentFile, { path: currentFile, hunks })
+      }
+      
+      // Extract path from "diff --git a/path b/path"
+      // Use the b/ path (destination) as the canonical path
+      const match = line.match(/diff --git a\/.+ b\/(.+)/)
+      if (match) {
+        currentFile = normalizePath(match[1])
+      } else {
+        currentFile = null
+      }
+      hunks = []
+      continue
+    }
+
+    // Skip binary files indicator
+    if (line.startsWith('Binary files ')) {
+      currentFile = null
+      hunks = []
+      continue
+    }
+
+    // Hunk header
+    if (line.startsWith('@@') && currentFile) {
+      try {
+        const hunk = parseHunkHeader(line)
+        if (hunk) {
+          hunks.push(hunk)
+        }
+      } catch {
+        // Skip malformed hunk headers
+      }
+    }
+  }
+
+  // Save last file
+  if (currentFile && hunks.length > 0) {
+    files.set(currentFile, { path: currentFile, hunks })
+  }
+
+  return files
+}
+
+/**
+ * Get file-level diff stats using --numstat (most reliable)
+ */
+export function getFileStats(dir: string, logger: Logger = createConsoleLogger()): Map<string, FileDiffStats> {
+  const cmd = `git diff ${GIT_DIFF_OPTIONS} --numstat HEAD`
+  const output = safeExec(cmd, dir, logger)
+  return parseNumstat(output)
+}
+
+/**
+ * Get hunk-level diff for definition analysis
+ */
+export function getHunkDiff(dir: string, logger: Logger = createConsoleLogger()): Map<string, FileDiff> {
+  const cmd = `git diff ${GIT_DIFF_OPTIONS} --unified=0 HEAD`
+  const output = safeExec(cmd, dir, logger)
+  return parseDiff(output)
+}
+
+/**
+ * Combined function to get all diff data needed
+ * Returns both file stats and hunk data, with error isolation.
+ * Filters out submodule paths to prevent misleading stats (submodule pointer
+ * changes show as 1/1 in numstat, and produce "Subproject commit" pseudo-diffs).
+ */
+export function getAllDiffData(dir: string, submodulePaths?: Set<string>): {
+  fileStats: Map<string, FileDiffStats>
+  fileDiffs: Map<string, FileDiff>
+}
+export function getAllDiffData(dir: string, submodulePaths: Set<string> | undefined, logger: Logger): {
+  fileStats: Map<string, FileDiffStats>
+  fileDiffs: Map<string, FileDiff>
+}
+export function getAllDiffData(
+  dir: string,
+  submodulePaths?: Set<string>,
+  logger: Logger = createConsoleLogger()
+): {
+  fileStats: Map<string, FileDiffStats>
+  fileDiffs: Map<string, FileDiff>
+} {
+  // Get file stats (for file-level +N-M display)
+  let fileStats: Map<string, FileDiffStats>
+  try {
+    fileStats = getFileStats(dir, logger)
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err)
+    logger.warn(`Warning: failed to get file stats: ${message}`)
+    fileStats = new Map()
+  }
+
+  // Get hunk data (for definition-level analysis)
+  let fileDiffs: Map<string, FileDiff>
+  try {
+    fileDiffs = getHunkDiff(dir, logger)
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err)
+    logger.warn(`Warning: failed to get hunk diff: ${message}`)
+    fileDiffs = new Map()
+  }
+
+  // Filter out submodule paths - their diff output is misleading:
+  // - numstat shows 1/1 for pointer changes (not real line counts)
+  // - unified diff shows "Subproject commit" pseudo-patches
+  if (submodulePaths && submodulePaths.size > 0) {
+    for (const subPath of submodulePaths) {
+      fileStats.delete(subPath)
+      fileDiffs.delete(subPath)
+    }
+  }
+
+  return { fileStats, fileDiffs }
+}
+
+/**
+ * Calculate diff stats for a single definition based on file hunks
+ * 
+ * A definition is "added" if all its lines are new additions.
+ * Otherwise it's "updated" if any of its lines were changed.
+ */
+export function calculateDefinitionDiff(
+  def: Definition,
+  hunks: DiffHunk[]
+): DefinitionDiff | null {
+  try {
+    const defStart = def.line
+    const defEnd = def.endLine
+    const defLineCount = defEnd - defStart + 1
+
+    let addedInDef = 0
+    let deletedInDef = 0
+
+    for (const hunk of hunks) {
+      // Check if this hunk's NEW lines overlap with definition range
+      const hunkNewStart = hunk.newStart
+      const hunkNewEnd = hunk.newStart + hunk.newCount - 1
+
+      // Calculate overlap between [defStart, defEnd] and [hunkNewStart, hunkNewEnd]
+      const overlapStart = Math.max(defStart, hunkNewStart)
+      const overlapEnd = Math.min(defEnd, hunkNewEnd)
+
+      if (overlapStart <= overlapEnd) {
+        // There's overlap - count the added lines in this overlap
+        const addedLines = overlapEnd - overlapStart + 1
+        addedInDef += addedLines
+      }
+
+      // For deleted lines, check if hunk's new position overlaps with definition
+      if (hunk.oldCount > 0) {
+        if (hunkNewStart <= defEnd && hunkNewEnd >= defStart) {
+          deletedInDef += hunk.oldCount
+        }
+      }
+    }
+
+    // No changes in this definition
+    if (addedInDef === 0 && deletedInDef === 0) {
+      return null
+    }
+
+    // Determine status
+    // "added" = the entire definition consists of new lines AND nothing was deleted
+    const status = addedInDef >= defLineCount && deletedInDef === 0 ? 'added' : 'updated'
+
+    return {
+      status,
+      added: addedInDef,
+      deleted: deletedInDef,
+    }
+  } catch {
+    // Any calculation error - return null (no diff info)
+    return null
+  }
+}
+
+/**
+ * Calculate total diff stats for a file by summing all hunks
+ * @deprecated Use getFileStats() with --numstat instead for reliability
+ */
+export function calculateFileDiff(hunks: DiffHunk[]): FileDiffStats | null {
+  if (hunks.length === 0) {
+    return null
+  }
+
+  let added = 0
+  let deleted = 0
+
+  for (const hunk of hunks) {
+    added += hunk.newCount
+    deleted += hunk.oldCount
+  }
+
+  if (added === 0 && deleted === 0) {
+    return null
+  }
+
+  return { added, deleted }
+}
+
+/**
+ * Apply diff information to definitions for a file
+ */
+export function applyDiffToDefinitions(
+  definitions: Definition[],
+  fileDiff: FileDiff | undefined
+): Definition[] {
+  if (!fileDiff || fileDiff.hunks.length === 0) {
+    return definitions
+  }
+
+  return definitions.map(def => {
+    try {
+      const diff = calculateDefinitionDiff(def, fileDiff.hunks)
+      if (diff) {
+        return { ...def, diff }
+      }
+    } catch {
+      // Skip diff for this definition on error
+    }
+    return def
+  })
+}
+
+// Legacy exports for backwards compatibility
+export { getFileStats as getGitDiffAll }

package/src/extract/markdown.test.ts

@@ -0,0 +1,159 @@
+// Tests for markdown description extraction.
+
+import { describe, expect, test } from 'bun:test'
+import { writeFile, unlink, mkdir } from 'fs/promises'
+import { join } from 'path'
+import { tmpdir } from 'os'
+import { extractMarkdownDescription } from './markdown.js'
+
+const TEST_DIR = join(tmpdir(), 'agentmap-markdown-test')
+
+async function testMarkdown(content: string): Promise<string | null> {
+  await mkdir(TEST_DIR, { recursive: true })
+  const filepath = join(TEST_DIR, 'README.md')
+  await writeFile(filepath, content, 'utf8')
+  try {
+    return await extractMarkdownDescription(filepath)
+  } finally {
+    await unlink(filepath).catch(() => {})
+  }
+}
+
+describe('Markdown extraction', () => {
+  test('simple heading and paragraph', async () => {
+    const desc = await testMarkdown(`# My Project
+
+This is a description of my project.
+It does amazing things.
+`)
+    expect(desc).toMatchInlineSnapshot(`
+"My Project
+This is a description of my project.
+It does amazing things."
+`)
+  })
+
+  test('ignores HTML comments', async () => {
+    const desc = await testMarkdown(`<!-- This is a comment -->
+# Title
+
+Some content here.
+`)
+    expect(desc).toMatchInlineSnapshot(`
+"Title
+Some content here."
+`)
+  })
+
+  test('ignores badge images', async () => {
+    const desc = await testMarkdown(`![Build Status](https://shields.io/badge/build-passing)
+![Coverage](https://img.shields.io/coverage/80)
+
+# My Library
+
+A useful library.
+`)
+    expect(desc).toMatchInlineSnapshot(`
+"My Library
+A useful library."
+`)
+  })
+
+  test('handles lists', async () => {
+    const desc = await testMarkdown(`# Features
+
+- Feature one
+- Feature two
+- Feature three
+`)
+    expect(desc).toMatchInlineSnapshot(`
+"Features
+- Feature one
+- Feature two
+- Feature three"
+`)
+  })
+
+  test('handles code blocks', async () => {
+    const desc = await testMarkdown(`# Usage
+
+Install the package:
+
+\`\`\`bash
+npm install mypackage
+\`\`\`
+`)
+    expect(desc).toMatchInlineSnapshot(`
+"Usage
+Install the package:
+\`\`\`bash
+npm install mypackage
+\`\`\`"
+`)
+  })
+
+  test('handles blockquotes', async () => {
+    const desc = await testMarkdown(`# Quote Example
+
+> This is a blockquote
+> with multiple lines
+`)
+    expect(desc).toMatchInlineSnapshot(`
+"Quote Example
+> This is a blockquote
+with multiple lines"
+`)
+  })
+
+  test('truncates long content with indicator', async () => {
+    // Create 40 lines without blank lines between (so all fit in first 50 lines read)
+    const lines = Array.from({ length: 40 }, (_, i) => `- Item ${i + 1}`).join('\n')
+    const desc = await testMarkdown(`# Title\n\n${lines}`)
+    const descLines = desc?.split('\n') ?? []
+    // 20 content lines + 1 truncation indicator (Title + 19 items, then indicator)
+    expect(descLines.length).toBe(21)
+    expect(descLines[20]).toBe('... and 21 more lines')
+  })
+
+  test('returns null for empty markdown', async () => {
+    const desc = await testMarkdown(``)
+    expect(desc).toBeNull()
+  })
+
+  test('returns null for only HTML comments', async () => {
+    const desc = await testMarkdown(`<!-- Just a comment -->
+
+<!-- Another comment -->
+`)
+    expect(desc).toBeNull()
+  })
+
+  test('handles mixed content', async () => {
+    const desc = await testMarkdown(`<!-- Header comment -->
+![Badge](https://example.com/badge.svg)
+
+# agentmap
+
+A compact, YAML-based inventory of your codebase.
+
+## Features
+
+- Fast scanning
+- Tree-sitter parsing
+
+\`\`\`bash
+npm install agentmap
+\`\`\`
+`)
+    expect(desc).toMatchInlineSnapshot(`
+"agentmap
+A compact, YAML-based inventory of your codebase.
+Features
+- Fast scanning
+- Tree-sitter parsing
+\`\`\`bash
+npm install agentmap
+\`\`\`"
+`)
+  })
+})

package/src/extract/markdown.ts

@@ -0,0 +1,202 @@
+// Extract description from markdown files using marked AST.
+
+import { Lexer, type Token, type Tokens } from 'marked'
+import { readFirstLines } from './utils.js'
+
+const MAX_LINES = 50
+const MAX_DESC_LINES = 20
+
+/**
+ * Truncate lines to MAX_DESC_LINES, adding indicator if truncated
+ */
+function truncateDescription(lines: string[]): string {
+  const trimmed = lines.join('\n').trim()
+  const trimmedLines = trimmed.split('\n')
+  
+  if (trimmedLines.length <= MAX_DESC_LINES) {
+    return trimmed
+  }
+  
+  const truncated = trimmedLines.slice(0, MAX_DESC_LINES)
+  const remaining = trimmedLines.length - MAX_DESC_LINES
+  truncated.push(`... and ${remaining} more lines`)
+  return truncated.join('\n')
+}
+
+/**
+ * Extract plain text from inline tokens, skipping images.
+ */
+function extractInlineText(tokens: Token[] | undefined): string {
+  if (!tokens) return ''
+  
+  const parts: string[] = []
+  for (const token of tokens) {
+    // Skip images
+    if (token.type === 'image') {
+      continue
+    }
+    
+    // Handle text
+    if (token.type === 'text') {
+      const text = token as Tokens.Text
+      if (text.text) {
+        parts.push(text.text)
+      }
+      continue
+    }
+    
+    // Handle links - extract the text content
+    if (token.type === 'link') {
+      const link = token as Tokens.Link
+      if (link.text) {
+        parts.push(link.text)
+      }
+      continue
+    }
+    
+    // Handle strong/em - extract nested text
+    if (token.type === 'strong' || token.type === 'em') {
+      const styled = token as Tokens.Strong | Tokens.Em
+      const inner = extractInlineText(styled.tokens)
+      if (inner) {
+        parts.push(inner)
+      }
+      continue
+    }
+    
+    // Handle codespan (inline code)
+    if (token.type === 'codespan') {
+      const code = token as Tokens.Codespan
+      if (code.text) {
+        parts.push('`' + code.text + '`')
+      }
+      continue
+    }
+  }
+  
+  return parts.join('')
+}
+
+/**
+ * Extract text content from markdown tokens recursively.
+ * Skips HTML, comments, and images. Returns plain text lines.
+ */
+function extractTextFromTokens(tokens: Token[]): string[] {
+  const lines: string[] = []
+  
+  for (const token of tokens) {
+    // Skip HTML (includes comments)
+    if (token.type === 'html') {
+      continue
+    }
+    
+    // Skip spaces
+    if (token.type === 'space') {
+      continue
+    }
+    
+    // Handle headings - extract inline text
+    if (token.type === 'heading') {
+      const heading = token as Tokens.Heading
+      const text = extractInlineText(heading.tokens)
+      if (text) {
+        lines.push(text)
+      }
+      continue
+    }
+    
+    // Handle paragraphs - extract inline text (skips images)
+    if (token.type === 'paragraph') {
+      const para = token as Tokens.Paragraph
+      const text = extractInlineText(para.tokens)
+      if (text) {
+        lines.push(text)
+      }
+      continue
+    }
+    
+    // Handle lists - extract text from items
+    if (token.type === 'list') {
+      const list = token as Tokens.List
+      for (const item of list.items) {
+        const text = extractInlineText(item.tokens)
+        if (text) {
+          lines.push('- ' + text.split('\n')[0])
+        }
+      }
+      continue
+    }
+    
+    // Handle blockquotes - extract nested tokens
+    if (token.type === 'blockquote') {
+      const quote = token as Tokens.Blockquote
+      if (quote.tokens) {
+        const nestedLines = extractTextFromTokens(quote.tokens)
+        lines.push(...nestedLines.map(l => '> ' + l))
+      }
+      continue
+    }
+    
+    // Handle code blocks - include with fence
+    if (token.type === 'code') {
+      const code = token as Tokens.Code
+      if (code.lang) {
+        lines.push('```' + code.lang)
+      } else {
+        lines.push('```')
+      }
+      lines.push(...code.text.split('\n'))
+      lines.push('```')
+      continue
+    }
+    
+    // Handle text tokens (inline)
+    if (token.type === 'text') {
+      const text = token as Tokens.Text
+      if (text.text) {
+        lines.push(text.text)
+      }
+      continue
+    }
+  }
+  
+  return lines
+}
+
+/**
+ * Extract description from a markdown file using marked lexer.
+ * Parses first N lines, extracts plain text from AST nodes,
+ * ignoring HTML comments and images.
+ * Falls back to raw content if parsing fails.
+ */
+export async function extractMarkdownDescription(filepath: string): Promise<string | null> {
+  const head = await readFirstLines(filepath, MAX_LINES)
+  if (head === null) {
+    // File couldn't be read - skip silently
+    return null
+  }
+  
+  try {
+    // Parse markdown to tokens using marked lexer
+    const lexer = new Lexer()
+    const tokens = lexer.lex(head)
+    
+    // Extract text from tokens
+    const lines = extractTextFromTokens(tokens)
+    
+    // Filter empty lines
+    const contentLines = lines.filter(l => l.trim() !== '')
+    if (contentLines.length === 0) {
+      return null
+    }
+    
+    return truncateDescription(contentLines)
+  } catch {
+    // Fallback: return raw content if parsing fails
+    const lines = head.split('\n').filter(l => l.trim() !== '')
+    if (lines.length === 0) {
+      return null
+    }
+    return truncateDescription(lines)
+  }
+}