agentmap 0.8.0 → 0.9.0

package/src/scanner.ts

@@ -0,0 +1,364 @@
+// Scan directory for files with header comments/docstrings and recurse into submodules.
+
+import { execSync } from 'child_process'
+import { realpathSync } from 'fs'
+import pLimit from 'p-limit'
+import picomatch from 'picomatch'
+import { readFile } from 'fs/promises'
+import { join, normalize } from 'path'
+import { extractMarkerFromCode, extractMarkdownDescription } from './extract/marker.js'
+import { extractDefinitions } from './extract/definitions.js'
+import { getAllDiffData, applyDiffToDefinitions } from './extract/git-status.js'
+import { getSubmodules, getSubmodulePaths } from './extract/submodules.js'
+import { createConsoleLogger } from './logger.js'
+import { parseCode, detectLanguage, LANGUAGE_EXTENSIONS } from './parser/index.js'
+import type { FileResult, GenerateOptions, FileDiff, FileDiffStats, SubmoduleInfo } from './types.js'
+import type { Logger } from './logger.js'
+
+/**
+ * Maximum number of files to process (safety limit)
+ * If exceeded, returns empty results to avoid scanning huge directories
+ */
+const MAX_FILES = 5_000_000
+
+/**
+ * Supported file extensions (from LANGUAGE_EXTENSIONS)
+ */
+const SUPPORTED_EXTENSIONS = new Set(Object.keys(LANGUAGE_EXTENSIONS))
+
+/**
+ * Check if a file has a supported extension
+ */
+function isSupportedFile(filepath: string): boolean {
+  const ext = filepath.slice(filepath.lastIndexOf('.'))
+  return SUPPORTED_EXTENSIONS.has(ext)
+}
+
+/**
+ * Check if a file is a README file (case-insensitive, with or without .md extension)
+ */
+function isReadmeFile(filepath: string): boolean {
+  const filename = filepath.split(/[/\\]/).pop()?.toLowerCase() ?? ''
+  return filename === 'readme.md' || filename === 'readme'
+}
+
+/**
+ * A tracked file with its blob SHA (for dedup) and normalized path.
+ */
+interface GitFileEntry {
+  path: string
+  sha: string
+}
+
+/**
+ * Get tracked files using git ls-files -z -s.
+ * NUL-delimited for safe cross-platform parsing of any filename.
+ * Filters out symlinks (mode 120000).
+ * Returns path + blob SHA for duplicate detection.
+ *
+ * Format per entry: "<mode> <sha> <stage>\t<path>\0"
+ */
+function getGitFiles(dir: string): GitFileEntry[] {
+  const maxBuffer = 1024 * 10000000
+  try {
+    const stdout = execSync('git ls-files -z -s', {
+      cwd: dir,
+      maxBuffer,
+      encoding: 'utf8',
+    })
+
+    const results: GitFileEntry[] = []
+
+    // Split on NUL byte, filter empty trailing entry
+    const entries = stdout.split('\0').filter(Boolean)
+    for (const entry of entries) {
+      // Format: "<mode> <sha> <stage>\t<path>"
+      const tabIdx = entry.indexOf('\t')
+      if (tabIdx === -1) continue
+
+      const meta = entry.slice(0, tabIdx)
+      const path = entry.slice(tabIdx + 1)
+
+      const spaceIdx = meta.indexOf(' ')
+      if (spaceIdx === -1) continue
+
+      const mode = meta.slice(0, spaceIdx)
+      const sha = meta.slice(spaceIdx + 1, meta.indexOf(' ', spaceIdx + 1))
+
+      // Skip symlinks (mode 120000)
+      if (mode === '120000') continue
+
+      results.push({ path: normalize(path), sha })
+    }
+
+    return results
+  } catch {
+    return []
+  }
+}
+
+/**
+ * Build a map of blob SHA → shortest path for duplicate detection.
+ * Files sharing the same blob SHA are exact duplicates in git.
+ */
+function buildDuplicateMap(files: GitFileEntry[], pathPrefix: string): Map<string, string> {
+  // Group paths by SHA
+  const shaToEntries = new Map<string, string[]>()
+  for (const { path, sha } of files) {
+    const prefixed = joinRelativePath(pathPrefix, path)
+    const existing = shaToEntries.get(sha)
+    if (existing) {
+      existing.push(prefixed)
+    } else {
+      shaToEntries.set(sha, [prefixed])
+    }
+  }
+
+  // For each group with >1 file, shortest path is the original
+  const duplicateOf = new Map<string, string>()
+  for (const paths of shaToEntries.values()) {
+    if (paths.length < 2) continue
+    paths.sort((a, b) => a.length - b.length || a.localeCompare(b))
+    const original = paths[0]
+    for (let i = 1; i < paths.length; i++) {
+      duplicateOf.set(paths[i], original)
+    }
+  }
+
+  return duplicateOf
+}
+
+
+
+/**
+ * Result of scanning a directory, including both file results and submodule info
+ */
+export interface ScanResult {
+  files: FileResult[]
+  submodules: SubmoduleInfo[]
+}
+
+interface ScanRepoOptions {
+  repoDir: string
+  pathPrefix: string
+  includeDiff: boolean
+  includeSubmodules: boolean
+  logger: Logger
+  isIncluded?: (path: string) => boolean
+  isIgnored?: (path: string) => boolean
+  visitedRepoDirs: Set<string>
+}
+
+function normalizeRelativePath(path: string): string {
+  return path.replace(/\\/g, '/')
+}
+
+function joinRelativePath(prefix: string, path: string): string {
+  const normalizedPath = normalizeRelativePath(path)
+  return prefix ? `${prefix}/${normalizedPath}` : normalizedPath
+}
+
+function getCanonicalRepoDir(dir: string): string {
+  try {
+    return realpathSync(dir)
+  } catch {
+    return dir
+  }
+}
+
+async function scanRepo(options: ScanRepoOptions): Promise<ScanResult> {
+  const canonicalRepoDir = getCanonicalRepoDir(options.repoDir)
+  if (options.visitedRepoDirs.has(canonicalRepoDir)) {
+    return { files: [], submodules: [] }
+  }
+  options.visitedRepoDirs.add(canonicalRepoDir)
+
+  let submodules: SubmoduleInfo[] = []
+  const directSubmodules = options.includeSubmodules ? getSubmodules(options.repoDir) : []
+  const directSubmodulePathSet = options.includeSubmodules
+    ? new Set(directSubmodules.map(submodule => submodule.path))
+    : getSubmodulePaths(options.repoDir)
+
+  if (options.includeSubmodules) {
+    submodules = directSubmodules.map(submodule => ({
+      ...submodule,
+      path: joinRelativePath(options.pathPrefix, submodule.path),
+    }))
+  }
+
+  const normalizedSubmodulePaths = new Set<string>()
+  for (const path of directSubmodulePathSet) {
+    normalizedSubmodulePaths.add(normalize(path))
+  }
+
+  const allGitFiles = getGitFiles(options.repoDir)
+
+  // Build duplicate map before filtering (needs all files to detect dupes)
+  const duplicateOf = buildDuplicateMap(allGitFiles, options.pathPrefix)
+
+  let gitFiles = allGitFiles.filter(f => !normalizedSubmodulePaths.has(f.path))
+  gitFiles = gitFiles.filter(f => isSupportedFile(f.path) || isReadmeFile(f.path))
+
+  gitFiles = gitFiles.filter(f => {
+    const relativePath = joinRelativePath(options.pathPrefix, f.path)
+
+    if (options.isIncluded && !options.isIncluded(relativePath)) {
+      return false
+    }
+
+    if (options.isIgnored && options.isIgnored(relativePath)) {
+      return false
+    }
+
+    return true
+  })
+
+  if (gitFiles.length > MAX_FILES) {
+    options.logger?.warn(`Warning: Too many files (${gitFiles.length} > ${MAX_FILES}), skipping scan`)
+    return { files: [], submodules }
+  }
+
+  let fileStats: Map<string, FileDiffStats> | null = null
+  let fileDiffs: Map<string, FileDiff> | null = null
+
+  if (options.includeDiff) {
+    try {
+      const diffData = getAllDiffData(options.repoDir, directSubmodulePathSet, options.logger)
+      fileStats = diffData.fileStats
+      fileDiffs = diffData.fileDiffs
+    } catch {
+      fileStats = null
+      fileDiffs = null
+    }
+  }
+
+  const limit = pLimit(20)
+  const resultPromises = gitFiles.map(({ path: relativePath }) => {
+    const fullPath = join(options.repoDir, relativePath)
+    const normalizedPath = normalizeRelativePath(relativePath)
+    const prefixedRelativePath = joinRelativePath(options.pathPrefix, relativePath)
+    const fileDiff = fileDiffs?.get(normalizedPath)
+    const stats = fileStats?.get(normalizedPath)
+    const dupOriginal = duplicateOf.get(prefixedRelativePath)
+
+    return limit(async () => {
+      try {
+        // If this file is a duplicate, return a stub pointing to the original
+        if (dupOriginal) {
+          return {
+            relativePath: prefixedRelativePath,
+            duplicateOf: dupOriginal,
+            definitions: [],
+          } satisfies FileResult
+        }
+        return await processFile(fullPath, prefixedRelativePath, fileDiff, stats)
+      } catch {
+        return null
+      }
+    })
+  })
+
+  const nestedResults = options.includeSubmodules
+    ? await Promise.all(directSubmodules.map(async submodule => {
+      if (!submodule.initialized) {
+        return { files: [], submodules: [] }
+      }
+
+      return scanRepo({
+        ...options,
+        repoDir: join(options.repoDir, submodule.path),
+        pathPrefix: joinRelativePath(options.pathPrefix, submodule.path),
+      })
+    }))
+    : []
+
+  const results = await Promise.all(resultPromises)
+
+  for (const nested of nestedResults) {
+    submodules.push(...nested.submodules)
+  }
+
+  return {
+    files: [
+      ...results.filter((result): result is FileResult => result !== null),
+      ...nestedResults.flatMap(result => result.files),
+    ],
+    submodules,
+  }
+}
+
+/**
+ * Scan directory and process files with header comments
+ */
+export async function scanDirectory(options: GenerateOptions = {}): Promise<ScanResult> {
+  const dir = options.dir ?? process.cwd()
+  const logger = options.logger ?? createConsoleLogger()
+  const ignorePatterns = (options.ignore ?? []).filter((p): p is string => !!p)
+  const filterPatterns = (options.filter ?? []).filter((p): p is string => !!p)
+  return scanRepo({
+    repoDir: dir,
+    pathPrefix: '',
+    includeDiff: options.diff ?? false,
+    includeSubmodules: options.submodules !== false,
+    logger,
+    isIncluded: filterPatterns.length > 0 ? picomatch(filterPatterns) : undefined,
+    isIgnored: ignorePatterns.length > 0 ? picomatch(ignorePatterns) : undefined,
+    visitedRepoDirs: new Set(),
+  })
+}
+
+/**
+ * Process a single file - check for marker and extract definitions
+ */
+async function processFile(
+  fullPath: string,
+  relativePath: string,
+  fileDiff?: FileDiff,
+  fileStats?: FileDiffStats
+): Promise<FileResult | null> {
+  // Handle README.md files specially
+  if (isReadmeFile(relativePath)) {
+    const description = await extractMarkdownDescription(fullPath)
+    if (!description) {
+      return null
+    }
+    return {
+      relativePath,
+      description,
+      definitions: [],
+      diff: fileStats,
+    }
+  }
+
+  // Detect language first
+  const language = detectLanguage(relativePath)
+  if (!language) {
+    return null
+  }
+
+  // Read file once for both marker extraction and definition parsing
+  const code = await readFile(fullPath, 'utf8')
+
+  // Check for marker using the code we already read
+  const marker = await extractMarkerFromCode(code, language)
+  if (!marker.found) {
+    return null
+  }
+
+  // Parse and extract definitions using the same code
+  const tree = await parseCode(code, language)
+  let definitions = extractDefinitions(tree.rootNode, language)
+
+  // Apply diff info if available (for definition-level stats)
+  if (fileDiff) {
+    definitions = applyDiffToDefinitions(definitions, fileDiff)
+  }
+
+  return {
+    relativePath,
+    description: marker.description,
+    definitions,
+    // Use pre-calculated file stats from --numstat (more reliable)
+    diff: fileStats,
+  }
+}

package/src/test-helpers/git-test-helpers.ts

@@ -0,0 +1,62 @@
+// Helpers for creating temporary git repositories in tests.
+
+import { execSync } from 'child_process'
+import { mkdtempSync, mkdirSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+
+export interface TempRepo {
+  dir: string
+}
+
+function run(command: string, cwd: string): string {
+  return execSync(command, {
+    cwd,
+    encoding: 'utf8',
+    stdio: ['pipe', 'pipe', 'pipe'],
+  })
+}
+
+export function createTempDir(prefix: string): string {
+  return mkdtempSync(join(tmpdir(), prefix))
+}
+
+export function initRepo(dir: string): TempRepo {
+  run('git init', dir)
+  run('git config user.name "agentmap tests"', dir)
+  run('git config user.email "agentmap@example.com"', dir)
+  return { dir }
+}
+
+export function createRepo(prefix: string): TempRepo {
+  const dir = createTempDir(prefix)
+  mkdirSync(dir, { recursive: true })
+  return initRepo(dir)
+}
+
+export function writeTrackedFile(repo: TempRepo, relativePath: string, content: string): void {
+  const filePath = join(repo.dir, relativePath)
+  mkdirSync(join(filePath, '..'), { recursive: true })
+  writeFileSync(filePath, content, 'utf8')
+}
+
+export function commitAll(repo: TempRepo, message: string): void {
+  run('git add .', repo.dir)
+  run(`git commit -m ${JSON.stringify(message)}`, repo.dir)
+}
+
+export function shortHead(repo: TempRepo): string {
+  return run('git rev-parse --short HEAD', repo.dir).trim()
+}
+
+export function addSubmodule(repo: TempRepo, sourceDir: string, targetPath: string): void {
+  run(`git -c protocol.file.allow=always submodule add ${JSON.stringify(sourceDir)} ${JSON.stringify(targetPath)}`, repo.dir)
+}
+
+export function updateSubmodulesRecursive(repo: TempRepo): void {
+  run('git -c protocol.file.allow=always submodule update --init --recursive', repo.dir)
+}
+
+export function deinitSubmodule(repo: TempRepo, targetPath: string): void {
+  run(`git submodule deinit -f ${JSON.stringify(targetPath)}`, repo.dir)
+}

package/src/types.ts

@@ -0,0 +1,191 @@
+// Core type definitions for the codebase map.
+
+import type Parser from 'web-tree-sitter'
+import type { Logger } from './logger.js'
+
+/**
+ * Supported programming languages
+ */
+export type Language =
+  | 'typescript'
+  | 'javascript'
+  | 'python'
+  | 'rust'
+  | 'go'
+  | 'zig'
+  | 'cpp'
+
+/**
+ * Symbol definitions mapping: name -> description string
+ */
+export interface DefEntry {
+  [symbolName: string]: string
+}
+
+/**
+ * Git diff stats for a file (total lines added/deleted)
+ */
+export interface FileDiffStats {
+  added: number
+  deleted: number
+}
+
+/**
+ * A file entry in the map
+ */
+export interface FileEntry {
+  description?: string
+  diff?: string  // formatted as "+N-M" or "+N" or "-M"
+  defs?: DefEntry
+  exports?: DefEntry  // used instead of defs when truncating files with exported symbols
+}
+
+/**
+ * A submodule entry in the map
+ */
+export interface SubmoduleEntry {
+  submodule: string   // "branch @ sha" or "detached @ sha"
+  dirty?: string      // "modified" if submodule has uncommitted changes
+}
+
+/**
+ * A submodule node in the map.
+ * Carries submodule metadata and can also contain nested files/directories.
+ */
+export interface SubmoduleNode extends SubmoduleEntry {
+  [name: string]: MapNode | FileEntry | SubmoduleNode | string | undefined
+}
+
+/**
+ * Info about a git submodule discovered in the repo
+ */
+export interface SubmoduleInfo {
+  /** Relative path of the submodule in the parent repo */
+  path: string
+  /** Current HEAD commit SHA (short) */
+  commit: string
+  /** Checked-out branch name, or undefined if detached HEAD */
+  branch?: string
+  /** Remote URL from .gitmodules */
+  url?: string
+  /** Whether the submodule has uncommitted changes */
+  dirty?: boolean
+  /** Whether the submodule is initialized */
+  initialized: boolean
+}
+
+/**
+ * Recursive map node - either a directory (with children), a file entry, or a submodule entry
+ */
+export interface MapNode {
+  [name: string]: MapNode | FileEntry | SubmoduleNode
+}
+
+/**
+ * Result of extracting marker and description from a file
+ */
+export interface MarkerResult {
+  found: boolean
+  description?: string
+}
+
+/**
+ * Types of definitions we extract
+ */
+export type DefinitionType = 
+  | 'function' 
+  | 'class' 
+  | 'struct'
+  | 'union'
+  | 'trait'
+  | 'type' 
+  | 'interface' 
+  | 'const' 
+  | 'enum'
+
+/**
+ * Git status for a definition
+ */
+export type DefinitionStatus = 'added' | 'updated'
+
+/**
+ * Git diff stats for a definition
+ */
+export interface DefinitionDiff {
+  status: DefinitionStatus
+  added: number    // lines added
+  deleted: number  // lines deleted
+}
+
+/**
+ * A definition extracted from source code
+ */
+export interface Definition {
+  name: string
+  line: number     // 1-based start line
+  endLine: number  // 1-based end line
+  type: DefinitionType
+  exported: boolean
+  extern?: boolean  // true for extern declarations (C/C++/Zig)
+  diff?: DefinitionDiff  // only present when --diff flag used
+}
+
+/**
+ * Result of processing a single file
+ */
+export interface FileResult {
+  relativePath: string
+  description?: string
+  definitions: Definition[]
+  diff?: FileDiffStats  // only present when --diff flag used
+  /** If set, this file is an exact duplicate of another file (by git blob SHA) */
+  duplicateOf?: string
+}
+
+/**
+ * Options for generating the map
+ */
+export interface GenerateOptions {
+  /** Directory to scan (default: cwd) */
+  dir?: string
+  /** Glob patterns to ignore */
+  ignore?: string[]
+  /** Glob patterns to filter - only include matching files */
+  filter?: string[]
+  /** Include git diff status for definitions */
+  diff?: boolean
+  /** Git ref to diff against (default: HEAD for unstaged, --cached for staged) */
+  diffBase?: string
+  /** Max definitions per file before truncation (default: 25) */
+  maxDefs?: number
+  /** Max characters for file descriptions before truncation (default: 300). Rounds up to full line. */
+  maxDescChars?: number
+  /** Include submodule info in the map (default: true) */
+  submodules?: boolean
+  /** Logger implementation (default: console logger) */
+  logger?: Logger
+}
+
+/**
+ * A hunk from git diff output
+ */
+export interface DiffHunk {
+  oldStart: number
+  oldCount: number
+  newStart: number
+  newCount: number
+}
+
+/**
+ * Parsed diff for a single file
+ */
+export interface FileDiff {
+  path: string
+  hunks: DiffHunk[]
+}
+
+/**
+ * Re-export parser types
+ */
+export type SyntaxNode = Parser.SyntaxNode
+export type SyntaxTree = Parser.Tree