agentmap 0.8.0 → 0.9.0

package/src/map/builder.ts

@@ -0,0 +1,175 @@
+// Build the nested map object from file results.
+
+import { basename } from 'path'
+import type { Definition, FileEntry, FileResult, FileDiffStats, MapNode, SubmoduleInfo, SubmoduleNode } from '../types.js'
+
+/**
+ * Build a nested map object from file results and submodule info
+ */
+export function buildMap(results: FileResult[], rootName: string, submodules?: SubmoduleInfo[]): MapNode {
+  const root: MapNode = {}
+
+  // Insert submodule entries
+  if (submodules) {
+    for (const sub of submodules) {
+      insertSubmodule(root, sub)
+    }
+  }
+
+  for (const result of results) {
+    insertFile(root, result)
+  }
+
+  // Wrap in root name
+  return { [rootName]: root }
+}
+
+/**
+ * Format file diff stats as a string like "+15-3" or "+15" or "-3"
+ */
+function formatFileDiff(diff: FileDiffStats): string {
+  const parts: string[] = []
+  if (diff.added > 0) {
+    parts.push(`+${diff.added}`)
+  }
+  if (diff.deleted > 0) {
+    parts.push(`-${diff.deleted}`)
+  }
+  return parts.join('')
+}
+
+/**
+ * Format a definition as a string like "exported fn updated (+5-2)"
+ * No commas, exported/extern before the type, "fn" instead of "function".
+ */
+function formatDefinition(def: Definition): string {
+  const parts: string[] = []
+
+  if (def.exported) {
+    parts.push('exported')
+  }
+
+  if (def.extern) {
+    parts.push('extern')
+  }
+
+  parts.push(def.type === 'function' ? 'fn' : def.type)
+
+  // Add diff info if present
+  if (def.diff) {
+    const diffParts: string[] = []
+    if (def.diff.added > 0) {
+      diffParts.push(`+${def.diff.added}`)
+    }
+    if (def.diff.deleted > 0) {
+      diffParts.push(`-${def.diff.deleted}`)
+    }
+
+    if (diffParts.length > 0) {
+      parts.push(`${def.diff.status} (${diffParts.join('')})`)
+    } else {
+      parts.push(def.diff.status)
+    }
+  }
+
+  return parts.join(' ')
+}
+
+/**
+ * Insert a file result into the map at its path location
+ */
+function insertFile(root: MapNode, result: FileResult): void {
+  const parts = result.relativePath.split('/')
+  let current = root
+
+  // Navigate/create directory structure
+  for (let i = 0; i < parts.length - 1; i++) {
+    const dir = parts[i]
+    if (!current[dir]) {
+      current[dir] = {}
+    }
+    current = current[dir] as MapNode
+  }
+
+  // Create file entry
+  const filename = parts[parts.length - 1]
+
+  // Duplicate files: just point to the original, no defs
+  if (result.duplicateOf) {
+    current[filename] = { description: `duplicate of ${result.duplicateOf}` }
+    return
+  }
+
+  const entry: FileEntry = {}
+
+  if (result.description) {
+    entry.description = result.description
+  }
+
+  if (result.diff) {
+    entry.diff = formatFileDiff(result.diff)
+  }
+
+  if (result.definitions.length > 0) {
+    entry.defs = {}
+    for (const def of result.definitions) {
+      entry.defs[def.name] = formatDefinition(def)
+    }
+  }
+
+  current[filename] = entry
+}
+
+/**
+ * Insert a submodule entry into the map at its path location.
+ * Format: "branch @ sha" or "detached @ sha" or "uninitialized @ sha"
+ */
+function insertSubmodule(root: MapNode, sub: SubmoduleInfo): void {
+  const parts = sub.path.split('/')
+  let current = root
+
+  // Navigate/create directory structure for nested submodule paths
+  for (let i = 0; i < parts.length - 1; i++) {
+    const dir = parts[i]
+    if (!current[dir]) {
+      current[dir] = {}
+    }
+    current = current[dir] as MapNode
+  }
+
+  // Build the submodule label
+  let label: string
+  if (!sub.initialized) {
+    label = `uninitialized @ ${sub.commit}`
+  } else if (sub.branch) {
+    label = `${sub.branch} @ ${sub.commit}`
+  } else {
+    label = `detached @ ${sub.commit}`
+  }
+
+  const name = parts[parts.length - 1]
+  const existing = current[name]
+  const entry: SubmoduleNode = existing && typeof existing === 'object'
+    ? existing as SubmoduleNode
+    : { submodule: label }
+
+  entry.submodule = label
+  if (sub.dirty) {
+    entry.dirty = 'modified'
+  } else {
+    delete entry.dirty
+  }
+
+  current[name] = entry
+}
+
+/**
+ * Get the root name from a directory path
+ */
+export function getRootName(dir: string): string {
+  // Handle trailing slashes
+  const cleaned = dir.replace(/\/+$/, '')
+  // Get basename, or use 'root' for current directory
+  const name = basename(cleaned)
+  return name === '.' || name === '' ? 'root' : name
+}

package/src/map/truncate.ts

@@ -0,0 +1,188 @@
+// Truncate definitions and descriptions in map to limit context size.
+
+import type { DefEntry, FileEntry, MapNode, SubmoduleEntry, SubmoduleNode } from '../types.js'
+
+const DEFAULT_MAX_DEFS = 25
+const DEFAULT_MAX_DESC_CHARS = 300
+
+export interface TruncateOptions {
+  maxDefs?: number
+  maxDescChars?: number
+}
+
+/**
+ * Check if a def value indicates exported or extern
+ */
+function isExportedDef(value: string): boolean {
+  return value.includes('exported') || value.includes('extern')
+}
+
+/**
+ * Truncate description by character count, rounding up to include the full line
+ * that crosses the limit (in excess).
+ */
+function truncateDescriptionByChars(description: string, maxChars: number): string {
+  if (description.length <= maxChars) return description
+
+  const lines = description.split('\n')
+
+  // Single-line fallback: hard-truncate at maxChars
+  if (lines.length === 1) {
+    return description.slice(0, maxChars) + '...'
+  }
+
+  let charCount = 0
+
+  for (let i = 0; i < lines.length; i++) {
+    // +1 for the newline separator (except first line)
+    charCount += lines[i].length + (i > 0 ? 1 : 0)
+    if (charCount >= maxChars) {
+      // Include this line (in excess), then stop
+      const kept = lines.slice(0, i + 1)
+      const remaining = lines.length - kept.length
+      if (remaining > 0) {
+        kept.push(`... and ${remaining} more lines`)
+      }
+      return kept.join('\n')
+    }
+  }
+
+  return description
+}
+
+/**
+ * Check if a value is a FileEntry (has description or defs)
+ */
+function isFileEntry(value: unknown): value is FileEntry {
+  if (!value || typeof value !== 'object') return false
+  const obj = value as Record<string, unknown>
+  return 'description' in obj || 'defs' in obj
+}
+
+/**
+ * Check if a value is a SubmoduleEntry (has submodule key)
+ */
+function isSubmoduleEntry(value: unknown): value is SubmoduleEntry {
+  if (!value || typeof value !== 'object') return false
+  return 'submodule' in (value as Record<string, unknown>)
+}
+
+/**
+ * Recursively truncate a submodule node while preserving its metadata keys.
+ */
+function truncateSubmoduleNode(entry: SubmoduleNode, options: TruncateOptions): SubmoduleNode {
+  const result: SubmoduleNode = { submodule: entry.submodule }
+
+  if (entry.dirty) {
+    result.dirty = entry.dirty
+  }
+
+  for (const [key, value] of Object.entries(entry)) {
+    if (key === 'submodule' || key === 'dirty') {
+      continue
+    }
+
+    if (isFileEntry(value)) {
+      result[key] = truncateFileEntry(value, options)
+    } else if (isSubmoduleEntry(value)) {
+      result[key] = truncateSubmoduleNode(value as SubmoduleNode, options)
+    } else if (value && typeof value === 'object') {
+      result[key] = truncateMap(value as MapNode, options)
+    }
+  }
+
+  return result
+}
+
+/**
+ * Truncate a file entry: cap description by chars and defs by count.
+ */
+function truncateFileEntry(entry: FileEntry, options: TruncateOptions): FileEntry {
+  let result = entry
+
+  // Truncate description by character count
+  const maxDescChars = options.maxDescChars ?? DEFAULT_MAX_DESC_CHARS
+  if (result.description && result.description.length > maxDescChars) {
+    result = { ...result, description: truncateDescriptionByChars(result.description, maxDescChars) }
+  }
+
+  // Truncate defs
+  result = truncateDefs(result, options.maxDefs)
+  return result
+}
+
+/**
+ * Truncate definitions in a file entry to maxDefs
+ * If file has exported symbols, shows only exports field instead
+ * Otherwise uses current truncation behavior
+ */
+export function truncateDefs(entry: FileEntry, maxDefsOrOptions?: number | TruncateOptions): FileEntry {
+  const maxDefs = typeof maxDefsOrOptions === 'number'
+    ? maxDefsOrOptions
+    : (maxDefsOrOptions?.maxDefs ?? DEFAULT_MAX_DEFS)
+
+  if (!entry.defs) return entry
+
+  const defNames = Object.keys(entry.defs)
+  if (defNames.length <= maxDefs) return entry
+
+  // Filter to only exported/extern definitions
+  const exportedNames = defNames.filter(name => isExportedDef(entry.defs![name]))
+
+  // If we have exports, use exports field instead of defs
+  if (exportedNames.length > 0) {
+    const exports: DefEntry = {}
+    const maxExports = Math.min(exportedNames.length, maxDefs)
+
+    for (let i = 0; i < maxExports; i++) {
+      const name = exportedNames[i]
+      exports[name] = entry.defs[name]
+    }
+
+    // Add marker if exports were also truncated
+    if (exportedNames.length > maxDefs) {
+      const remaining = exportedNames.length - maxDefs
+      exports[`__more_${remaining}__`] = `${remaining} more exports`
+    }
+
+    // Return with exports instead of defs
+    const { defs, ...rest } = entry
+    return { ...rest, exports }
+  }
+
+  // No exports found - use current truncation behavior
+  const truncated: DefEntry = {}
+  for (let i = 0; i < maxDefs; i++) {
+    const name = defNames[i]
+    truncated[name] = entry.defs[name]
+  }
+
+  const remaining = defNames.length - maxDefs
+  // Add marker that will be converted to comment
+  truncated[`__more_${remaining}__`] = `${remaining} more definitions`
+
+  return { ...entry, defs: truncated }
+}
+
+/**
+ * Recursively truncate defs and descriptions in all files in the map
+ */
+export function truncateMap(node: MapNode, maxDefsOrOptions?: number | TruncateOptions): MapNode {
+  const options: TruncateOptions = typeof maxDefsOrOptions === 'number'
+    ? { maxDefs: maxDefsOrOptions }
+    : (maxDefsOrOptions ?? {})
+
+  const result: MapNode = {}
+
+  for (const [key, value] of Object.entries(node)) {
+    if (isFileEntry(value)) {
+      result[key] = truncateFileEntry(value, options)
+    } else if (isSubmoduleEntry(value)) {
+      result[key] = truncateSubmoduleNode(value as SubmoduleNode, options)
+    } else if (value && typeof value === 'object') {
+      result[key] = truncateMap(value as MapNode, options)
+    }
+  }
+
+  return result
+}

package/src/map/yaml.ts

@@ -0,0 +1,66 @@
+// Format map object to YAML string.
+
+import yaml from 'js-yaml'
+import type { MapNode } from '../types.js'
+
+/**
+ * Check if a key is a README file (case-insensitive)
+ */
+function isReadme(key: string): boolean {
+  const lower = key.toLowerCase()
+  return lower === 'readme.md' || lower === 'readme'
+}
+
+/**
+ * Custom key sorter: description first, then submodule/dirty, then diff, then defs/exports, then README files, then alphabetical
+ */
+function sortKeys(a: string, b: string): number {
+  // description always first
+  if (a === 'description') return -1
+  if (b === 'description') return 1
+  // submodule second (for submodule entries)
+  if (a === 'submodule') return -1
+  if (b === 'submodule') return 1
+  // dirty third (for submodule entries)
+  if (a === 'dirty') return -1
+  if (b === 'dirty') return 1
+  // diff fourth
+  if (a === 'diff') return -1
+  if (b === 'diff') return 1
+  // defs/exports fifth
+  if (a === 'defs' || a === 'exports') return -1
+  if (b === 'defs' || b === 'exports') return 1
+  // README files come before other files
+  const aIsReadme = isReadme(a)
+  const bIsReadme = isReadme(b)
+  if (aIsReadme && !bIsReadme) return -1
+  if (bIsReadme && !aIsReadme) return 1
+  // alphabetical for everything else
+  return a.localeCompare(b)
+}
+
+/**
+ * Convert __more_N__ markers to YAML comments
+ */
+function markersToComments(yamlStr: string): string {
+  return yamlStr.replace(
+    /^(\s*)__more_(\d+)__: (\d+ more (?:definitions|exports))$/gm,
+    '$1# ... $3'
+  )
+}
+
+/**
+ * Convert map object to YAML string
+ * Automatically converts truncation markers to comments
+ */
+export function toYaml(map: MapNode): string {
+  const yamlStr = yaml.dump(map, {
+    indent: 2,
+    lineWidth: -1,  // Don't wrap lines
+    noRefs: true,   // Don't use YAML references
+    sortKeys,       // Custom ordering: description first
+    quotingType: '"',
+    forceQuotes: false,
+  })
+  return markersToComments(yamlStr)
+}

package/src/parser/index.ts

@@ -0,0 +1,53 @@
+// Tree-sitter parser initialization and code parsing.
+
+import Parser from 'web-tree-sitter'
+import type { Language, SyntaxTree } from '../types.js'
+import { loadGrammar } from './languages.js'
+
+let initialized = false
+let sharedParser: Parser | null = null
+
+/**
+ * Initialize the tree-sitter parser
+ */
+export async function initParser(): Promise<void> {
+  if (initialized) return
+  await Parser.init()
+  initialized = true
+}
+
+/**
+ * Get the shared parser instance
+ */
+async function getParser(): Promise<Parser> {
+  if (sharedParser) return sharedParser
+  await initParser()
+  sharedParser = new Parser()
+  return sharedParser
+}
+
+/**
+ * Parse source code and return the syntax tree
+ */
+export async function parseCode(
+  code: string,
+  language: Language
+): Promise<SyntaxTree> {
+  const parser = await getParser()
+  const grammar = await loadGrammar(language)
+  parser.setLanguage(grammar)
+  return parser.parse(code)
+}
+
+/**
+ * Reset the parser (for testing)
+ */
+export function resetParser(): void {
+  if (sharedParser) {
+    sharedParser.delete()
+    sharedParser = null
+  }
+  initialized = false
+}
+
+export { detectLanguage, loadGrammar, LANGUAGE_EXTENSIONS } from './languages.js'

package/src/parser/languages.ts

@@ -0,0 +1,64 @@
+// Language detection and grammar loading for tree-sitter.
+
+import Parser from 'web-tree-sitter'
+import type { Language } from '../types.js'
+import { createRequire } from 'module'
+import { LANGUAGE_EXTENSIONS, GRAMMAR_PATHS } from '../languages/index.js'
+
+const require = createRequire(import.meta.url)
+
+// Re-export for backwards compatibility
+export { LANGUAGE_EXTENSIONS }
+
+/**
+ * Detect language from file path extension
+ */
+export function detectLanguage(filepath: string): Language | null {
+  const ext = filepath.slice(filepath.lastIndexOf('.'))
+  return LANGUAGE_EXTENSIONS[ext] ?? null
+}
+
+/**
+ * Get the WASM grammar path for a language
+ */
+function getGrammarPath(language: Language): string {
+  return require.resolve(GRAMMAR_PATHS[language])
+}
+
+/**
+ * Cache for loaded grammars
+ */
+const grammarCache = new Map<Language, Parser.Language>()
+
+/**
+ * Ensure Parser is initialized before loading grammars
+ */
+let parserInitialized = false
+async function ensureParserInit(): Promise<void> {
+  if (!parserInitialized) {
+    await Parser.init()
+    parserInitialized = true
+  }
+}
+
+/**
+ * Load a tree-sitter grammar for the given language
+ */
+export async function loadGrammar(language: Language): Promise<Parser.Language> {
+  await ensureParserInit()
+  
+  const cached = grammarCache.get(language)
+  if (cached) return cached
+
+  const path = getGrammarPath(language)
+  const grammar = await Parser.Language.load(path)
+  grammarCache.set(language, grammar)
+  return grammar
+}
+
+/**
+ * Clear grammar cache (for testing)
+ */
+export function clearGrammarCache(): void {
+  grammarCache.clear()
+}

package/src/scanner.test.ts

@@ -0,0 +1,95 @@
+// Tests for recursive scanning across initialized git submodules.
+
+import { describe, expect, test } from 'bun:test'
+import { scanDirectory } from './scanner.js'
+import { addSubmodule, commitAll, createRepo, deinitSubmodule, updateSubmodulesRecursive, writeTrackedFile } from './test-helpers/git-test-helpers.js'
+
+describe('scanDirectory submodule recursion', () => {
+  test('includes files from initialized nested submodules under prefixed paths', async () => {
+    const nestedRepo = createRepo('agentmap-submodule-nested-')
+    writeTrackedFile(nestedRepo, 'README.md', '# Nested repo\n\nNested submodule README.')
+    commitAll(nestedRepo, 'Add nested repo README')
+
+    const childRepo = createRepo('agentmap-submodule-child-')
+    writeTrackedFile(childRepo, 'README.md', '# Child repo\n\nChild submodule README.')
+    commitAll(childRepo, 'Add child repo README')
+    addSubmodule(childRepo, nestedRepo.dir, 'deps/nested-lib')
+    commitAll(childRepo, 'Add nested submodule')
+
+    const rootRepo = createRepo('agentmap-submodule-root-')
+    writeTrackedFile(rootRepo, 'README.md', '# Root repo\n\nRoot repo README.')
+    commitAll(rootRepo, 'Add root README')
+    addSubmodule(rootRepo, childRepo.dir, 'vendor/child-lib')
+    commitAll(rootRepo, 'Add child submodule')
+    updateSubmodulesRecursive(rootRepo)
+
+    const result = await scanDirectory({ dir: rootRepo.dir })
+
+    expect(result.files.map(file => file.relativePath).sort()).toMatchInlineSnapshot(`
+[
+  "README.md",
+  "vendor/child-lib/README.md",
+  "vendor/child-lib/deps/nested-lib/README.md",
+]
+`)
+
+    expect(result.submodules.map(submodule => ({
+      path: submodule.path,
+      initialized: submodule.initialized,
+      dirty: submodule.dirty,
+      commitLength: submodule.commit.length,
+    }))).toMatchInlineSnapshot(`
+[
+  {
+    "commitLength": 7,
+    "dirty": false,
+    "initialized": true,
+    "path": "vendor/child-lib",
+  },
+  {
+    "commitLength": 7,
+    "dirty": false,
+    "initialized": true,
+    "path": "vendor/child-lib/deps/nested-lib",
+  },
+]
+`)
+  })
+
+  test('keeps uninitialized submodules as metadata-only nodes', async () => {
+    const childRepo = createRepo('agentmap-submodule-uninit-child-')
+    writeTrackedFile(childRepo, 'README.md', '# Child repo\n\nChild submodule README.')
+    commitAll(childRepo, 'Add child repo README')
+
+    const rootRepo = createRepo('agentmap-submodule-uninit-root-')
+    writeTrackedFile(rootRepo, 'README.md', '# Root repo\n\nRoot repo README.')
+    commitAll(rootRepo, 'Add root README')
+    addSubmodule(rootRepo, childRepo.dir, 'vendor/child-lib')
+    commitAll(rootRepo, 'Add child submodule')
+    deinitSubmodule(rootRepo, 'vendor/child-lib')
+
+    const result = await scanDirectory({ dir: rootRepo.dir })
+
+    expect(result.files.map(file => file.relativePath).sort()).toMatchInlineSnapshot(`
+[
+  "README.md",
+]
+`)
+
+    expect(result.submodules.map(submodule => ({
+      path: submodule.path,
+      initialized: submodule.initialized,
+      dirty: submodule.dirty,
+      commitLength: submodule.commit.length,
+    }))).toMatchInlineSnapshot(`
+[
+  {
+    "commitLength": 7,
+    "dirty": false,
+    "initialized": false,
+    "path": "vendor/child-lib",
+  },
+]
+`)
+  })
+})