@rahmantech/knl-view 0.1.0 → 0.3.0

package/server/graph/modules_builder.js

@@ -0,0 +1,290 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
+import { GraphNode } from './model.js'
+import { SYNTAX_LANGUAGES, extractCode } from './syntax.js'
+import { desiredSourceEdgeKeys } from './sources.js'
+import {
+  resolveModuleLocation,
+  formatExternalPackageNodeId,
+  formatExternalSymbolNodeId,
+  defaultEcosystemModel,
+  updateModulesCatalog
+} from './modules_catalog.js'
+
+const MODULE_DEFAULT_IGNORES = [
+  'node_modules',
+  '.git',
+  '.github',
+  'test',
+  'tests',
+  '__tests__',
+  'docs',
+  'doc',
+  'example',
+  'examples',
+  'fixtures',
+  'coverage'
+]
+
+/**
+ * Scan source files of an external module directory.
+ */
+export async function scanModuleFiles(moduleDir, ecosystem) {
+  const files = {}
+  const walk = async (dirAbs, rel) => {
+    let entries
+    try {
+      entries = await fs.readdir(dirAbs, { withFileTypes: true })
+    } catch {
+      return
+    }
+    for (const entry of entries) {
+      const childRel = rel ? `${rel}/${entry.name}` : entry.name
+      if (entry.isDirectory()) {
+        if (MODULE_DEFAULT_IGNORES.includes(entry.name.toLowerCase())) continue
+        await walk(path.join(dirAbs, entry.name), childRel)
+        continue
+      }
+      if (!entry.isFile()) continue
+      const ext = path.extname(entry.name).toLowerCase()
+      if (!SYNTAX_LANGUAGES[ext]) continue
+      if (ecosystem === 'go' && entry.name.endsWith('_test.go')) continue
+      if (ecosystem === 'pypi' && entry.name.startsWith('test_')) continue
+      files[childRel] = path.join(dirAbs, entry.name)
+    }
+  }
+  await walk(moduleDir, '')
+  return files
+}
+
+/**
+ * Parse an eco::name string into components: { ecosystem, moduleName }
+ */
+export function parseModuleSpecifier(target) {
+  if (!target || typeof target !== 'string') return null
+  const idx = target.indexOf('::')
+  if (idx <= 0) return null
+  const ecosystem = target.slice(0, idx).toLowerCase()
+  const moduleName = target.slice(idx + 2)
+  if (!moduleName) return null
+  return { ecosystem, moduleName }
+}
+
+/**
+ * Build graph for an external module from disk using Tree-Sitter.
+ */
+export async function buildModuleGraph(ecosystem, moduleName, options = {}) {
+  const resolved = options.location
+    ? { location: options.location, version: options.version || '0.0.0', name: moduleName, ecosystem }
+    : await resolveModuleLocation(ecosystem, moduleName, options)
+
+  if (!resolved || !resolved.location) {
+    throw new Error(`Could not resolve module directory for ${ecosystem}::${moduleName} on disk. Ensure it is installed.`)
+  }
+
+  const { location, version } = resolved
+  const model = defaultEcosystemModel(ecosystem)
+  const moduleNodeId = formatExternalPackageNodeId(ecosystem, moduleName, version)
+
+  const files = await scanModuleFiles(location, ecosystem)
+  const symbolNodes = []
+  const symbolNamesSet = new Set()
+  const edges = []
+  const symbolIndex = new Map() // name -> [id, ...]
+  const fileSymbolsMap = new Map()
+
+  for (const [rel, abs] of Object.entries(files)) {
+    const ext = path.extname(rel).toLowerCase()
+    let content
+    try {
+      content = await fs.readFile(abs, 'utf8')
+    } catch {
+      continue
+    }
+
+    let extracted
+    try {
+      extracted = await extractCode(ext, content)
+    } catch {
+      continue
+    }
+
+    const nameCounts = new Map()
+    for (const sym of extracted.symbols || []) {
+      nameCounts.set(sym.name, (nameCounts.get(sym.name) || 0) + 1)
+    }
+
+    const currentFileSymbols = []
+    for (const sym of extracted.symbols || []) {
+      const symId = nameCounts.get(sym.name) > 1
+        ? formatExternalSymbolNodeId(ecosystem, moduleName, version, `${sym.name}@${sym.line}`)
+        : formatExternalSymbolNodeId(ecosystem, moduleName, version, sym.name)
+
+      symbolNamesSet.add(sym.name)
+      currentFileSymbols.push({ ...sym, id: symId })
+      symbolIndex.set(sym.name, [...(symbolIndex.get(sym.name) || []), symId])
+
+      edges.push({ from: moduleNodeId, to: symId, type: 'defines' })
+      symbolNodes.push(new GraphNode({
+        id: symId,
+        title: sym.name,
+        type: 'symbol',
+        metadata: { kind: sym.kind, line: sym.line }
+      }))
+    }
+
+    fileSymbolsMap.set(rel, { symbols: currentFileSymbols, calls: extracted.calls || [] })
+  }
+
+  // Internal calls between symbols inside the module
+  for (const [, { symbols, calls }] of fileSymbolsMap.entries()) {
+    for (const call of calls) {
+      let owner = null
+      for (const sym of symbols) {
+        const start = sym.line - 1
+        const end = start + sym.lines
+        if (call.row >= start && call.row < end) {
+          if (!owner || sym.lines < owner.lines) owner = sym
+        }
+      }
+      if (!owner) continue
+      const targetMatches = symbolIndex.get(call.name) || []
+      if (targetMatches.length === 1 && targetMatches[0] !== owner.id) {
+        edges.push({ from: owner.id, to: targetMatches[0], type: 'calls' })
+      }
+    }
+  }
+
+  const moduleNode = new GraphNode({
+    id: moduleNodeId,
+    title: `${moduleName}@${version}`,
+    type: 'module',
+    metadata: { ecosystem, name: moduleName, version }
+  })
+
+  return {
+    moduleNode,
+    symbolNodes,
+    edges,
+    symbols: [...symbolNamesSet].sort(),
+    name: moduleName,
+    version,
+    ecosystem,
+    model,
+    location
+  }
+}
+
+/**
+ * Sync the constructed module graph to AnchorDB in model modules-<ecosystem>.
+ */
+export async function syncModuleToAnchorDB(client, moduleGraph, options = {}) {
+  if (!client || !moduleGraph) return { linked: 0, saved: 0, failed: 0 }
+
+  const desired = desiredSourceEdgeKeys(moduleGraph.edges)
+  let linked = 0
+  let failed = 0
+
+  for (const key of desired) {
+    const [from, type, to] = key.split('\0')
+    try {
+      await client.link(from, type, to, 1.0)
+      linked += 1
+    } catch {
+      failed += 1
+    }
+  }
+
+  const pending = [
+    { name: moduleGraph.moduleNode.id, props: { ...moduleGraph.moduleNode.metadata, tombstone: 0 } },
+    ...moduleGraph.symbolNodes.map((s) => ({ name: s.id, props: { ...s.metadata, tombstone: 0 } }))
+  ]
+
+  let saved = 0
+  if (typeof client.saveBatch === 'function') {
+    try {
+      await client.saveBatch(pending)
+      saved = pending.length
+    } catch {
+      for (const item of pending) {
+        try {
+          await client.saveWithID(item.name, item.props)
+          saved += 1
+        } catch {
+          failed += 1
+        }
+      }
+    }
+  } else {
+    for (const item of pending) {
+      try {
+        await client.saveWithID(item.name, item.props)
+        saved += 1
+      } catch {
+        failed += 1
+      }
+    }
+  }
+
+  await updateModulesCatalog(moduleGraph.ecosystem, {
+    [`${moduleGraph.name}@${moduleGraph.version}`]: {
+      name: moduleGraph.name,
+      version: moduleGraph.version,
+      ecosystem: moduleGraph.ecosystem,
+      model: moduleGraph.model,
+      nodeId: moduleGraph.moduleNode.id,
+      symbols: moduleGraph.symbols
+    }
+  }, options)
+
+  return {
+    linked,
+    saved,
+    failed,
+    symbols: moduleGraph.symbols.length,
+    edges: moduleGraph.edges.length,
+    model: moduleGraph.model
+  }
+}
+
+/**
+ * Trace lineage for a module or symbol in its ecosystem model.
+ */
+export async function traceModuleSymbol(client, targetId, { direction = 'both', maxHops = 2 } = {}) {
+  const forwardRelations = targetId.includes('#') ? ['calls'] : ['defines']
+  const backwardRelations = targetId.includes('#') ? ['called_by', 'defined_in'] : []
+
+  const rels = []
+  if (direction === 'out' || direction === 'both') rels.push(...forwardRelations)
+  if (direction === 'in' || direction === 'both') rels.push(...backwardRelations)
+
+  const visited = new Map() // id -> hop
+  visited.set(targetId, 0)
+  let frontier = [targetId]
+
+  for (let hop = 1; hop <= maxHops && frontier.length; hop++) {
+    const nextFrontier = []
+    for (const current of frontier) {
+      for (const rel of rels) {
+        let rows = []
+        try {
+          rows = await client.causalChain([current], rel, 100)
+        } catch {
+          continue
+        }
+        for (const r of rows || []) {
+          if (!r?.word || visited.has(r.word)) continue
+          visited.set(r.word, hop)
+          nextFrontier.push(r.word)
+        }
+      }
+    }
+    frontier = nextFrontier
+  }
+
+  const results = []
+  for (const [id, hop] of visited.entries()) {
+    if (id !== targetId) results.push({ id, hop })
+  }
+  return results.sort((a, b) => a.hop - b.hop || a.id.localeCompare(b.id))
+}

package/server/graph/modules_catalog.js

@@ -0,0 +1,288 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
+import { knlGlobalDir } from '../config.js'
+
+/**
+ * Returns the path to the global modules catalog directory (~/.knlview/modules).
+ */
+export function modulesCatalogDir(home = null, options = {}) {
+  const base = knlGlobalDir(home, options)
+  return path.join(base, 'modules')
+}
+
+/**
+ * Default global AnchorDB model name for an ecosystem.
+ */
+export function defaultEcosystemModel(ecosystem) {
+  return `modules-${ecosystem}`
+}
+
+export function formatModuleKey(pkgName, version) {
+  return `${pkgName}@${version}`
+}
+
+export function formatExternalPackageNodeId(ecosystem, pkgName, version) {
+  return `ext:${ecosystem}/${pkgName}@${version}`
+}
+
+export function formatExternalSymbolNodeId(ecosystem, pkgName, version, symbol) {
+  return `ext:${ecosystem}/${pkgName}@${version}#${symbol}`
+}
+
+/**
+ * Read the catalog for a specific ecosystem ('npm', 'go', 'cargo', 'pypi').
+ */
+export async function readModulesCatalog(ecosystem, options = {}) {
+  const dir = modulesCatalogDir(options.home, options)
+  const file = path.join(dir, `${ecosystem}.json`)
+  try {
+    const raw = await fs.readFile(file, 'utf8')
+    const parsed = JSON.parse(raw)
+    return parsed && typeof parsed === 'object' ? parsed : {}
+  } catch {
+    return {}
+  }
+}
+
+/**
+ * Check if a module version is already indexed in the catalog.
+ */
+export async function isModuleSynced(ecosystem, pkgName, version, options = {}) {
+  const catalog = await readModulesCatalog(ecosystem, options)
+  const key = formatModuleKey(pkgName, version)
+  return Boolean(catalog[key]?.syncedAt)
+}
+
+/**
+ * Record discovered modules and symbols to ~/.knlview/modules/<ecosystem>.json.
+ */
+export async function updateModulesCatalog(ecosystem, entries, options = {}) {
+  if (!entries || !Object.keys(entries).length) return
+  const dir = modulesCatalogDir(options.home, options)
+  const file = path.join(dir, `${ecosystem}.json`)
+  try {
+    await fs.mkdir(dir, { recursive: true })
+    const existing = await readModulesCatalog(ecosystem, options)
+    for (const [key, item] of Object.entries(entries)) {
+      const prev = existing[key] || {
+        name: item.name,
+        version: item.version,
+        ecosystem,
+        model: item.model || defaultEcosystemModel(ecosystem),
+        nodeId: item.nodeId || formatExternalPackageNodeId(ecosystem, item.name, item.version),
+        symbols: []
+      }
+      const symSet = new Set([...(prev.symbols || []), ...(item.symbols || [])])
+      existing[key] = {
+        name: item.name,
+        version: item.version,
+        ecosystem,
+        model: item.model || prev.model || defaultEcosystemModel(ecosystem),
+        nodeId: item.nodeId || prev.nodeId || formatExternalPackageNodeId(ecosystem, item.name, item.version),
+        symbols: [...symSet].sort(),
+        syncedAt: item.syncedAt || prev.syncedAt || Date.now()
+      }
+    }
+    await fs.writeFile(file, JSON.stringify(existing, null, 2) + '\n', 'utf8')
+  } catch {
+    // Fail gracefully: catalog is an informational index, never fatal.
+  }
+}
+
+/**
+ * Detect project dependency versions from on-disk manifests:
+ * package.json, go.mod, Cargo.toml, requirements.txt.
+ */
+export async function detectProjectManifests(rootAbs) {
+  const manifests = {
+    npm: {},
+    go: {},
+    cargo: {},
+    pypi: {}
+  }
+
+  // 1. npm (package.json)
+  try {
+    const pkgRaw = await fs.readFile(path.join(rootAbs, 'package.json'), 'utf8')
+    const pkg = JSON.parse(pkgRaw)
+    const all = { ...pkg.dependencies, ...pkg.devDependencies, ...pkg.peerDependencies }
+    for (const [name, ver] of Object.entries(all)) {
+      if (typeof ver === 'string') {
+        manifests.npm[name] = ver.replace(/^[\^~>=<]/, '').trim() || 'latest'
+      }
+    }
+  } catch {}
+
+  // 2. Go (go.mod)
+  try {
+    const text = await fs.readFile(path.join(rootAbs, 'go.mod'), 'utf8')
+    const singleMatch = text.matchAll(/^require\s+([^\s()]+)\s+([^\s()]+)/gm)
+    for (const m of singleMatch) manifests.go[m[1]] = m[2]
+    const blockMatch = text.match(/require\s*\(([\s\S]*?)\)/m)
+    if (blockMatch) {
+      for (const line of blockMatch[1].split('\n')) {
+        const parts = line.trim().split(/\s+/)
+        if (parts.length >= 2 && !parts[0].startsWith('//')) {
+          manifests.go[parts[0]] = parts[1]
+        }
+      }
+    }
+  } catch {}
+
+  // 3. Rust (Cargo.toml)
+  try {
+    const text = await fs.readFile(path.join(rootAbs, 'Cargo.toml'), 'utf8')
+    let inDeps = false
+    for (const line of text.split('\n')) {
+      const trimmed = line.trim()
+      if (
+        trimmed.startsWith('[dependencies]') ||
+        trimmed.startsWith('[dev-dependencies]') ||
+        trimmed.startsWith('[build-dependencies]')
+      ) {
+        inDeps = true
+        continue
+      }
+      if (trimmed.startsWith('[')) {
+        inDeps = false
+        continue
+      }
+      if (inDeps && trimmed && !trimmed.startsWith('#')) {
+        const eqIdx = trimmed.indexOf('=')
+        if (eqIdx > 0) {
+          const name = trimmed.slice(0, eqIdx).trim()
+          const val = trimmed.slice(eqIdx + 1).trim()
+          let ver = 'unknown'
+          const strMatch = val.match(/^"([^"]+)"/)
+          if (strMatch) ver = strMatch[1]
+          const objMatch = val.match(/version\s*=\s*"([^"]+)"/)
+          if (objMatch) ver = objMatch[1]
+          manifests.cargo[name] = ver
+        }
+      }
+    }
+  } catch {}
+
+  // 4. Python (requirements.txt)
+  try {
+    const reqText = await fs.readFile(path.join(rootAbs, 'requirements.txt'), 'utf8')
+    for (const line of reqText.split('\n')) {
+      const trimmed = line.trim()
+      if (!trimmed || trimmed.startsWith('#')) continue
+      const m = trimmed.match(/^([a-zA-Z0-9_.-]+)(?:[=<>~!\s]*([a-zA-Z0-9_.-]+))?/)
+      if (m) manifests.pypi[m[1].toLowerCase()] = m[2] || 'unknown'
+    }
+  } catch {}
+
+  return manifests
+}
+
+/**
+ * Resolve the directory of an installed external module on disk.
+ * Supports npm (node_modules), Go (GOPATH/pkg/mod or vendor), Cargo (cargo cache or vendor),
+ * and Python (venv site-packages).
+ */
+export async function resolveModuleLocation(ecosystem, moduleName, options = {}) {
+  const root = path.resolve(options.projectRoot || process.cwd())
+  const manifests = options.manifests || await detectProjectManifests(root)
+
+  if (ecosystem === 'npm') {
+    // 1. Check node_modules in projectRoot and walk up if necessary
+    let curr = root
+    let foundDir = null
+    while (true) {
+      const candidate = path.join(curr, 'node_modules', moduleName)
+      try {
+        const stat = await fs.stat(candidate)
+        if (stat.isDirectory()) {
+          foundDir = candidate
+          break
+        }
+      } catch {}
+      const parent = path.dirname(curr)
+      if (parent === curr) break
+      curr = parent
+    }
+
+    let version = manifests.npm[moduleName] || 'unknown'
+    if (foundDir) {
+      try {
+        const pkgText = await fs.readFile(path.join(foundDir, 'package.json'), 'utf8')
+        const pkg = JSON.parse(pkgText)
+        if (pkg.version) version = pkg.version
+      } catch {}
+      return { location: foundDir, version, name: moduleName, ecosystem }
+    }
+  } else if (ecosystem === 'go') {
+    // 1. Check local vendor
+    const vendorDir = path.join(root, 'vendor', moduleName)
+    try {
+      const stat = await fs.stat(vendorDir)
+      if (stat.isDirectory()) {
+        return { location: vendorDir, version: manifests.go[moduleName] || 'vendor', name: moduleName, ecosystem }
+      }
+    } catch {}
+
+    // 2. Check GOPATH
+    const gopath = process.env.GOPATH || path.join(options.home || process.env.HOME || '', 'go')
+    const version = manifests.go[moduleName]
+    if (version) {
+      const modDir = path.join(gopath, 'pkg', 'mod', `${moduleName}@${version}`)
+      try {
+        const stat = await fs.stat(modDir)
+        if (stat.isDirectory()) {
+          return { location: modDir, version, name: moduleName, ecosystem }
+        }
+      } catch {}
+    }
+  } else if (ecosystem === 'cargo') {
+    // 1. Check local vendor
+    const vendorDir = path.join(root, 'vendor', moduleName)
+    try {
+      const stat = await fs.stat(vendorDir)
+      if (stat.isDirectory()) {
+        return { location: vendorDir, version: manifests.cargo[moduleName] || 'vendor', name: moduleName, ecosystem }
+      }
+    } catch {}
+
+    // 2. Check cargo registry cache (~/.cargo/registry/src)
+    const cargoHome = process.env.CARGO_HOME || path.join(options.home || process.env.HOME || '', '.cargo')
+    const regSrc = path.join(cargoHome, 'registry', 'src')
+    const version = manifests.cargo[moduleName]
+    try {
+      const registries = await fs.readdir(regSrc)
+      for (const reg of registries) {
+        const targetPrefix = version ? `${moduleName}-${version}` : `${moduleName}-`
+        const regDir = path.join(regSrc, reg)
+        const entries = await fs.readdir(regDir)
+        const match = entries.find((e) => version ? e === targetPrefix : e.startsWith(targetPrefix))
+        if (match) {
+          const location = path.join(regDir, match)
+          const matchedVersion = version || match.slice(moduleName.length + 1)
+          return { location, version: matchedVersion, name: moduleName, ecosystem }
+        }
+      }
+    } catch {}
+  } else if (ecosystem === 'pypi') {
+    const version = manifests.pypi[moduleName.toLowerCase()] || 'unknown'
+    // 1. Check .venv and venv site-packages
+    const venvDirs = ['.venv', 'venv']
+    for (const venv of venvDirs) {
+      const libDir = path.join(root, venv, 'lib')
+      try {
+        const pyDirs = await fs.readdir(libDir)
+        for (const py of pyDirs) {
+          const sp = path.join(libDir, py, 'site-packages', moduleName)
+          try {
+            const stat = await fs.stat(sp)
+            if (stat.isDirectory()) {
+              return { location: sp, version, name: moduleName, ecosystem }
+            }
+          } catch {}
+        }
+      } catch {}
+    }
+  }
+
+  return null
+}

package/server/graph/parser.js

@@ -36,7 +36,7 @@ export function extractFrontmatterAndTitle(content) {
 
 function parseSimpleYaml(text) {
   const result = {}
-  const lines = text.split('\n')
+  const lines = text.replace(/\r/g, '').split('\n')
   for (const line of lines) {
     const match = line.match(/^([a-zA-Z0-9_-]+)\s*:\s*(.*)$/)
     if (!match) continue
@@ -150,7 +150,8 @@ function appendList(existing, val) {
   return existing.concat(items.filter((item) => !existing.includes(item)))
 }
 
-function parseSingleDecisionBlock(relPath, rawBlock) {  const lines = rawBlock.split('\n')
+function parseSingleDecisionBlock(relPath, rawBlock) {
+  const lines = rawBlock.replace(/\r/g, '').split('\n')
   let title = 'Untitled Decision'
   let id = null
   let decidedAt = null