@rahmantech/knl-view 0.2.0 → 0.3.1

package/bin/knl-view.js

@@ -22,6 +22,19 @@ import {
 import { readProjectConfig, readGlobalConfig, projectSlug, anchordbClientOptions, resolveModel, resolveSettings, knlGlobalDir, globalConfigPath } from '../server/config.js'
 import { createAnchorWireClient, mirrorStillValid } from '../server/graph/anchordb_client.js'
 import { syncSourcesToAnchorDB, traceSources, purgeArchivedRecords } from '../server/graph/sources.js'
+import {
+  parseModuleSpecifier,
+  buildModuleGraph,
+  syncModuleToAnchorDB,
+  traceModuleSymbol
+} from '../server/graph/modules_builder.js'
+import {
+  readModulesCatalog,
+  defaultEcosystemModel,
+  detectProjectManifests,
+  isModuleSynced,
+  SUPPORTED_ECOSYSTEMS
+} from '../server/graph/modules_catalog.js'
 import { readModelRegistry, registerModel, generateModelName, modelRegistryPath } from '../server/models_registry.js'
 import {
   latestRelease,
@@ -97,6 +110,11 @@ Commands:
                             "sources" model via the REST collector
   sources trace --target=<path>  File lineage over the sources graph
                             (--hops, --direction in|out|both)
+  modules build-graph <target>  Build & sync external module graph into global
+                            AnchorDB model (modules-<eco>) on-demand
+                            Target: <eco>::<module_name> (e.g. npm::express)
+  modules trace <target> [sym]  Trace calls/definitions for an external module/symbol
+  modules list [ecosystem]  List locally indexed modules from ~/.knlview/modules
   setup                   Bootstrap project in one command (scaffold base, skill, and models)
 
 Options:
@@ -112,6 +130,7 @@ Options:
                           (default: both; see README "Two-way lineage")
   --target <id>           Target node id for the subgraph query
   --hops <number>         Graph trace hop radius (default: 2)
+  --force                 Rebuild external module graph even if already indexed
   --init                  Scaffold a new base via the wizard (does not start the server)
   --structure <layout>    Layout for non-interactive --init:
                           standard | empty | custom (default: standard)
@@ -146,6 +165,7 @@ try {
       target: { type: 'string' },
       direction: { type: 'string' },
       hops: { type: 'string', default: '2' },
+      force: { type: 'boolean', default: false },
       init: { type: 'boolean', default: false },
       structure: { type: 'string', default: 'standard' },
       folders: { type: 'string', default: '' },
@@ -245,6 +265,18 @@ if (sourcesSub) {
   process.exit(0)
 }
 
+// The "modules" command: on-demand external boundary graph in global AnchorDB
+// modules models (modules-npm, modules-go, modules-cargo, modules-pypi).
+const modulesSub = positionals?.[0] === 'modules' ? positionals?.[1] : null
+if (modulesSub) {
+  try {
+    await runModulesCommand(modulesSub, positionals?.slice(2), { values, settings })
+  } catch (err) {
+    fail(`Modules command failed: ${err.message}`)
+  }
+  process.exit(0)
+}
+
 // Register the project's whole models map to the global registry (updating
 // path/timestamp when this project owns them) and warn about names claimed
 // by other projects. This local registration needs no server connection.
@@ -1247,6 +1279,136 @@ async function runSourcesCollector(base, model, { quiet }) {
   }
 }
 
+// The "modules" subcommands: on-demand external boundary graph in global
+// AnchorDB modules models (modules-npm, modules-go, modules-cargo, modules-pypi).
+async function runModulesCommand(sub, args, { values, settings }) {
+  if (sub === 'list') {
+    const ecoFilter = args?.[0]?.toLowerCase()
+    if (ecoFilter && !SUPPORTED_ECOSYSTEMS.has(ecoFilter)) {
+      fail(`Unknown ecosystem "${args[0]}". Supported: ${[...SUPPORTED_ECOSYSTEMS].join(', ')}`)
+    }
+    const ecosystems = ecoFilter ? [ecoFilter] : [...SUPPORTED_ECOSYSTEMS]
+    const all = {}
+    for (const eco of ecosystems) {
+      all[eco] = await readModulesCatalog(eco)
+    }
+    process.stdout.write(JSON.stringify(all, null, 2) + '\n')
+    return
+  }
+
+  if (sub === 'build-graph' || sub === 'build') {
+    const target = args?.[0] || values.target
+    if (!target) {
+      fail('Target is required:\n  knl-view modules build-graph <ecosystem>::<module_name> [--force]')
+    }
+    const parsed = parseModuleSpecifier(target)
+    if (!parsed) {
+      fail(`Invalid module target "${target}".\nExpected format: <ecosystem>::<module_name> (e.g. npm::express, go::github.com/gin-gonic/gin)`)
+    }
+    const { ecosystem, moduleName } = parsed
+
+    if (!settings.anchordb) {
+      fail('External modules graphs live in AnchorDB, which is not configured.\n' +
+        'Run "knl-view install anchordb" first (optional layer).')
+    }
+
+    const model = defaultEcosystemModel(ecosystem)
+    const base = anchordbClientOptions(settings.anchordb)
+
+    if (!values.force) {
+      const manifestVersions = await detectProjectManifests(process.cwd())
+      const expectedVersion = manifestVersions[ecosystem]?.[moduleName]
+      if (expectedVersion) {
+        const already = await isModuleSynced(ecosystem, moduleName, expectedVersion)
+        if (already) {
+          const catalog = await readModulesCatalog(ecosystem)
+          const key = `${moduleName}@${expectedVersion}`
+          process.stdout.write(`module ${ecosystem}::${key} is already synced in model ${model} (${catalog[key]?.symbolCount || 0} symbols). Use --force to rebuild.\n`)
+          return
+        }
+      }
+    }
+
+    process.stdout.write(`building graph for ${ecosystem}::${moduleName} ...\n`)
+    const moduleGraph = await buildModuleGraph(ecosystem, moduleName, { projectRoot: process.cwd() })
+
+    const client = await createAnchorWireClient({ ...base, model })
+    try {
+      const result = await syncModuleToAnchorDB(client, moduleGraph)
+      process.stdout.write(`module synced: ${ecosystem}::${moduleGraph.name}@${moduleGraph.version} -> model "${model}" (${result.symbols} symbols, ${result.edges} edges, +${result.linked} linked)\n`)
+    } finally {
+      client.close()
+    }
+    return
+  }
+
+  if (sub === 'trace') {
+    const rawTarget = args?.[0] || values.target
+    if (!rawTarget) {
+      fail('Target is required:\n  knl-view modules trace <ecosystem>::<module_name> [symbol] [--hops=2] [--direction=in|out|both]')
+    }
+
+    let ecosystem, symbolId
+    if (rawTarget.startsWith('ext:')) {
+      const m = rawTarget.match(/^ext:([a-z]+)\//)
+      if (!m) fail(`Cannot determine ecosystem from node ID "${rawTarget}".`)
+      ecosystem = m[1]
+      symbolId = rawTarget
+    } else {
+      const parsed = parseModuleSpecifier(rawTarget)
+      if (!parsed) {
+        fail(`Invalid module target "${rawTarget}".\nExpected format: <ecosystem>::<module_name> [symbol]`)
+      }
+      ecosystem = parsed.ecosystem
+      const catalog = await readModulesCatalog(ecosystem)
+      const subSymbol = args?.[1]
+      const entryKey = Object.keys(catalog).find((k) => k.startsWith(parsed.moduleName + '@') || k === parsed.moduleName)
+      if (!entryKey) {
+        fail(`Module "${rawTarget}" is not indexed in catalog ~/.knlview/modules/${ecosystem}.json.\nRun "knl-view modules build-graph ${rawTarget}" first.`)
+      }
+      const entry = catalog[entryKey]
+      symbolId = subSymbol ? `${entry.nodeId}#${subSymbol}` : entry.nodeId
+    }
+
+    if (!settings.anchordb) {
+      fail('The modules graph lives in AnchorDB, which is not configured.\n' +
+        'Run "knl-view install anchordb" first.')
+    }
+
+    const model = defaultEcosystemModel(ecosystem)
+    const base = anchordbClientOptions(settings.anchordb)
+    const client = await createAnchorWireClient({ ...base, model })
+    try {
+      let direction = 'both'
+      if (values.direction !== undefined) {
+        if (!['in', 'out', 'both'].includes(values.direction)) {
+          fail(`--direction must be in, out, or both, got "${values.direction}".`)
+        }
+        direction = values.direction
+      }
+      const hops = Number.parseInt(values.hops || '2', 10)
+      const maxHops = Number.isInteger(hops) && hops > 0 ? hops : 2
+      const results = await traceModuleSymbol(client, symbolId, { direction, maxHops })
+      process.stdout.write(JSON.stringify({
+        center: symbolId,
+        model,
+        direction,
+        maxHops,
+        trace: results
+      }, null, 2) + '\n')
+    } finally {
+      client.close()
+    }
+    return
+  }
+
+  fail('Unknown modules subcommand.\n' +
+    'Usage:\n' +
+    '  knl-view modules build-graph <ecosystem>::<module_name> [--force]\n' +
+    '  knl-view modules trace <ecosystem>::<module_name> [symbol] [--hops=2] [--direction=in|out|both]\n' +
+    '  knl-view modules list [ecosystem]')
+}
+
 function fail(message) {
   process.stderr.write(`knl-view: ${message}\n`)
   process.exit(1)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rahmantech/knl-view",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "Local viewer for .knowledges knowledge bases: a CLI and a web app, with an optional AnchorDB graph layer.",
   "type": "module",
   "license": "Apache-2.0",

package/server/consensus.js

@@ -48,10 +48,6 @@ export function claimsIndexPath(model, home = null, options = {}) {
   return path.join(knlGlobalDir(home, options), 'claims', `${model}.json`)
 }
 
-export function consensusMirrorPath(model, home = null, options = {}) {
-  return path.join(knlGlobalDir(home, options), 'consensus', `${model}.json`)
-}
-
 /**
  * Read the claim index. Missing file = empty index, not an error.
  * @returns {Promise<{claims: object, error: string|null}>}

package/server/graph/modules_builder.js

@@ -0,0 +1,300 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
+import { GraphNode } from './model.js'
+import { SYNTAX_LANGUAGES, extractCode } from './syntax.js'
+import { desiredSourceEdgeKeys } from './sources.js'
+import {
+  resolveModuleLocation,
+  formatExternalPackageNodeId,
+  formatExternalSymbolNodeId,
+  defaultEcosystemModel,
+  updateModulesCatalog,
+  SUPPORTED_ECOSYSTEMS,
+  isValidModuleName
+} from './modules_catalog.js'
+
+const MODULE_DEFAULT_IGNORES = [
+  'node_modules',
+  '.git',
+  '.github',
+  'test',
+  'tests',
+  '__tests__',
+  'docs',
+  'doc',
+  'example',
+  'examples',
+  'fixtures',
+  'coverage'
+]
+
+/**
+ * Scan source files of an external module directory with depth and file count guard-rails.
+ */
+export async function scanModuleFiles(moduleDir, ecosystem, { maxDepth = 20, maxFiles = 5000 } = {}) {
+  const files = {}
+  let totalFiles = 0
+
+  const walk = async (dirAbs, rel, depth = 0) => {
+    if (depth > maxDepth || totalFiles >= maxFiles) return
+    let entries
+    try {
+      entries = await fs.readdir(dirAbs, { withFileTypes: true })
+    } catch {
+      return
+    }
+    for (const entry of entries) {
+      if (totalFiles >= maxFiles) break
+      const childRel = rel ? `${rel}/${entry.name}` : entry.name
+      if (entry.isDirectory()) {
+        if (MODULE_DEFAULT_IGNORES.includes(entry.name.toLowerCase())) continue
+        await walk(path.join(dirAbs, entry.name), childRel, depth + 1)
+        continue
+      }
+      if (!entry.isFile()) continue
+      const ext = path.extname(entry.name).toLowerCase()
+      if (!SYNTAX_LANGUAGES[ext]) continue
+      if (ecosystem === 'go' && entry.name.endsWith('_test.go')) continue
+      if (ecosystem === 'pypi' && entry.name.startsWith('test_')) continue
+      files[childRel] = path.join(dirAbs, entry.name)
+      totalFiles += 1
+    }
+  }
+  await walk(moduleDir, '', 0)
+  return files
+}
+
+/**
+ * Parse an eco::name string into components: { ecosystem, moduleName }
+ * Strictly validates ecosystem against SUPPORTED_ECOSYSTEMS and moduleName against path traversal.
+ */
+export function parseModuleSpecifier(target) {
+  if (!target || typeof target !== 'string') return null
+  const idx = target.indexOf('::')
+  if (idx <= 0) return null
+  const ecosystem = target.slice(0, idx).toLowerCase()
+  const moduleName = target.slice(idx + 2)
+  if (!moduleName || !SUPPORTED_ECOSYSTEMS.has(ecosystem) || !isValidModuleName(moduleName)) {
+    return null
+  }
+  return { ecosystem, moduleName }
+}
+
+/**
+ * Build graph for an external module from disk using Tree-Sitter.
+ */
+export async function buildModuleGraph(ecosystem, moduleName, options = {}) {
+  const resolved = options.location
+    ? { location: options.location, version: options.version || '0.0.0', name: moduleName, ecosystem }
+    : await resolveModuleLocation(ecosystem, moduleName, options)
+
+  if (!resolved || !resolved.location) {
+    throw new Error(`Could not resolve module directory for ${ecosystem}::${moduleName} on disk. Ensure it is installed.`)
+  }
+
+  const { location, version } = resolved
+  const model = defaultEcosystemModel(ecosystem)
+  const moduleNodeId = formatExternalPackageNodeId(ecosystem, moduleName, version)
+
+  const files = await scanModuleFiles(location, ecosystem)
+  const symbolNodes = []
+  const symbolNamesSet = new Set()
+  const edges = []
+  const symbolIndex = new Map() // name -> [id, ...]
+  const fileSymbolsMap = new Map()
+
+  for (const [rel, abs] of Object.entries(files)) {
+    const ext = path.extname(rel).toLowerCase()
+    let content
+    try {
+      content = await fs.readFile(abs, 'utf8')
+    } catch {
+      continue
+    }
+
+    let extracted
+    try {
+      extracted = await extractCode(ext, content)
+    } catch {
+      continue
+    }
+
+    const nameCounts = new Map()
+    for (const sym of extracted.symbols || []) {
+      nameCounts.set(sym.name, (nameCounts.get(sym.name) || 0) + 1)
+    }
+
+    const currentFileSymbols = []
+    for (const sym of extracted.symbols || []) {
+      const symId = nameCounts.get(sym.name) > 1
+        ? formatExternalSymbolNodeId(ecosystem, moduleName, version, `${sym.name}@${sym.line}`)
+        : formatExternalSymbolNodeId(ecosystem, moduleName, version, sym.name)
+
+      symbolNamesSet.add(sym.name)
+      currentFileSymbols.push({ ...sym, id: symId })
+      symbolIndex.set(sym.name, [...(symbolIndex.get(sym.name) || []), symId])
+
+      edges.push({ from: moduleNodeId, to: symId, type: 'defines' })
+      symbolNodes.push(new GraphNode({
+        id: symId,
+        title: sym.name,
+        type: 'symbol',
+        metadata: { kind: sym.kind, line: sym.line }
+      }))
+    }
+
+    fileSymbolsMap.set(rel, { symbols: currentFileSymbols, calls: extracted.calls || [] })
+  }
+
+  // Internal calls between symbols inside the module
+  for (const [, { symbols, calls }] of fileSymbolsMap.entries()) {
+    for (const call of calls) {
+      let owner = null
+      for (const sym of symbols) {
+        const start = sym.line - 1
+        const end = start + sym.lines
+        if (call.row >= start && call.row < end) {
+          if (!owner || sym.lines < owner.lines) owner = sym
+        }
+      }
+      if (!owner) continue
+      const targetMatches = symbolIndex.get(call.name) || []
+      if (targetMatches.length === 1 && targetMatches[0] !== owner.id) {
+        edges.push({ from: owner.id, to: targetMatches[0], type: 'calls' })
+      }
+    }
+  }
+
+  const moduleNode = new GraphNode({
+    id: moduleNodeId,
+    title: `${moduleName}@${version}`,
+    type: 'module',
+    metadata: { ecosystem, name: moduleName, version }
+  })
+
+  return {
+    moduleNode,
+    symbolNodes,
+    edges,
+    symbols: [...symbolNamesSet].sort(),
+    name: moduleName,
+    version,
+    ecosystem,
+    model,
+    location
+  }
+}
+
+/**
+ * Sync the constructed module graph to AnchorDB in model modules-<ecosystem>.
+ */
+export async function syncModuleToAnchorDB(client, moduleGraph, options = {}) {
+  if (!client || !moduleGraph) return { linked: 0, saved: 0, failed: 0 }
+
+  const desired = desiredSourceEdgeKeys(moduleGraph.edges)
+  let linked = 0
+  let failed = 0
+
+  for (const key of desired) {
+    const [from, type, to] = key.split('\0')
+    try {
+      await client.link(from, type, to, 1.0)
+      linked += 1
+    } catch {
+      failed += 1
+    }
+  }
+
+  const pending = [
+    { name: moduleGraph.moduleNode.id, props: { ...moduleGraph.moduleNode.metadata, tombstone: 0 } },
+    ...moduleGraph.symbolNodes.map((s) => ({ name: s.id, props: { ...s.metadata, tombstone: 0 } }))
+  ]
+
+  let saved = 0
+  if (typeof client.saveBatch === 'function') {
+    try {
+      await client.saveBatch(pending)
+      saved = pending.length
+    } catch {
+      for (const item of pending) {
+        try {
+          await client.saveWithID(item.name, item.props)
+          saved += 1
+        } catch {
+          failed += 1
+        }
+      }
+    }
+  } else {
+    for (const item of pending) {
+      try {
+        await client.saveWithID(item.name, item.props)
+        saved += 1
+      } catch {
+        failed += 1
+      }
+    }
+  }
+
+  await updateModulesCatalog(moduleGraph.ecosystem, {
+    [`${moduleGraph.name}@${moduleGraph.version}`]: {
+      name: moduleGraph.name,
+      version: moduleGraph.version,
+      ecosystem: moduleGraph.ecosystem,
+      model: moduleGraph.model,
+      nodeId: moduleGraph.moduleNode.id,
+      symbolCount: moduleGraph.symbols.length
+    }
+  }, options)
+
+  return {
+    linked,
+    saved,
+    failed,
+    symbols: moduleGraph.symbols.length,
+    edges: moduleGraph.edges.length,
+    model: moduleGraph.model
+  }
+}
+
+/**
+ * Trace lineage for a module or symbol in its ecosystem model.
+ */
+export async function traceModuleSymbol(client, targetId, { direction = 'both', maxHops = 2 } = {}) {
+  const forwardRelations = targetId.includes('#') ? ['calls'] : ['defines']
+  const backwardRelations = targetId.includes('#') ? ['called_by', 'defined_in'] : []
+
+  const rels = []
+  if (direction === 'out' || direction === 'both') rels.push(...forwardRelations)
+  if (direction === 'in' || direction === 'both') rels.push(...backwardRelations)
+
+  const visited = new Map() // id -> hop
+  visited.set(targetId, 0)
+  let frontier = [targetId]
+
+  for (let hop = 1; hop <= maxHops && frontier.length; hop++) {
+    const nextFrontier = []
+    for (const current of frontier) {
+      for (const rel of rels) {
+        let rows = []
+        try {
+          rows = await client.causalChain([current], rel, 100)
+        } catch {
+          continue
+        }
+        for (const r of rows || []) {
+          if (!r?.word || visited.has(r.word)) continue
+          visited.set(r.word, hop)
+          nextFrontier.push(r.word)
+        }
+      }
+    }
+    frontier = nextFrontier
+  }
+
+  const results = []
+  for (const [id, hop] of visited.entries()) {
+    if (id !== targetId) results.push({ id, hop })
+  }
+  return results.sort((a, b) => a.hop - b.hop || a.id.localeCompare(b.id))
+}